From 503f19aabe96cdfbe1a36c58a867cd6edbc1a73b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 16 Nov 2023 09:27:09 +0000 Subject: [PATCH 01/40] Sort cases --- Python/generated_cases.c.h | 9348 +++++++++++------------ Tools/cases_generator/generate_cases.py | 6 +- 2 files changed, 4679 insertions(+), 4675 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fe0cbfe6330e51..587efc9268afb0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,1230 +8,1703 @@ #endif #define TIER_ONE 1 - TARGET(NOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); - } - - TARGET(RESUME) { + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; - static_assert(0 == 0, "incorrect cache size"); - TIER_ONE_ONLY - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); } - this_instr->op.code = RESUME_CHECK; + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; } + STACK_GROW(1); + stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(RESUME_CHECK) { + TARGET(BEFORE_WITH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); -#if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } - - TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + INSTRUCTION_STATS(BEFORE_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ + */ + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); } - next_instr = this_instr; + GOTO_ERROR(error); } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = this_instr; - DISPATCH(); + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; } + STACK_GROW(1); + stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_FAST_CHECK) { + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *rhs; + PyObject *lhs; + PyObject *res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; + } + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_FAST) { + TARGET(BINARY_OP_ADD_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_FLOAT + { + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_FAST_AND_CLEAR) { + TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_INT + { + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_FAST_LOAD_FAST) { + TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); - STACK_GROW(2); - stack_pointer[-2] = value1; - stack_pointer[-1] = value2; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_UNICODE + { + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; + } + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_CONST) { + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_INPLACE_ADD_UNICODE + { + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } + STACK_SHRINK(2); DISPATCH(); } - TARGET(STORE_FAST) { + TARGET(BINARY_OP_MULTIPLY_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_FLOAT + { + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(STORE_FAST_LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); - stack_pointer[-1] = value2; - DISPATCH(); - } - - TARGET(STORE_FAST_STORE_FAST) { + TARGET(BINARY_OP_MULTIPLY_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - STACK_SHRINK(2); + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_INT + { + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(POP_TOP) { + TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_SUBTRACT_FLOAT + { + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(PUSH_NULL) { + TARGET(BINARY_OP_SUBTRACT_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + PyObject *right; + PyObject *left; PyObject *res; - res = NULL; - STACK_GROW(1); + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_SUBTRACT_INT + { + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); } - TARGET(END_FOR) { + TARGET(BINARY_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(END_FOR); - PyObject *value; - // POP_TOP - value = stack_pointer[-1]; - { - Py_DECREF(value); + INSTRUCTION_STATS(BINARY_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res = NULL; } - // POP_TOP - value = stack_pointer[-2]; - { - Py_DECREF(value); + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; STACK_SHRINK(2); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); - } - Py_DECREF(receiver); - Py_DECREF(value); - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(END_SEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); - } - Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(UNARY_NEGATIVE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NOT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL) { + TARGET(BINARY_SUBSCR) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *container; PyObject *res; - // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); + _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } - STAT_INC(TO_BOOL, deferred); + STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _TO_BOOL + // _BINARY_SUBSCR { - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; } + STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); } - TARGET(TO_BOOL_BOOL) { + TARGET(BINARY_SUBSCR_DICT) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - PyObject *value; - value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + PyObject *sub; + PyObject *dict; + PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(TO_BOOL_INT) { + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *cached = ht->_spec_cache.getitem; + DEOPT_IF(cached == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + uint32_t cached_version = ht->_spec_cache.getitem_version; + DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + Py_INCREF(getitem); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); + STACK_SHRINK(2); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(BINARY_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - PyObject *value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + PyObject *sub; + PyObject *list; PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - Py_DECREF(value); - res = Py_True; - } + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); } - TARGET(TO_BOOL_LIST) { + TARGET(BINARY_SUBSCR_STR_INT) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - PyObject *value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + PyObject *sub; + PyObject *str; PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); } - TARGET(TO_BOOL_NONE) { + TARGET(BINARY_SUBSCR_TUPLE_INT) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - PyObject *value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + PyObject *sub; + PyObject *tuple; PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); } - TARGET(TO_BOOL_STR) { + TARGET(BUILD_CONST_KEY_MAP) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + GOTO_ERROR(error); // Pop the keys and values. } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); } - stack_pointer[-1] = res; + Py_DECREF(keys); + if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } + STACK_SHRINK(oparg); + stack_pointer[-1] = map; DISPATCH(); } - TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; + TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + PyObject **values; + PyObject *list; + values = stack_pointer - oparg; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { STACK_SHRINK(oparg); goto error; } + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = list; DISPATCH(); } - TARGET(UNARY_INVERT) { + TARGET(BUILD_MAP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; + INSTRUCTION_STATS(BUILD_MAP); + PyObject **values; + PyObject *map; + values = stack_pointer - oparg*2; + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); + } + if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } + STACK_SHRINK(oparg*2); + STACK_GROW(1); + stack_pointer[-1] = map; DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_INT) { + TARGET(BUILD_SET) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + PyObject **values; + PyObject *set; + values = stack_pointer - oparg; + set = PySet_New(NULL); + if (set == NULL) + GOTO_ERROR(error); + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); } - // _BINARY_OP_MULTIPLY_INT - { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + if (err != 0) { + Py_DECREF(set); + if (true) { STACK_SHRINK(oparg); goto error; } } - STACK_SHRINK(1); - stack_pointer[-1] = res; + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = set; DISPATCH(); } - TARGET(BINARY_OP_ADD_INT) { + TARGET(BUILD_SLICE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_INT - { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } + STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-1] = slice; DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_INT) { + TARGET(BUILD_STRING) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_INT - { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + PyObject **pieces; + PyObject *str; + pieces = stack_pointer - oparg; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (str == NULL) { STACK_SHRINK(oparg); goto error; } + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = str; DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_FLOAT) { + TARGET(BUILD_TUPLE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_FLOAT - { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - STACK_SHRINK(1); - stack_pointer[-1] = res; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + PyObject **values; + PyObject *tup; + values = stack_pointer - oparg; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { STACK_SHRINK(oparg); goto error; } + STACK_SHRINK(oparg); + STACK_GROW(1); + stack_pointer[-1] = tup; DISPATCH(); } - TARGET(BINARY_OP_ADD_FLOAT) { + TARGET(CACHE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_FLOAT - { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(BINARY_OP_SUBTRACT_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_FLOAT - { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_UNICODE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_UNICODE - { - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; - } - STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - PyObject *right; - PyObject *left; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_INPLACE_ADD_UNICODE - { - assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - } - STACK_SHRINK(2); - DISPATCH(); + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); } - TARGET(BINARY_SUBSCR) { + TARGET(CALL) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; + // _SPECIALIZE_CALL + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); DISPATCH_SAME_OPARG(); } - STAT_INC(BINARY_SUBSCR, deferred); + STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _BINARY_SUBSCR + // _CALL { - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } } + STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + PyObject **args; + PyObject *null; + PyObject *callable; + args = stack_pointer - oparg; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + GOTO_ERROR(error); } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + Py_DECREF(tp); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; - DISPATCH(); + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + goto start_frame; } - TARGET(STORE_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + PyObject *null; + PyObject *callable; + PyObject *self; + PyObject *self_or_null; + PyObject *func; + PyObject **args; + _PyInterpreterFrame *new_frame; + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = stack_pointer - oparg; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + STACK_SHRINK(oparg); + STACK_SHRINK(2); + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + STORE_SP(); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - STACK_SHRINK(4); DISPATCH(); } - TARGET(BINARY_SUBSCR_LIST_INT) { + TARGET(CALL_BUILTIN_CLASS) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - PyObject *sub; - PyObject *list; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_STR_INT) { + TARGET(CALL_BUILTIN_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - PyObject *sub; - PyObject *str; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_TUPLE_INT) { + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - PyObject *sub; - PyObject *tuple; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_DICT) { + TARGET(CALL_BUILTIN_O) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - PyObject *sub; - PyObject *dict; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error; + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); + TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; + PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } + callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; + func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + GOTO_ERROR(error); + } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + GOTO_ERROR(error); + } + Py_SETREF(callargs, tuple); + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) GOTO_ERROR(error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyObject_Call(func, callargs, kwargs); + } + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + assert(PEEK(2 + (oparg & 1)) == NULL); + if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } + STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + stack_pointer[-1] = result; + CHECK_EVAL_BREAKER(); + DISPATCH(); } - TARGET(LIST_APPEND) { + TARGET(CALL_INTRINSIC_1) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + INSTRUCTION_STATS(CALL_INTRINSIC_1); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(SET_ADD) { + TARGET(CALL_INTRINSIC_2) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + PyObject *value1; + PyObject *value2; + PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(STORE_SUBSCR) { + TARGET(CALL_ISINSTANCE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; - PyObject *v; - // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - // _STORE_SUBSCR - v = stack_pointer[-3]; - { - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + GOTO_ERROR(error); } - STACK_SHRINK(3); - DISPATCH(); - } - - TARGET(STORE_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - STACK_SHRINK(3); - DISPATCH(); - } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - TARGET(STORE_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - STACK_SHRINK(3); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(DELETE_SUBSCR) { + TARGET(CALL_KW) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwnames; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + kwnames = stack_pointer[-1]; + args = stack_pointer - 1 - oparg; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames + ); + Py_DECREF(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + Py_DECREF(kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } + STACK_SHRINK(oparg); STACK_SHRINK(2); + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_INTRINSIC_1) { + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyObject *value; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + GOTO_ERROR(error); + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); } - TARGET(CALL_INTRINSIC_2) { + TARGET(CALL_LIST_APPEND) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + PyObject **args; + PyObject *self; + PyObject *callable; + args = stack_pointer - oparg; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); + DEOPT_IF(!PyList_Check(self), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } - TARGET(RAISE_VARARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; args = stack_pointer - oparg; - TIER_ONE_ONLY - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - if (true) { STACK_SHRINK(oparg); goto error; } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); + DISPATCH(); } - TARGET(INTERPRETER_EXIT) { + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; - } + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - TARGET(RETURN_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); - #if TIER_ONE - assert(frame != &entry_frame); - #endif - STORE_SP(); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } -#endif + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); + DISPATCH(); } - TARGET(RETURN_CONST) { + TARGET(CALL_METHOD_DESCRIPTOR_O) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_CONST); - PyObject *value; - PyObject *retval; - // LOAD_CONST - { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - // _POP_FRAME - retval = value; - { - assert(EMPTY()); - #if TIER_ONE - assert(frame != &entry_frame); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + PyObject *self_or_null; + PyObject *callable; + PyObject **args; + _PyInterpreterFrame *new_frame; + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = stack_pointer - oparg; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; #endif + } + // _PUSH_FRAME + STACK_SHRINK(oparg); + STACK_SHRINK(2); + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); STORE_SP(); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(frame->return_offset); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -1242,457 +1715,467 @@ DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_CONST) { + TARGET(CALL_PY_WITH_DEFAULTS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(GET_AITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(tstate->interp->eval_frame, CALL); + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; } - - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; + DEOPT_IF(argcount > code->co_argcount, CALL); + DEOPT_IF(argcount < min_args, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; } - - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); + new_frame->localsplus[i] = Py_NewRef(def); } - stack_pointer[-1] = iter; + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(GET_ANEXT) { + TARGET(CALL_TUPLE_1) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } + TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = stack_pointer - oparg; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = res; + DISPATCH(); + } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - GOTO_ERROR(error); - } + TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; + } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; + + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } - TARGET(GET_AWAITABLE) { + TARGET(CHECK_EXC_MATCH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + INSTRUCTION_STATS(CHECK_EXC_MATCH); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; } - Py_DECREF(iterable); + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; + DISPATCH(); + } - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } + TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; + PyObject *none; + PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + TIER_ONE_ONLY + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); + none = Py_None; } - - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; + else { + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + STACK_SHRINK(1); + stack_pointer[-2] = none; + stack_pointer[-1] = value; DISPATCH(); } - TARGET(SEND) { + TARGET(COMPARE_OP) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *receiver; - PyObject *v; - PyObject *retval; - // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_Send(receiver, next_instr); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } - STAT_INC(SEND, deferred); + STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _SEND - v = stack_pointer[-1]; + // _COMPARE_OP { - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); - } - else { - GOTO_ERROR(error); - } + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; } - Py_DECREF(v); } - stack_pointer[-1] = retval; + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(COMPARE_OP_FLOAT) { + frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - PyObject *v; - PyObject *receiver; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + STACK_SHRINK(1); + stack_pointer[-1] = res; + DISPATCH(); } - TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); + TARGET(COMPARE_OP_INT) { frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + STACK_SHRINK(1); + stack_pointer[-1] = res; + DISPATCH(); } - TARGET(YIELD_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); + TARGET(COMPARE_OP_STR) { frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + STACK_SHRINK(1); + stack_pointer[-1] = res; + DISPATCH(); } - TARGET(POP_EXCEPT) { + TARGET(CONTAINS_OP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); + INSTRUCTION_STATS(CONTAINS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; STACK_SHRINK(1); + stack_pointer[-1] = b; DISPATCH(); } - TARGET(RERAISE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; - TIER_ONE_ONLY - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); - } - } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + INSTRUCTION_STATS(CONVERT_VALUE); + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; + convertion_func_ptr conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = CONVERSION_FUNCTIONS[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; + DISPATCH(); } - TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(COPY) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; - TIER_ONE_ONLY - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - STACK_SHRINK(2); + INSTRUCTION_STATS(COPY); + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = Py_NewRef(bottom); + STACK_GROW(1); + stack_pointer[-1] = top; DISPATCH(); } - TARGET(CLEANUP_THROW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; - } - else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); } - STACK_SHRINK(1); - stack_pointer[-2] = none; - stack_pointer[-1] = value; DISPATCH(); } - TARGET(LOAD_ASSERTION_ERROR) { + TARGET(DELETE_ATTR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + INSTRUCTION_STATS(DELETE_ATTR); + PyObject *owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + STACK_SHRINK(1); DISPATCH(); } - TARGET(LOAD_BUILD_CLASS) { + TARGET(DELETE_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); } - STACK_GROW(1); - stack_pointer[-1] = bc; + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); DISPATCH(); } - TARGET(STORE_NAME) { + TARGET(DELETE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - PyObject *v; - v = stack_pointer[-1]; + INSTRUCTION_STATS(DELETE_FAST); + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; + err = PyDict_DelItem(GLOBALS(), name); + // Can't use ERROR_IF here. + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + GOTO_ERROR(error); } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); DISPATCH(); } @@ -1719,2827 +2202,2561 @@ DISPATCH(); } - TARGET(UNPACK_SEQUENCE) { + TARGET(DELETE_SUBSCR) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - } - STACK_SHRINK(1); - STACK_GROW(oparg); + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_2_error; + STACK_SHRINK(2); DISPATCH(); } - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + TARGET(DICT_MERGE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); STACK_SHRINK(1); - STACK_GROW(oparg); DISPATCH(); } - TARGET(UNPACK_SEQUENCE_TUPLE) { + TARGET(DICT_UPDATE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + if (true) goto pop_1_error; } - Py_DECREF(seq); + Py_DECREF(update); STACK_SHRINK(1); - STACK_GROW(oparg); DISPATCH(); } - TARGET(UNPACK_SEQUENCE_LIST) { + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; + TIER_ONE_ONLY + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + Py_DECREF(awaitable); + Py_DECREF(exc); + } + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(END_FOR) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + PyObject *value; + // POP_TOP + value = stack_pointer[-1]; + { + Py_DECREF(value); } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + // POP_TOP + value = stack_pointer[-2]; + { + Py_DECREF(value); + } + STACK_SHRINK(2); DISPATCH(); } - TARGET(UNPACK_EX) { + TARGET(END_SEND) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; - seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + INSTRUCTION_STATS(END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + Py_DECREF(receiver); + STACK_SHRINK(1); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(STORE_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *v; - // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_ATTR - v = stack_pointer[-2]; - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; - } - STACK_SHRINK(2); - DISPATCH(); - } - - TARGET(DELETE_ATTR) { + TARGET(ENTER_EXECUTOR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - STACK_SHRINK(1); - DISPATCH(); - } + INSTRUCTION_STATS(ENTER_EXECUTOR); + TIER_ONE_ONLY + CHECK_EVAL_BREAKER(); - TARGET(STORE_GLOBAL) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - STACK_SHRINK(1); - DISPATCH(); + Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } + frame = executor->execute(executor, frame, stack_pointer); + if (frame == NULL) { + frame = tstate->current_frame; + goto resume_with_error; + } + goto enter_tier_one; } - TARGET(DELETE_GLOBAL) { + TARGET(EXIT_INIT_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } + INSTRUCTION_STATS(EXIT_INIT_CHECK); + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } + STACK_SHRINK(1); DISPATCH(); } - TARGET(LOAD_LOCALS) { + TARGET(EXTENDED_ARG) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; - DISPATCH(); + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); } - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + TARGET(FORMAT_SIMPLE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + INSTRUCTION_STATS(FORMAT_SIMPLE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } + else { + res = value; } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_NAME) { + TARGET(FORMAT_WITH_SPEC) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - STACK_GROW(1); - stack_pointer[-1] = v; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + PyObject *fmt_spec; + PyObject *value; + PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_GLOBAL) { + TARGET(FOR_ITER) { frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_GLOBAL + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } - STAT_INC(LOAD_GLOBAL, deferred); + STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _LOAD_GLOBAL + // _FOR_ITER { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); } - if (true) goto error; + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - Py_INCREF(res); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } - } - } - null = NULL; + // Common case: no jump, leave it to the code generator } STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = next; DISPATCH(); } - TARGET(LOAD_GLOBAL_MODULE) { + TARGET(FOR_ITER_GEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - PyObject *res; - PyObject *null = NULL; - // _GUARD_GLOBALS_VERSION + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + PyObject *iter; + iter = stack_pointer[-1]; + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + PyGenObject *gen = (PyGenObject *)iter; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyFrame_StackPush(gen_frame, Py_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + PyObject *iter; + PyObject *next; + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } - // _LOAD_GLOBAL_MODULE + // _ITER_JUMP_LIST { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); } STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = next; DISPATCH(); } - TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - PyObject *res; - PyObject *null = NULL; - // _GUARD_GLOBALS_VERSION + TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + PyObject *iter; + PyObject *next; + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } - // _GUARD_BUILTINS_VERSION + // _ITER_JUMP_RANGE { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + Py_DECREF(r); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } } - // _LOAD_GLOBAL_BUILTINS + // _ITER_NEXT_RANGE { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; } STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = next; DISPATCH(); } - TARGET(DELETE_FAST) { + TARGET(FOR_ITER_TUPLE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + PyObject *iter; + PyObject *next; + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + STACK_GROW(1); + stack_pointer[-1] = next; DISPATCH(); } - TARGET(MAKE_CELL) { + TARGET(GET_AITER) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - GOTO_ERROR(error); + INSTRUCTION_STATS(GET_AITER); + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; } - SETLOCAL(oparg, cell); + + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + if (true) goto pop_1_error; + } + + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + if (true) goto pop_1_error; + } + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(DELETE_DEREF) { + TARGET(GET_ANEXT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - DISPATCH(); - } + INSTRUCTION_STATS(GET_ANEXT); + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); - TARGET(LOAD_FROM_DICT_OR_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); - } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { GOTO_ERROR(error); } - Py_INCREF(value); - } - Py_DECREF(class_dict); - stack_pointer[-1] = value; - DISPATCH(); - } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } - TARGET(LOAD_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + GOTO_ERROR(error); + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + GOTO_ERROR(error); + } + + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + + Py_DECREF(next_iter); + GOTO_ERROR(error); + } else { + Py_DECREF(next_iter); + } } - Py_INCREF(value); STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[-1] = awaitable; DISPATCH(); } - TARGET(STORE_DEREF) { + TARGET(GET_AWAITABLE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - STACK_SHRINK(1); - DISPATCH(); - } + INSTRUCTION_STATS(GET_AWAITABLE); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); - TARGET(COPY_FREE_VARS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - DISPATCH(); - } - TARGET(BUILD_STRING) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; - pieces = stack_pointer - oparg; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(BUILD_TUPLE) { + TARGET(GET_ITER) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; - values = stack_pointer - oparg; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + INSTRUCTION_STATS(GET_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(BUILD_LIST) { + TARGET(GET_LEN) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; - values = stack_pointer - oparg; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); + INSTRUCTION_STATS(GET_LEN); + PyObject *obj; + PyObject *len_o; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; STACK_GROW(1); - stack_pointer[-1] = list; + stack_pointer[-1] = len_o; DISPATCH(); } - TARGET(LIST_EXTEND) { + TARGET(GET_YIELD_FROM_ITER) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); PyObject *iterable; - PyObject *list; + PyObject *iter; iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + GOTO_ERROR(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + if (iter == NULL) { + GOTO_ERROR(error); } Py_DECREF(iterable); - if (true) goto pop_1_error; } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(SET_UPDATE) { + TARGET(IMPORT_FROM) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; - STACK_SHRINK(1); + INSTRUCTION_STATS(IMPORT_FROM); + PyObject *from; + PyObject *res; + from = stack_pointer[-1]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + STACK_GROW(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BUILD_SET) { + TARGET(IMPORT_NAME) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; - values = stack_pointer - oparg; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } - } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + INSTRUCTION_STATS(IMPORT_NAME); + PyObject *fromlist; + PyObject *level; + PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + STACK_SHRINK(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BUILD_MAP) { + TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + int is_meth = PEEK(oparg + 1) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 2); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(CALL); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; - values = stack_pointer - oparg*2; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); - } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - TARGET(SETUP_ANNOTATIONS) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error; - } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = PEEK(oparg + 2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 3); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PEEK(total_args + 1); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); } - TARGET(BUILD_CONST_KEY_MAP) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); } - Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + Py_DECREF(receiver); + Py_DECREF(value); + STACK_SHRINK(2); DISPATCH(); } - TARGET(DICT_UPDATE) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - PyObject *update; - PyObject *dict; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); } - Py_DECREF(update); - if (true) goto pop_1_error; + PyErr_SetRaisedException(NULL); } - Py_DECREF(update); + Py_DECREF(receiver); STACK_SHRINK(1); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(DICT_MERGE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; - PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr; } - Py_DECREF(update); - STACK_SHRINK(1); + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + oparg + 1; + } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(MAP_ADD) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); } - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + TARGET(INSTRUMENTED_JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); } - TARGET(LOAD_SUPER_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *class; - PyObject *global_super; - PyObject *self; - PyObject *attr; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_SUPER_ATTR - self = stack_pointer[-1]; - { - TIER_ONE_ONLY - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - if (err) goto pop_3_error; - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } - } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; - } - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(LOAD_SUPER_ATTR_ATTR) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_SUPER_ATTR_METHOD) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + PyObject *value = POP(); + int flag = Py_IsNone(value); + int offset; + if (flag) { + offset = oparg; } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; + else { + Py_DECREF(value); + offset = 0; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + PyObject *value = POP(); + int offset; + int nflag = Py_IsNone(value); + if (nflag) { + offset = 0; } - // _LOAD_ATTR - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; - } - } - else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - } + else { + Py_DECREF(value); + offset = oparg; } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_ATTR_INSTANCE_VALUE) { + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - DISPATCH(); - } - - TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_ATTR_WITH_HINT) { + TARGET(INSTRUMENTED_RESUME) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { + GOTO_ERROR(error); + } + next_instr = this_instr; } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = this_instr; + DISPATCH(); } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(LOAD_ATTR_SLOT) { + TARGET(INSTRUMENTED_RETURN_CONST) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - DISPATCH(); + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(LOAD_ATTR_CLASS) { + TARGET(INSTRUMENTED_RETURN_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - DISPATCH(); + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(LOAD_ATTR_PROPERTY) { + TARGET(INSTRUMENTED_YIELD_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; } - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); + TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; + } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). + TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + stack_pointer[-1] = b; + DISPATCH(); } - TARGET(STORE_ATTR_INSTANCE_VALUE) { + TARGET(JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - PyObject *owner; - PyObject *value; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _GUARD_DORV_VALUES - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); - } - // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #if ENABLE_SPECIALIZATION + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + /* We are using unsigned values, but we really want signed values, so + * do the 2s complement comparison manually */ + uint16_t ucounter = this_instr[1].cache + (1 << 15); + uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + OPT_STAT_INC(attempts); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + if (optimized < 0) goto error; + if (optimized) { + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } else { - Py_DECREF(old_value); + int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + if (backoff < MINIMUM_TIER2_BACKOFF) { + backoff = MINIMUM_TIER2_BACKOFF; + } + else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { + backoff++; + } + assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); + this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; } - Py_DECREF(owner); } - STACK_SHRINK(2); + #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } - TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); - } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - STACK_SHRINK(2); + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); DISPATCH(); } - TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - PyObject *owner; - PyObject *value; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _STORE_ATTR_SLOT - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); - } - STACK_SHRINK(2); + TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); DISPATCH(); } - TARGET(COMPARE_OP) { + TARGET(LIST_APPEND) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } + Py_DECREF(iterable); + if (true) goto pop_1_error; + } + assert(Py_IsNone(none_val)); + Py_DECREF(iterable); + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(LOAD_ASSERTION_ERROR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + STACK_GROW(1); + stack_pointer[-1] = value; + DISPATCH(); + } + + TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR + owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); + _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } - STAT_INC(COMPARE_OP, deferred); + STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _COMPARE_OP + // _LOAD_ATTR { - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } + } + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; } } - STACK_SHRINK(1); - stack_pointer[-1] = res; + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } DISPATCH(); } - TARGET(COMPARE_OP_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + // _CHECK_ATTR_CLASS + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + } + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(COMPARE_OP_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); - } + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(type_version != 0); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); - TARGET(COMPARE_OP_STR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + Py_INCREF(f); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = Py_NewRef(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(IS_OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; - DISPATCH(); - } - - TARGET(CONTAINS_OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONTAINS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; - DISPATCH(); - } - - TARGET(CHECK_EG_MATCH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(CHECK_EXC_MATCH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + PyObject *owner; + PyObject *attr; + PyObject *self; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; + // _CHECK_ATTR_METHOD_LAZY_DICT + { + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + STACK_GROW(1); + stack_pointer[-2] = attr; + stack_pointer[-1] = self; DISPATCH(); } - TARGET(IMPORT_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + PyObject *owner; + PyObject *attr; + PyObject *self; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + STACK_GROW(1); + stack_pointer[-2] = attr; + stack_pointer[-1] = self; DISPATCH(); } - TARGET(IMPORT_FROM) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; - from = stack_pointer[-1]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + PyObject *owner; + PyObject *attr; + PyObject *self; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; DISPATCH(); } - TARGET(JUMP_FORWARD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); - } - - TARGET(JUMP_BACKWARD) { + TARGET(LOAD_ATTR_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); - if (optimized < 0) goto error; - if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - } - else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; - } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; - } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; - } + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); } - #endif /* ENABLE_SPECIALIZATION */ + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - TIER_ONE_ONLY - CHECK_EVAL_BREAKER(); - - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; - Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { - current_executor = (_PyUOpExecutorObject *)executor; - GOTO_TIER_TWO(); + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + PyObject *owner; + PyObject *attr; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; - goto resume_with_error; + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); } - goto enter_tier_one; + stack_pointer[-1] = attr; + DISPATCH(); } - TARGET(POP_JUMP_IF_TRUE) { + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - STACK_SHRINK(1); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + PyObject *owner; + PyObject *attr; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; DISPATCH(); } - TARGET(POP_JUMP_IF_FALSE) { + TARGET(LOAD_ATTR_PROPERTY) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(type_version != 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); - DISPATCH(); + new_frame->localsplus[0] = owner; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(POP_JUMP_IF_NONE) { + TARGET(LOAD_ATTR_SLOT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _POP_JUMP_IF_TRUE - cond = b; + // _LOAD_ATTR_SLOT { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + uint16_t index = read_u16(&this_instr[4].cache); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - STACK_SHRINK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(POP_JUMP_IF_NOT_NONE) { + TARGET(LOAD_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; } else { - b = Py_False; - Py_DECREF(value); + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - // _POP_JUMP_IF_FALSE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - STACK_SHRINK(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + TARGET(LOAD_BUILD_CLASS) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; + } + STACK_GROW(1); + stack_pointer[-1] = bc; DISPATCH(); } - TARGET(GET_LEN) { + TARGET(LOAD_CONST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - PyObject *obj; - PyObject *len_o; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; + INSTRUCTION_STATS(LOAD_CONST); + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[-1] = value; DISPATCH(); } - TARGET(MATCH_CLASS) { + TARGET(LOAD_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - attrs = Py_None; // Failure! + INSTRUCTION_STATS(LOAD_DEREF); + PyObject *value; + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + Py_INCREF(value); + STACK_GROW(1); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(MATCH_MAPPING) { + TARGET(LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + INSTRUCTION_STATS(LOAD_FAST); + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[-1] = value; DISPATCH(); } - TARGET(MATCH_SEQUENCE) { + TARGET(LOAD_FAST_AND_CLEAR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[-1] = value; DISPATCH(); } - TARGET(MATCH_KEYS) { + TARGET(LOAD_FAST_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[-1] = value; DISPATCH(); } - TARGET(GET_ITER) { + TARGET(LOAD_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + STACK_GROW(2); + stack_pointer[-2] = value1; + stack_pointer[-1] = value2; DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { + TARGET(LOAD_FROM_DICT_OR_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - GOTO_ERROR(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + GOTO_ERROR(error); } - else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); GOTO_ERROR(error); } - Py_DECREF(iterable); + Py_INCREF(value); } - stack_pointer[-1] = iter; + Py_DECREF(class_dict); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(FOR_ITER) { + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } + } + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + DISPATCH(); + } + + TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } - STAT_INC(FOR_ITER, deferred); + STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _FOR_ITER + // _LOAD_GLOBAL { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + if (true) goto error; } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + Py_INCREF(res); } - // Common case: no jump, leave it to the code generator - } - STACK_GROW(1); - stack_pointer[-1] = next; - DISPATCH(); - } - - TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr; - } - else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + null = NULL; } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(FOR_ITER_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - PyObject *iter; - PyObject *next; - // _ITER_CHECK_LIST - iter = stack_pointer[-1]; + TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + PyObject *res; + PyObject *null = NULL; + // _GUARD_GLOBALS_VERSION { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); } - // _ITER_JUMP_LIST + // _GUARD_BUILTINS_VERSION { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); } - // _ITER_NEXT_LIST + // _LOAD_GLOBAL_BUILTINS { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; } STACK_GROW(1); - stack_pointer[-1] = next; + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(FOR_ITER_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - PyObject *iter; - PyObject *next; - // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE + TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + PyObject *res; + PyObject *null = NULL; + // _GUARD_GLOBALS_VERSION { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); } - // _ITER_NEXT_TUPLE + // _LOAD_GLOBAL_MODULE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; } STACK_GROW(1); - stack_pointer[-1] = next; + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(FOR_ITER_RANGE) { + TARGET(LOAD_LOCALS) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - PyObject *iter; - PyObject *next; - // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; - { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } + Py_INCREF(locals); STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[-1] = locals; DISPATCH(); } - TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - PyObject *iter; - iter = stack_pointer[-1]; - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(BEFORE_ASYNC_WITH) { + TARGET(LOAD_NAME) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BEFORE_ASYNC_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); + INSTRUCTION_STATS(LOAD_NAME); + PyObject *v; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { GOTO_ERROR(error); } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } } STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = v; DISPATCH(); } - TARGET(BEFORE_WITH) { + TARGET(LOAD_SUPER_ATTR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *class; + PyObject *global_super; + PyObject *self; + PyObject *attr; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); } - GOTO_ERROR(error); + STAT_INC(LOAD_SUPER_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); + // _LOAD_SUPER_ATTR + self = stack_pointer[-1]; + { + TIER_ONE_ONLY + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } + } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } DISPATCH(); } - TARGET(WITH_EXCEPT_START) { + TARGET(LOAD_SUPER_ATTR_ATTR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) goto pop_3_error; + STACK_SHRINK(2); + stack_pointer[-1] = attr; + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *self_or_null; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; } - else { - Py_DECREF(tb); + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + STACK_SHRINK(1); + stack_pointer[-2] = attr; + stack_pointer[-1] = self_or_null; DISPATCH(); } - TARGET(PUSH_EXC_INFO) { + TARGET(MAKE_CELL) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; - } - else { - prev_exc = Py_None; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + GOTO_ERROR(error); } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + SETLOCAL(oparg, cell); DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - PyObject *owner; - PyObject *attr; - PyObject *self; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + + Py_DECREF(codeobj); + if (func_obj == NULL) { + GOTO_ERROR(error); } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = (PyObject *)func_obj; + stack_pointer[-1] = func; + DISPATCH(); + } + + TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + PyObject *value; + PyObject *key; + PyObject *dict; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + attrs = Py_None; // Failure! } + STACK_SHRINK(2); + stack_pointer[-1] = attrs; + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = values_or_none; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - PyObject *owner; - PyObject *attr; - PyObject *self; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } + TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - PyObject *owner; - PyObject *attr; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; + TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; + STACK_GROW(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - PyObject *owner; - PyObject *attr; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + PyObject *cond; + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } } - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + // _POP_JUMP_IF_TRUE + cond = b; { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - stack_pointer[-1] = attr; + STACK_SHRINK(1); DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + TARGET(POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - PyObject *owner; - PyObject *attr; - PyObject *self; - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } } - // _LOAD_ATTR_METHOD_LAZY_DICT + // _POP_JUMP_IF_FALSE + cond = b; { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + STACK_SHRINK(1); DISPATCH(); } - TARGET(INSTRUMENTED_CALL) { + TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(CALL); + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + PyObject *cond; + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + STACK_SHRINK(1); + DISPATCH(); } - TARGET(CALL) { + TARGET(POP_TOP) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - // _SPECIALIZE_CALL - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; } - // _CALL - { - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + else { + prev_exc = Py_None; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + STACK_GROW(1); + stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; + DISPATCH(); + } + + TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + PyObject *res; + res = NULL; + STACK_GROW(1); stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + TARGET(RAISE_VARARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - PyObject *null; - PyObject *callable; - PyObject *self; - PyObject *self_or_null; - PyObject *func; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; - _PyInterpreterFrame *new_frame; - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + TIER_ONE_ONLY + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = args[1]; + /* fall through */ + case 1: + exc = args[0]; + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; + } + if (true) { STACK_SHRINK(oparg); goto error; } + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; + TIER_ONE_ONLY + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + GOTO_ERROR(error); } } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + + TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); + } + + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + static_assert(0 == 0, "incorrect cache size"); + TIER_ONE_ONLY + assert(frame == tstate->current_frame); + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; } - // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } - #endif + this_instr->op.code = RESUME_CHECK; } DISPATCH(); } - TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - PyObject *self_or_null; - PyObject *callable; - PyObject **args; - _PyInterpreterFrame *new_frame; - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); +#if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; +#endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + DISPATCH(); + } + + TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); + PyObject *value; + PyObject *retval; + // LOAD_CONST { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); } - // _SAVE_RETURN_OFFSET + // _POP_FRAME + retval = value; { + assert(EMPTY()); #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; + assert(frame != &entry_frame); #endif - } - // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); STORE_SP(); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(0); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -4550,1213 +4767,996 @@ DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(CALL_TYPE_1) { + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - DISPATCH(); + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(CALL_STR_1) { + TARGET(RETURN_VALUE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + assert(EMPTY()); + #if TIER_ONE + assert(frame != &entry_frame); + #endif + STORE_SP(); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif DISPATCH(); } - TARGET(CALL_TUPLE_1) { + TARGET(SEND) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - PyObject **args; - PyObject *null; - PyObject *callable; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - GOTO_ERROR(error); + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + PyObject *receiver; + PyObject *v; + PyObject *retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + // _SEND + v = stack_pointer[-1]; + { + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + GOTO_ERROR(error); + } + } + Py_DECREF(v); } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + stack_pointer[-1] = retval; + DISPATCH(); } - TARGET(EXIT_INIT_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); - } + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; STACK_SHRINK(1); - DISPATCH(); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); } - TARGET(CALL_BUILTIN_CLASS) { + TARGET(SETUP_ANNOTATIONS) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) goto error; } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); } - Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_BUILTIN_O) { + TARGET(SET_ADD) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_BUILTIN_FAST) { + TARGET(SET_FUNCTION_ATTRIBUTE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + stack_pointer[-1] = func; DISPATCH(); } - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + TARGET(SET_UPDATE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_LEN) { + TARGET(STORE_ATTR) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *v; + // _SPECIALIZE_STORE_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); + // _STORE_ATTR + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + STACK_SHRINK(2); DISPATCH(); } - TARGET(CALL_ISINSTANCE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + PyObject *owner; + PyObject *value; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); + // _GUARD_DORV_VALUES + { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + } + STACK_SHRINK(2); DISPATCH(); } - TARGET(CALL_LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - PyObject **args; - PyObject *self; - PyObject *callable; - args = stack_pointer - oparg; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_O) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + PyObject *owner; + PyObject *value; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + STACK_SHRINK(2); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); + STACK_SHRINK(2); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + TARGET(STORE_DEREF) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + TARGET(STORE_FAST) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); + TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; + DISPATCH(); } - TARGET(CALL_KW) { + TARGET(STORE_FAST_STORE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = stack_pointer - 1 - oparg; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) goto pop_1_error; } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + STACK_SHRINK(1); + DISPATCH(); + } + + TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + STACK_SHRINK(4); + DISPATCH(); + } + + TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *container; + PyObject *v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + // _STORE_SUBSCR + v = stack_pointer[-3]; + { + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(2); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + STACK_SHRINK(3); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + TARGET(STORE_SUBSCR_DICT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + STACK_SHRINK(3); + DISPATCH(); } - TARGET(CALL_FUNCTION_EX) { + TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + STACK_SHRINK(3); + DISPATCH(); + } + + TARGET(SWAP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } - callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - GOTO_ERROR(error); - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + INSTRUCTION_STATS(SWAP); + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + DISPATCH(); + } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); } - result = PyObject_Call(func, callargs, kwargs); + STAT_INC(TO_BOOL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; - CHECK_EVAL_BREAKER(); + // _TO_BOOL + { + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + uint32_t version = read_u32(&this_instr[2].cache); + // This one is a bit weird, because we expect *some* failures: + assert(version); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + STAT_INC(TO_BOOL, hit); + Py_DECREF(value); + res = Py_True; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(MAKE_FUNCTION) { + TARGET(TO_BOOL_BOOL) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; - - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } - - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + PyObject *value; + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); DISPATCH(); } - TARGET(SET_FUNCTION_ATTRIBUTE) { + TARGET(TO_BOOL_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortal(value)); + res = Py_False; } - STACK_SHRINK(1); - stack_pointer[-1] = func; + else { + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; DISPATCH(); } - TARGET(RETURN_GENERATOR) { + TARGET(TO_BOOL_LIST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); - } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_NONE) { frame->instr_ptr = next_instr; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - LOAD_IP(frame->return_offset); - goto resume_frame; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; + DISPATCH(); } - TARGET(BUILD_SLICE) { + TARGET(TO_BOOL_STR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value)); + res = Py_False; + } + else { + assert(Py_SIZE(value)); + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; DISPATCH(); } - TARGET(CONVERT_VALUE) { + TARGET(UNARY_INVERT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); + INSTRUCTION_STATS(UNARY_INVERT); PyObject *value; - PyObject *result; + PyObject *res; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); + res = PyNumber_Invert(value); Py_DECREF(value); - if (result == NULL) goto pop_1_error; - stack_pointer[-1] = result; + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(FORMAT_SIMPLE) { + TARGET(UNARY_NEGATIVE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); + INSTRUCTION_STATS(UNARY_NEGATIVE); PyObject *value; PyObject *res; value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - } - else { - res = value; - } + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); } - TARGET(FORMAT_WITH_SPEC) { + TARGET(UNARY_NOT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; + INSTRUCTION_STATS(UNARY_NOT); PyObject *value; PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; DISPATCH(); } - TARGET(COPY) { + TARGET(UNPACK_EX) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + INSTRUCTION_STATS(UNPACK_EX); + PyObject *seq; + seq = stack_pointer[-1]; + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } - TARGET(BINARY_OP) { + TARGET(UNPACK_SEQUENCE) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs; - PyObject *lhs; - PyObject *res; - // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } - STAT_INC(BINARY_OP, deferred); + STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); + (void)seq; + (void)counter; } - // _BINARY_OP + // _UNPACK_SEQUENCE { - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; } STACK_SHRINK(1); - stack_pointer[-1] = res; + STACK_GROW(oparg); DISPATCH(); } - TARGET(SWAP) { + TARGET(UNPACK_SEQUENCE_LIST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; - DISPATCH(); - } - - TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - if (next_opcode < 0) goto error; - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } - - TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + STACK_SHRINK(1); + STACK_GROW(oparg); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + STACK_SHRINK(1); + STACK_GROW(oparg); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + Py_DECREF(seq); + STACK_SHRINK(1); + STACK_GROW(oparg); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyObject *value = POP(); - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; - } - else { - Py_DECREF(value); - offset = 0; - } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_func: FOURTH = the context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyObject *value = POP(); - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; } else { - Py_DECREF(value); - offset = oparg; + Py_DECREF(tb); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; + STACK_GROW(1); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(EXTENDED_ARG) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TARGET(CACHE) { + TARGET(YIELD_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); - } - - TARGET(RESERVED) { + INSTRUCTION_STATS(YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; } #undef TIER_ONE diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index d1dbfeae8d74f6..695cef5764ae3e 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -769,6 +769,7 @@ def write_instructions( # Write and count instructions of all kinds n_macros = 0 + cases = [] for thing in self.everything: match thing: case parsing.InstDef(): @@ -776,11 +777,14 @@ def write_instructions( case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - stacking.write_macro_instr(mac, self.out) + cases.append((mac.name, mac)) case parsing.Pseudo(): pass case _: assert_never(thing) + cases.sort() + for _, mac in cases: + stacking.write_macro_instr(mac, self.out) self.out.write_raw("\n") self.out.write_raw("#undef TIER_ONE\n") From be741bcbc978520403c401b6ab6d2196ebe7971d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 16 Nov 2023 10:49:06 +0000 Subject: [PATCH 02/40] Reorganize code gen --- Include/internal/pycore_opcode_metadata.h | 6 +- Makefile.pre.in | 3 +- Python/abstract_interp_cases.c.h | 3 +- Python/bytecodes.c | 4 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 1912 +++++++++++---------- Tools/cases_generator/analysis.py | 4 +- Tools/cases_generator/analyzer.py | 410 +++++ Tools/cases_generator/cwriter.py | 89 + Tools/cases_generator/lexer.py | 4 +- Tools/cases_generator/parser.py | 41 + Tools/cases_generator/stack.py | 77 + Tools/cases_generator/tier1_generator.py | 312 ++++ 13 files changed, 1937 insertions(+), 930 deletions(-) create mode 100644 Tools/cases_generator/analyzer.py create mode 100644 Tools/cases_generator/cwriter.py create mode 100644 Tools/cases_generator/parser.py create mode 100644 Tools/cases_generator/stack.py create mode 100644 Tools/cases_generator/tier1_generator.py diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 69ac38829f9632..c73891990e8179 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1280,11 +1280,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case _INIT_CALL_PY_EXACT_ARGS: return 1; case _PUSH_FRAME: - return 1; + return 0; case CALL_BOUND_METHOD_EXACT_ARGS: - return 1; + return 0; case CALL_PY_EXACT_ARGS: - return 1; + return 0; case CALL_PY_WITH_DEFAULTS: return 1; case CALL_TYPE_1: diff --git a/Makefile.pre.in b/Makefile.pre.in index a74cb7ebc706d9..2061569f19093b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1587,7 +1587,6 @@ regen-cases: $(PYTHON_FOR_REGEN) \ $(srcdir)/Tools/cases_generator/generate_cases.py \ $(CASESFLAG) \ - -o $(srcdir)/Python/generated_cases.c.h.new \ -n $(srcdir)/Include/opcode_ids.h.new \ -t $(srcdir)/Python/opcode_targets.h.new \ -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ @@ -1595,6 +1594,8 @@ regen-cases: -p $(srcdir)/Lib/_opcode_metadata.py.new \ -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c + $(PYTHON_FOR_REGEN) \ + $(srcdir)/Tools/cases_generator/tier1_generator.py $(srcdir)/Python/bytecodes.c $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index a2f6aa8def8f69..2bd5f920490cd7 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -768,7 +768,8 @@ } case _PUSH_FRAME: { - PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a7dcb8416eb8c..c986e90aa0d745 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -800,11 +800,11 @@ dummy_func( // We also push it onto the stack on exit, but that's a // different frame, and it's accounted for by _PUSH_FRAME. op(_POP_FRAME, (retval --)) { - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif STORE_SP(); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -3110,7 +3110,7 @@ dummy_func( // The 'unused' output effect represents the return value // (which will be pushed when the frame returns). // It is needed so CALL_PY_EXACT_ARGS matches its family. - op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { + op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused if (0))) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4e29fb9f0fa93d..471abae6045210 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -652,11 +652,11 @@ PyObject *retval; retval = stack_pointer[-1]; STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif STORE_SP(); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 587efc9268afb0..dcc2cb05ac9ba7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,4 +1,4 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by /home/mark/cpython/Tools/cases_generator/tier1_generator.py // from: // Python/bytecodes.c // Do not edit! @@ -16,13 +16,14 @@ PyObject *exit; PyObject *res; mgr = stack_pointer[-1]; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); } GOTO_ERROR(error); } @@ -30,10 +31,10 @@ if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); GOTO_ERROR(error); @@ -45,12 +46,12 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(BEFORE_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -60,15 +61,14 @@ PyObject *res; mgr = stack_pointer[-1]; /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + * value returned from calling its __enter__ + */PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); } GOTO_ERROR(error); } @@ -76,10 +76,10 @@ if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); GOTO_ERROR(error); @@ -91,19 +91,18 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; PyObject *res; @@ -112,8 +111,9 @@ lhs = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); @@ -121,23 +121,24 @@ } STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } // _BINARY_OP { + assert(_PyEval_BinaryOps[oparg]); res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_ADD_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -154,17 +155,18 @@ } // _BINARY_OP_ADD_FLOAT { + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -181,17 +183,18 @@ } // _BINARY_OP_ADD_INT { + STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -203,22 +206,24 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { + STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -234,34 +239,34 @@ } // _BINARY_OP_INPLACE_ADD_UNICODE { + assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); + assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(BINARY_OP_MULTIPLY_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -278,17 +283,18 @@ } // _BINARY_OP_MULTIPLY_FLOAT { + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_MULTIPLY_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -305,17 +311,18 @@ } // _BINARY_OP_MULTIPLY_INT { + STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -327,22 +334,24 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_SUBTRACT_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -359,17 +368,18 @@ } // _BINARY_OP_SUBTRACT_INT { + STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -381,10 +391,11 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { res = NULL; } else { @@ -393,18 +404,17 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } - + TARGET(BINARY_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *res; @@ -414,7 +424,7 @@ { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); @@ -422,7 +432,7 @@ } STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _BINARY_SUBSCR { @@ -431,11 +441,11 @@ Py_DECREF(sub); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_DICT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -445,6 +455,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); @@ -454,11 +465,13 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + // not found or error + + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_GETITEM) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -489,7 +502,7 @@ frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(BINARY_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -501,9 +514,8 @@ list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); +DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -512,11 +524,11 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_STR_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -532,17 +544,17 @@ Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_TUPLE_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -554,9 +566,8 @@ tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); +DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -565,11 +576,11 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(BUILD_CONST_KEY_MAP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -578,89 +589,88 @@ PyObject **values; PyObject *map; keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; + if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + "bad BUILD_CONST_KEY_MAP keys argument"); + GOTO_ERROR(error); // Pop the keys and values. + } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; DISPATCH(); } - + TARGET(BUILD_LIST) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); PyObject **values; PyObject *list; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(BUILD_MAP) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); PyObject **values; PyObject *map; - values = stack_pointer - oparg*2; + values = &stack_pointer[-oparg*2]; + map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } - + TARGET(BUILD_SET) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); PyObject **values; PyObject *set; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; if (err == 0) - err = PySet_Add(set, item); + err = PySet_Add(set, item); Py_DECREF(item); } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(BUILD_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -669,80 +679,78 @@ PyObject *stop; PyObject *start; PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; + slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); DISPATCH(); } - + TARGET(BUILD_STRING) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); PyObject **pieces; PyObject *str; - pieces = stack_pointer - oparg; + pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(BUILD_TUPLE) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); PyObject **values; PyObject *tup; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(CACHE) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); Py_UNREACHABLE(); } - + TARGET(CALL) { frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; // _SPECIALIZE_CALL - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); @@ -750,12 +758,13 @@ } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _CALL { + // oparg counts all of the args, but *not* self: - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -771,7 +780,7 @@ callable = method; } // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { @@ -782,10 +791,10 @@ args, total_args, NULL ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -798,7 +807,7 @@ NULL); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -818,15 +827,14 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_ALLOC_AND_ENTER_INIT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -834,15 +842,15 @@ PyObject **args; PyObject *null; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */_PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -880,23 +888,23 @@ frame = tstate->current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; + * We don't check recursion depth here, + * as it will be checked after start_frame */tstate->py_recursion_remaining--; goto start_frame; } - + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); PyObject *null; PyObject *callable; + PyObject *func; PyObject *self; PyObject *self_or_null; - PyObject *func; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -910,12 +918,13 @@ } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { + STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = self; @@ -930,13 +939,15 @@ } // _CHECK_STACK_SPACE { + PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -952,37 +963,39 @@ } // _SAVE_RETURN_OFFSET { - #if TIER_ONE + +#if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO +#endif +#if TIER_TWO frame->return_offset = oparg; - #endif +#endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE +#if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif +#endif + } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(CALL_BUILTIN_CLASS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -991,9 +1004,10 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1009,14 +1023,13 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BUILTIN_FAST) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1025,9 +1038,10 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ int total_args = oparg; if (self_or_null != NULL) { @@ -1044,25 +1058,23 @@ args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1071,7 +1083,7 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -1085,24 +1097,22 @@ STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BUILTIN_O) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1111,7 +1121,7 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ @@ -1126,25 +1136,23 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1155,12 +1163,13 @@ PyObject *callargs; PyObject *func; PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } - callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; + // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { GOTO_ERROR(error); @@ -1177,7 +1186,7 @@ !PyFunction_Check(func) && !PyMethod_Check(func) ) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); @@ -1205,12 +1214,11 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { GOTO_ERROR(error); } @@ -1224,14 +1232,13 @@ Py_DECREF(callargs); Py_XDECREF(kwargs); assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_INTRINSIC_1) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1246,7 +1253,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(CALL_INTRINSIC_2) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1261,11 +1268,11 @@ Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(CALL_ISINSTANCE) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1274,9 +1281,10 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ int total_args = oparg; if (self_or_null != NULL) { @@ -1295,17 +1303,15 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - + TARGET(CALL_KW) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1318,11 +1324,12 @@ PyObject *callable; PyObject *res; kwnames = stack_pointer[-1]; - args = stack_pointer - 1 - oparg; + args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; + // oparg counts all of the args, but *not* self: - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -1339,7 +1346,7 @@ } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { @@ -1351,10 +1358,10 @@ ); Py_DECREF(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); + STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } assert(next_instr - this_instr == 1); @@ -1368,7 +1375,7 @@ kwnames); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -1389,14 +1396,13 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(2); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1405,7 +1411,7 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ @@ -1425,16 +1431,14 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - + TARGET(CALL_LIST_APPEND) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1442,9 +1446,10 @@ PyObject **args; PyObject *self; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); @@ -1452,17 +1457,17 @@ DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + goto pop_1_error; // Since arg is DECREF'ed already + } Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); + assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1471,9 +1476,10 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1488,7 +1494,7 @@ DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1497,14 +1503,13 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1513,7 +1518,7 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1531,23 +1536,21 @@ STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1556,7 +1559,7 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); @@ -1575,8 +1578,8 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); @@ -1584,14 +1587,13 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_O) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1600,7 +1602,7 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1619,8 +1621,8 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); @@ -1629,14 +1631,13 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_PY_EXACT_ARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1645,6 +1646,7 @@ PyObject *callable; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -1662,13 +1664,15 @@ } // _CHECK_STACK_SPACE { + PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -1684,37 +1688,39 @@ } // _SAVE_RETURN_OFFSET { - #if TIER_ONE + +#if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO +#endif +#if TIER_TWO frame->return_offset = oparg; - #endif +#endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE +#if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif +#endif + } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(CALL_PY_WITH_DEFAULTS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1722,7 +1728,7 @@ PyObject **args; PyObject *self_or_null; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); @@ -1754,11 +1760,11 @@ new_frame->localsplus[i] = Py_NewRef(def); } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + STACK_SHRINK(oparg + 2); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(CALL_STR_1) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1767,9 +1773,10 @@ PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); @@ -1777,15 +1784,14 @@ PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + Py_DECREF(&PyUnicode_Type); // I.e., callable +if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_TUPLE_1) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1794,7 +1800,7 @@ PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1804,15 +1810,14 @@ PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + Py_DECREF(&PyTuple_Type); // I.e., tuple +if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_TYPE_1) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1821,7 +1826,7 @@ PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1831,13 +1836,13 @@ STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + Py_DECREF(&PyType_Type); // I.e., callable + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - + TARGET(CHECK_EG_MATCH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1848,7 +1853,7 @@ PyObject *match; match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); if (true) goto pop_2_error; @@ -1857,7 +1862,7 @@ match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; @@ -1872,7 +1877,7 @@ stack_pointer[-1] = match; DISPATCH(); } - + TARGET(CHECK_EXC_MATCH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1884,8 +1889,8 @@ left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + Py_DECREF(right); + if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); @@ -1894,7 +1899,7 @@ stack_pointer[-1] = b; DISPATCH(); } - + TARGET(CLEANUP_THROW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -1922,19 +1927,18 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(1); - stack_pointer[-2] = none; - stack_pointer[-1] = value; + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - + TARGET(COMPARE_OP) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -1943,8 +1947,9 @@ left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); @@ -1952,7 +1957,7 @@ } STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _COMPARE_OP { @@ -1968,11 +1973,11 @@ res = res_bool ? Py_True : Py_False; } } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(COMPARE_OP_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1988,16 +1993,17 @@ double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); + int sign_ish = COMPARISON_BIT(dleft, dright); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(COMPARE_OP_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -2013,20 +2019,21 @@ DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); + _PyLong_DigitCount((PyLongObject *)right) <= 1); Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); + int sign_ish = COMPARISON_BIT(ileft, iright); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(COMPARE_OP_STR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -2048,11 +2055,12 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(CONTAINS_OP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2062,16 +2070,16 @@ PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); + int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } - + TARGET(CONVERT_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2079,7 +2087,8 @@ PyObject *value; PyObject *result; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + + convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); @@ -2088,7 +2097,7 @@ stack_pointer[-1] = result; DISPATCH(); } - + TARGET(COPY) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2096,13 +2105,14 @@ PyObject *bottom; PyObject *top; bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + stack_pointer[0] = top; + stack_pointer += 1; DISPATCH(); } - + TARGET(COPY_FREE_VARS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2119,7 +2129,7 @@ } DISPATCH(); } - + TARGET(DELETE_ATTR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2130,19 +2140,20 @@ int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(DELETE_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { + // Fortunately we don't need its superpower. + if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); GOTO_ERROR(error); } @@ -2150,7 +2161,7 @@ Py_DECREF(oldobj); DISPATCH(); } - + TARGET(DELETE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2160,7 +2171,7 @@ SETLOCAL(oparg, NULL); DISPATCH(); } - + TARGET(DELETE_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2169,16 +2180,16 @@ int err; err = PyDict_DelItem(GLOBALS(), name); // Can't use ERROR_IF here. - if (err != 0) { + if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } DISPATCH(); } - + TARGET(DELETE_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2188,20 +2199,20 @@ int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); + "no locals when deleting %R", name); GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. - if (err != 0) { + if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } DISPATCH(); } - + TARGET(DELETE_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2215,10 +2226,10 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(DICT_MERGE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2229,16 +2240,17 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; + if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(DICT_UPDATE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2250,17 +2262,17 @@ if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(END_ASYNC_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -2281,29 +2293,29 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(END_FOR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); PyObject *value; - // POP_TOP + // _POP_TOP value = stack_pointer[-1]; { Py_DECREF(value); } - // POP_TOP + // _POP_TOP value = stack_pointer[-2]; { Py_DECREF(value); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(END_SEND) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2312,19 +2324,20 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; + Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - + TARGET(ENTER_EXECUTOR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); + TIER_ONE_ONLY CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); @@ -2342,13 +2355,14 @@ } goto enter_tier_one; } - + TARGET(EXIT_INIT_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); PyObject *should_be_none; should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -2356,21 +2370,22 @@ Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(EXTENDED_ARG) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } - + TARGET(FORMAT_SIMPLE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2379,8 +2394,7 @@ PyObject *res; value = stack_pointer[-1]; /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { + * of format(value) is value itself. */if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -2391,7 +2405,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(FORMAT_WITH_SPEC) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2405,18 +2419,17 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(FOR_ITER) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; // _SPECIALIZE_FOR_ITER @@ -2424,7 +2437,7 @@ { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); @@ -2432,7 +2445,7 @@ } STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _FOR_ITER { @@ -2448,7 +2461,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2456,18 +2469,20 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator + } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - + TARGET(FOR_ITER_GEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; iter = stack_pointer[-1]; + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -2479,18 +2494,19 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - + TARGET(FOR_ITER_LIST) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { @@ -2498,6 +2514,7 @@ } // _ITER_JUMP_LIST { + _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); @@ -2516,6 +2533,7 @@ } // _ITER_NEXT_LIST { + _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -2523,25 +2541,28 @@ assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - + TARGET(FOR_ITER_RANGE) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); @@ -2549,12 +2570,13 @@ STACK_SHRINK(1); Py_DECREF(r); // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + JUMPBY(oparg + 1); DISPATCH(); } } // _ITER_NEXT_RANGE { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -2564,17 +2586,18 @@ next = PyLong_FromLong(value); if (next == NULL) goto error; } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - + TARGET(FOR_ITER_TUPLE) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { @@ -2582,6 +2605,7 @@ } // _ITER_JUMP_TUPLE { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); @@ -2600,6 +2624,7 @@ } // _ITER_NEXT_TUPLE { + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -2607,11 +2632,11 @@ assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - + TARGET(GET_AITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2621,16 +2646,15 @@ obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); Py_DECREF(obj); if (true) goto pop_1_error; } @@ -2640,19 +2664,19 @@ if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); Py_DECREF(iter); if (true) goto pop_1_error; } stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(GET_ANEXT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2663,13 +2687,13 @@ unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { GOTO_ERROR(error); } - } else { + } + else { if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } @@ -2682,9 +2706,9 @@ } else { _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); GOTO_ERROR(error); } @@ -2695,18 +2719,18 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); GOTO_ERROR(error); - } else { + } + else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } - + TARGET(GET_AWAITABLE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2715,11 +2739,9 @@ PyObject *iter; iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); if (iter != NULL && PyCoro_CheckExact(iter)) { @@ -2727,20 +2749,18 @@ if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); + being awaited on. */Py_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); + "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(GET_ITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2748,6 +2768,7 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -2755,7 +2776,7 @@ stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(GET_LEN) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2764,15 +2785,15 @@ PyObject *len_o; obj = stack_pointer[-1]; // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[0] = len_o; + stack_pointer += 1; DISPATCH(); } - + TARGET(GET_YIELD_FROM_ITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2780,15 +2801,15 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); + regular generator. */_PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); GOTO_ERROR(error); } iter = iterable; @@ -2807,7 +2828,7 @@ stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(IMPORT_FROM) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2819,11 +2840,11 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(IMPORT_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2839,35 +2860,37 @@ Py_DECREF(level); Py_DECREF(fromlist); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - + TARGET(INSTRUMENTED_CALL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); } - + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - + TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -2876,14 +2899,14 @@ int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } - + TARGET(INSTRUMENTED_END_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -2894,8 +2917,7 @@ receiver = stack_pointer[-2]; TIER_ONE_ONLY /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { + * to conform to PEP 380 */if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { GOTO_ERROR(error); @@ -2904,10 +2926,10 @@ } Py_DECREF(receiver); Py_DECREF(value); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(INSTRUMENTED_END_SEND) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -2925,15 +2947,16 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - + TARGET(INSTRUMENTED_FOR_ITER) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2951,7 +2974,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ @@ -2960,11 +2983,12 @@ INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_INSTRUCTION) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); if (next_opcode < 0) goto error; @@ -2976,7 +3000,7 @@ opcode = next_opcode; DISPATCH_GOTO(); } - + TARGET(INSTRUMENTED_JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -2985,7 +3009,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - + TARGET(INSTRUMENTED_JUMP_FORWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -2993,32 +3017,33 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + PyObject *cond = POP(); assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); int offset = flag * oparg; - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3033,13 +3058,13 @@ Py_DECREF(value); offset = 0; } - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3054,13 +3079,13 @@ Py_DECREF(value); offset = oparg; } - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif +#endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3069,13 +3094,13 @@ assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); int offset = flag * oparg; - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_RESUME) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -3094,7 +3119,7 @@ } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->instr_ptr != this_instr) { @@ -3105,15 +3130,16 @@ } DISPATCH(); } - + TARGET(INSTRUMENTED_RETURN_CONST) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); Py_INCREF(retval); assert(EMPTY()); @@ -3121,14 +3147,14 @@ _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); LOAD_IP(frame->return_offset); goto resume_frame; } - + TARGET(INSTRUMENTED_RETURN_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -3136,8 +3162,8 @@ PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); STACK_SHRINK(1); assert(EMPTY()); @@ -3145,20 +3171,21 @@ _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); LOAD_IP(frame->return_offset); goto resume_frame; } - + TARGET(INSTRUMENTED_YIELD_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; + assert(frame != &entry_frame); frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); @@ -3167,8 +3194,8 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); if (err) GOTO_ERROR(error); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -3182,7 +3209,7 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - + TARGET(INTERPRETER_EXIT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3197,7 +3224,7 @@ tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } - + TARGET(IS_OP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3207,15 +3234,16 @@ PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; + int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } - + TARGET(JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3223,20 +3251,19 @@ CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); + * do the 2s complement comparison manually */uint16_t ucounter = this_instr[1].cache + (1 << 15); uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); + assert(this_instr->op.code == ENTER_EXECUTOR); next_instr = this_instr; this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } @@ -3252,23 +3279,23 @@ this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; } } - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } - + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */JUMPBY(-oparg); DISPATCH(); } - + TARGET(JUMP_FORWARD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3276,7 +3303,7 @@ JUMPBY(oparg); DISPATCH(); } - + TARGET(LIST_APPEND) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3286,10 +3313,10 @@ v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(LIST_EXTEND) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3298,43 +3325,43 @@ PyObject *list; iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(LOAD_ASSERTION_ERROR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_ATTR) { frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -3342,8 +3369,9 @@ owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; @@ -3352,7 +3380,7 @@ } STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _LOAD_ATTR { @@ -3364,18 +3392,16 @@ /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } + */assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } else { /* meth is not an unbound method (but a regular attr, or something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); + */Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; } @@ -3387,12 +3413,12 @@ if (attr == NULL) goto pop_1_error; } } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_CLASS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3400,6 +3426,7 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { @@ -3408,6 +3435,7 @@ assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3417,12 +3445,12 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3432,6 +3460,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -3445,18 +3474,17 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(LOAD_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3464,6 +3492,7 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3474,6 +3503,7 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { + assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); @@ -3490,19 +3520,21 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3513,12 +3545,14 @@ } // _CHECK_ATTR_METHOD_LAZY_DICT { + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3529,19 +3563,20 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_METHOD_NO_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3550,9 +3585,11 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3561,19 +3598,20 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3584,6 +3622,7 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); @@ -3606,12 +3645,12 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3619,6 +3658,7 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { @@ -3642,18 +3682,20 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3662,9 +3704,11 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3673,15 +3717,17 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3692,6 +3738,7 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); @@ -3706,6 +3753,7 @@ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3713,9 +3761,10 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_PROPERTY) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3727,7 +3776,6 @@ PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); @@ -3742,12 +3790,12 @@ Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(LOAD_ATTR_SLOT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3755,6 +3803,7 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3766,6 +3815,7 @@ // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&this_instr[4].cache); + char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -3774,12 +3824,13 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -3787,6 +3838,7 @@ PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3797,6 +3849,7 @@ } // _CHECK_ATTR_WITH_HINT { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); @@ -3827,12 +3880,13 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_BUILD_CLASS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3841,14 +3895,14 @@ if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); + "__build_class__ not found"); if (true) goto error; } - STACK_GROW(1); - stack_pointer[-1] = bc; + stack_pointer[0] = bc; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_CONST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3856,16 +3910,17 @@ PyObject *value; value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; + PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -3873,11 +3928,11 @@ if (true) goto error; } Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3886,11 +3941,11 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST_AND_CLEAR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3898,12 +3953,12 @@ PyObject *value; value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3912,29 +3967,30 @@ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; + uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; value1 = GETLOCAL(oparg1); value2 = GETLOCAL(oparg2); Py_INCREF(value1); Py_INCREF(value2); - STACK_GROW(2); - stack_pointer[-2] = value1; - stack_pointer[-1] = value2; + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } - + TARGET(LOAD_FROM_DICT_OR_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3942,6 +3998,7 @@ PyObject *class_dict; PyObject *value; class_dict = stack_pointer[-1]; + PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -3962,7 +4019,7 @@ stack_pointer[-1] = value; DISPATCH(); } - + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3984,8 +4041,8 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } @@ -3994,21 +4051,21 @@ stack_pointer[-1] = v; DISPATCH(); } - + TARGET(LOAD_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; @@ -4017,7 +4074,7 @@ } STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _LOAD_GLOBAL { @@ -4026,14 +4083,13 @@ && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + * an exception if the key doesn't exist */_PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); } if (true) goto error; } @@ -4048,27 +4104,27 @@ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; if (res == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); if (true) goto error; } } } null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_GLOBAL_BUILTIN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4088,6 +4144,7 @@ // _LOAD_GLOBAL_BUILTINS { uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -4096,19 +4153,19 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_GLOBAL_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4117,9 +4174,11 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + /* Skip 1 cache entry */ // _LOAD_GLOBAL_MODULE { uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -4128,13 +4187,12 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_LOCALS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4143,24 +4201,25 @@ locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); + "no locals found"); if (true) goto error; } Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[0] = locals; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_NAME) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); PyObject *v; + PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); + "no locals found"); if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -4177,24 +4236,23 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_SUPER_ATTR) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *class; PyObject *global_super; PyObject *self; @@ -4205,8 +4263,9 @@ global_super = stack_pointer[-3]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -4215,7 +4274,7 @@ } STAT_INC(LOAD_SUPER_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR self = stack_pointer[-1]; @@ -4224,13 +4283,15 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); if (err) goto pop_3_error; } // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; + // handle any case whose performance we care about + PyObject *stack[] = { + class, self} + ; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -4258,13 +4319,12 @@ if (attr == NULL) goto pop_3_error; null = NULL; } - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_SUPER_ATTR_ATTR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4286,11 +4346,11 @@ Py_DECREF(class); Py_DECREF(self); if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_SUPER_ATTR_METHOD) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4311,7 +4371,7 @@ PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); if (attr == NULL) { @@ -4320,23 +4380,24 @@ } if (method_found) { self_or_null = self; // transfer ownership - } else { + } + else { Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; DISPATCH(); } - + TARGET(MAKE_CELL) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { GOTO_ERROR(error); @@ -4344,7 +4405,7 @@ SETLOCAL(oparg, cell); DISPATCH(); } - + TARGET(MAKE_FUNCTION) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4352,10 +4413,9 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - + PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - + PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { GOTO_ERROR(error); @@ -4367,7 +4427,7 @@ stack_pointer[-1] = func; DISPATCH(); } - + TARGET(MAP_ADD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4381,11 +4441,11 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); +if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + stack_pointer += -2; DISPATCH(); } - + TARGET(MATCH_CLASS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4397,25 +4457,27 @@ names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); + // None on failure. + assert(PyTuple_CheckExact(names)); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } + assert(PyTuple_CheckExact(attrs)); // Success! + } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; - attrs = Py_None; // Failure! - } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + // Error! + attrs = Py_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; DISPATCH(); } - + TARGET(MATCH_KEYS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4426,13 +4488,13 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[0] = values_or_none; + stack_pointer += 1; DISPATCH(); } - + TARGET(MATCH_MAPPING) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4442,11 +4504,11 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(MATCH_SEQUENCE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4456,18 +4518,18 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(NOP) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); DISPATCH(); } - + TARGET(POP_EXCEPT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4476,26 +4538,27 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -4519,15 +4582,15 @@ { assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -4551,15 +4614,15 @@ { assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -4568,14 +4631,14 @@ cond = stack_pointer[-1]; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif +#endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(POP_TOP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4583,10 +4646,10 @@ PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(PUSH_EXC_INFO) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4594,6 +4657,7 @@ PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -4603,53 +4667,56 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; DISPATCH(); } - + TARGET(PUSH_NULL) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); PyObject *res; res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(RAISE_VARARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { - case 2: + case 2: cause = args[1]; /* fall through */ - case 1: + case 1: exc = args[0]; /* fall through */ - case 0: + case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; - default: + default: _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); + "bad RAISE_VARARGS oparg"); break; } - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } + stack_pointer += -oparg; + DISPATCH(); } - + TARGET(RERAISE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -4657,7 +4724,8 @@ PyObject *exc; PyObject **values; exc = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; + TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { @@ -4678,27 +4746,27 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - + TARGET(RESERVED) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } - + TARGET(RESUME) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; - static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((code_version & 255) == 0); if (code_version != global_version) { @@ -4714,7 +4782,7 @@ } DISPATCH(); } - + TARGET(RESUME_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4729,48 +4797,51 @@ DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } - + TARGET(RETURN_CONST) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_CONST); PyObject *value; PyObject *retval; - // LOAD_CONST + // _LOAD_CONST { + value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); } // _POP_FRAME retval = value; { - assert(EMPTY()); - #if TIER_ONE + +#if TIER_ONE assert(frame != &entry_frame); - #endif - STORE_SP(); +#endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE +#if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif +#endif } DISPATCH(); } - + TARGET(RETURN_GENERATOR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); + assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4794,22 +4865,22 @@ LOAD_IP(frame->return_offset); goto resume_frame; } - + TARGET(RETURN_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); - #if TIER_ONE +#if TIER_ONE assert(frame != &entry_frame); - #endif - STORE_SP(); +#endif + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -4823,14 +4894,13 @@ #endif DISPATCH(); } - + TARGET(SEND) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *receiver; PyObject *v; PyObject *retval; @@ -4838,8 +4908,9 @@ receiver = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); @@ -4847,7 +4918,7 @@ } STAT_INC(SEND, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _SEND v = stack_pointer[-1]; @@ -4892,7 +4963,7 @@ stack_pointer[-1] = retval; DISPATCH(); } - + TARGET(SEND_GEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -4916,16 +4987,17 @@ frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - + TARGET(SETUP_ANNOTATIONS) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); + int err; PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); + "no locals found when setting up annotations"); if (true) goto error; } /* check if __annotations__ in locals()... */ @@ -4934,7 +5006,7 @@ ann_dict = PyDict_New(); if (ann_dict == NULL) goto error; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); + ann_dict); Py_DECREF(ann_dict); if (err) goto error; } @@ -4943,7 +5015,7 @@ } DISPATCH(); } - + TARGET(SET_ADD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4955,10 +5027,10 @@ int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(SET_FUNCTION_ATTRIBUTE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4967,35 +5039,36 @@ PyObject *attr; func = stack_pointer[-1]; attr = stack_pointer[-2]; + assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + stack_pointer[-2] = func; + stack_pointer += -1; DISPATCH(); } - + TARGET(SET_UPDATE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5007,17 +5080,16 @@ int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_ATTR) { frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; // _SPECIALIZE_STORE_ATTR @@ -5025,7 +5097,7 @@ { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -5034,7 +5106,7 @@ } STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _STORE_ATTR v = stack_pointer[-2]; @@ -5045,26 +5117,29 @@ Py_DECREF(owner); if (err) goto pop_2_error; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_VALUES { + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); @@ -5086,16 +5161,17 @@ } Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_ATTR_SLOT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5108,6 +5184,7 @@ value = stack_pointer[-2]; { uint16_t index = read_u16(&this_instr[4].cache); + char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -5115,10 +5192,10 @@ Py_XDECREF(old_value); Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -5167,10 +5244,10 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5181,10 +5258,10 @@ PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5192,10 +5269,10 @@ PyObject *value; value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5211,7 +5288,7 @@ stack_pointer[-1] = value2; DISPATCH(); } - + TARGET(STORE_FAST_STORE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5224,24 +5301,25 @@ uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); SETLOCAL(oparg2, value2); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); PyObject *v; v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5253,20 +5331,20 @@ int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); + "no locals found when storing %R", name); Py_DECREF(v); if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, v); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5291,17 +5369,16 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - STACK_SHRINK(4); + stack_pointer += -4; DISPATCH(); } - + TARGET(STORE_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *v; @@ -5310,8 +5387,9 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); @@ -5319,7 +5397,7 @@ } STAT_INC(STORE_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _STORE_SUBSCR v = stack_pointer[-3]; @@ -5331,10 +5409,10 @@ Py_DECREF(sub); if (err) goto pop_3_error; } - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } - + TARGET(STORE_SUBSCR_DICT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5350,10 +5428,10 @@ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } - + TARGET(STORE_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5366,24 +5444,22 @@ value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); +DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); +DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } - + TARGET(SWAP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5392,19 +5468,19 @@ PyObject *bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; + assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; DISPATCH(); } - + TARGET(TO_BOOL) { frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; // _SPECIALIZE_TO_BOOL @@ -5412,7 +5488,7 @@ { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); @@ -5420,7 +5496,7 @@ } STAT_INC(TO_BOOL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ } // _TO_BOOL { @@ -5432,7 +5508,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_ALWAYS_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -5441,8 +5517,9 @@ PyObject *res; value = stack_pointer[-1]; uint32_t version = read_u32(&this_instr[2].cache); + // This one is a bit weird, because we expect *some* failures: - assert(version); + assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); STAT_INC(TO_BOOL, hit); Py_DECREF(value); @@ -5450,7 +5527,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_BOOL) { frame->instr_ptr = next_instr; next_instr += 4; @@ -5461,7 +5538,7 @@ STAT_INC(TO_BOOL, hit); DISPATCH(); } - + TARGET(TO_BOOL_INT) { frame->instr_ptr = next_instr; next_instr += 4; @@ -5482,7 +5559,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_LIST) { frame->instr_ptr = next_instr; next_instr += 4; @@ -5497,7 +5574,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_NONE) { frame->instr_ptr = next_instr; next_instr += 4; @@ -5506,13 +5583,13 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); +DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_STR) { frame->instr_ptr = next_instr; next_instr += 4; @@ -5534,7 +5611,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(UNARY_INVERT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5542,13 +5619,14 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(UNARY_NEGATIVE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5562,7 +5640,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(UNARY_NOT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5575,35 +5653,35 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(UNPACK_EX) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); PyObject *seq; seq = stack_pointer[-1]; + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } - + TARGET(UNPACK_SEQUENCE) { frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - #if ENABLE_SPECIALIZATION +#if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -5611,22 +5689,22 @@ } STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ +#endif /* ENABLE_SPECIALIZATION */ (void)seq; (void)counter; } // _UNPACK_SEQUENCE { + PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; } - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(UNPACK_SEQUENCE_LIST) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5634,7 +5712,7 @@ PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5643,11 +5721,10 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(UNPACK_SEQUENCE_TUPLE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5655,7 +5732,7 @@ PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5664,11 +5741,10 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5676,7 +5752,7 @@ PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -5684,11 +5760,10 @@ values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(WITH_EXCEPT_START) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5700,6 +5775,7 @@ val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_func = stack_pointer[-4]; + /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -5707,9 +5783,7 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - + */PyObject *exc, *tb; assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -5721,15 +5795,17 @@ } assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; + PyObject *stack[4] = { + NULL, exc, val, tb} + ; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - + TARGET(YIELD_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -5737,9 +5813,9 @@ PyObject *retval; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -5758,5 +5834,5 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - + #undef TIER_ONE diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 603b15596f16de..76e625a42ea5cc 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -17,8 +17,8 @@ import parsing from parsing import StackEffect -BEGIN_MARKER = "// BEGIN BYTECODES //" -END_MARKER = "// END BYTECODES //" +BEGIN_MARKER = "// BEGIN BYTECODES //\n" +END_MARKER = "// END BYTECODES //\n" RESERVED_WORDS = { "co_consts": "Use FRAME_CO_CONSTS.", diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py new file mode 100644 index 00000000000000..7bab92d1de6db6 --- /dev/null +++ b/Tools/cases_generator/analyzer.py @@ -0,0 +1,410 @@ + +from dataclasses import dataclass +import lexer +import parser +from _typing_backports import assert_never + +@dataclass +class Properties: + escapes: bool + infallible: bool + deopts: bool + oparg: bool + jumps: bool + ends_with_eval_breaker: bool + needs_this: bool + always_exits: bool + stores_sp: bool + + def dump(self, indent): + print(indent, + "escapes:", self.escapes, + ", infallible:", self.infallible, + ", deopts:", self.deopts, + ", oparg:", self.oparg, + ", jumps:", self.jumps, + ", eval_breaker:", self.ends_with_eval_breaker, + ", needs_this:", self.needs_this, + ", always_exits:", self.always_exits, + ", stores_sp:", self.stores_sp, + sep="" + ) + + +SKIP_PROPERTIES = Properties( + False, + True, + False, + False, + False, + False, + False, + False, + False, +) + +@dataclass +class Skip: + "Unused cache entry" + size: int + + @property + def name(self): + return f"unused/{self.size}" + + @property + def properties(self): + return SKIP_PROPERTIES + +@dataclass +class StackItem: + name: str + type: str | None + condition: str | None + size: str + peek: bool = False + + def __str__(self): + cond = " if (" + self.condition + ")" if self.condition else "" + size = "[" + self.size + "]" if self.size != "1" else "" + type = "" if self.type is None else self.type + " " + return f"{type}{self.name}{size}{cond} {self.peek}" + + def is_array(self): + return self.type == "PyObject **" + +@dataclass +class StackEffect: + inputs: list[StackItem] + outputs: list[StackItem] + + def __str__(self): + return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" + +@dataclass +class CacheEntry: + name: str + size: int + + def __str__(self): + return f"{self.name}/{self.size}" + +@dataclass +class Uop: + name: str + context: parser.Context + annotations: list[str] + stack: StackEffect + caches: list[CacheEntry] + body: list[lexer.Token] + properties: Properties + _size: int = -1 + + def dump(self, indent): + print(indent, self.name, ", ".join(self.annotations) if self.annotations else "") + print(indent, self.stack, ", ".join([str(c) for c in self.caches])) + self.properties.dump(" " + indent) + + @property + def size(self): + if self._size < 0: + self._size = sum(c.size for c in self.caches) + return self._size + +Part = Uop | Skip + +@dataclass +class Instruction: + name: str + uops: list[Part] + _properties : Properties | None + is_target: bool = False + family: "Family" = None + + @property + def properties(self) -> Properties: + if self._properties is None: + self._properties = self._compute_properties() + return self._properties + + def _compute_properties(self) -> Properties: + return Properties( + any(part.properties.escapes for part in self.uops), + all(part.properties.infallible for part in self.uops), + any(part.properties.deopts for part in self.uops), + any(part.properties.oparg for part in self.uops), + any(part.properties.jumps for part in self.uops), + any(part.properties.ends_with_eval_breaker for part in self.uops), + any(part.properties.needs_this for part in self.uops), + any(part.properties.always_exits for part in self.uops), + any(part.properties.stores_sp for part in self.uops), + ) + + def dump(self, indent): + print(indent, self.name, "=", ", ".join([op.name for op in self.uops])) + self.properties.dump(" " + indent) + + @property + def size(self): + return 1 + sum(uop.size for uop in self.uops) + +@dataclass +class PseudoInstruction: + name: str + targets: list[Instruction] + + def dump(self, indent): + print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) + + +@dataclass +class Family: + name: str + members: list[Instruction] + + def dump(self, indent): + print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) + + +@dataclass +class Analysis: + instructions: dict[str, Instruction] + uops: dict[str, Uop] + families: dict[str, Family] + pseudos: dict[str, PseudoInstruction] + + +def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: + #To do -- support file and line output + # Construct a SyntaxError instance from message and token + return lexer.make_syntax_error( + message, "", tkn.line, tkn.column, "" + ) + +def override_error(name, context, prev_context, token) -> SyntaxError: + return analysis_error( + f"Duplicate definition of '{name}' @ {context} " + f"previous definition @ {prev_context}", + token + ) + +def convert_stack_item(item: parser.StackEffect) -> StackItem: + return StackItem(item.name, item.type, + item.cond, (item.size if item.size else "1")) + + +def analyze_stack(op: parser.InstDef) -> StackEffect: + inputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] + outputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.outputs] + for (input, output) in zip(inputs, outputs): + if input.name == output.name: + input.peek = output.peek = True + return StackEffect(inputs, outputs) + + +def analyze_caches(op: parser.InstDef) -> list[CacheEntry]: + caches: list[parser.CacheEffect] = [i for i in op.inputs if isinstance(i, parser.CacheEffect)] + return [ CacheEntry(i.name, int(i.size)) for i in caches ] + + +def variable_used(node: parser.InstDef, name: str) -> bool: + """Determine whether a variable with a given name is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + ) + +def is_infallible(op: parser.InstDef) -> bool: + return not ( + variable_used(op, "ERROR_IF") + or variable_used(op, "error") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + +from flags import makes_escaping_api_call + +EXITS = set([ + "DISPATCH", + "GO_TO_INSTRUCTION", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", +]) + +def eval_breaker_at_end(op: parser.InstDef): + return op.tokens[-5].text == "CHECK_EVAL_BREAKER" + +def always_exits(op: parser.InstDef): + depth = 0 + for tkn in op.tokens: + if tkn.kind == "LBRACE": + depth += 1 + elif tkn.kind == "RBRACE": + depth -= 1 + elif tkn.kind == "GOTO" or tkn.kind == "RETURN": + if depth <= 1: + return True + elif tkn.kind == "KEYWORD" or tkn.kind == "IDENTIFIER": + if depth <= 1 and tkn.text in EXITS: + return True + return False + +def compute_properties(op: parser.InstDef) -> Properties: + return Properties( + makes_escaping_api_call(op), + is_infallible(op), + variable_used(op, "DEOPT_IF"), + variable_used(op, "oparg"), + variable_used(op, "JUMPBY"), + eval_breaker_at_end(op), + variable_used(op, "this_instr"), + always_exits(op), + variable_used(op, "STORE_SP"), + ) + +def make_uop(name, op: parser.InstDef) -> Uop: + return Uop( + name, + op.context, + op.annotations, + analyze_stack(op), + analyze_caches(op), + op.block.tokens, + compute_properties(op), + ) + +def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: + assert op.kind == "op" + if op.name in uops: + if "override" not in op.annotations: + raise override_error(op.name, op.context, uops[op.name].context,op.tokens[0]) + uops[op.name] = make_uop(op.name, op) + +def add_instruction(name: str, parts: list[Part], instructions: dict[str, Instruction]): + instructions[name] = Instruction(name, parts, None) + +def desugar_inst(inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop]): + assert inst.kind == "inst" + name = inst.name + uop = make_uop("_" + inst.name, inst) + uops[inst.name] = uop + add_instruction(name, [uop], instructions) + +def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop]): + parts = [] + for part in macro.uops: + match part: + case parser.OpName(): + if part.name not in uops: + analysis_error(f"No Uop named {part.name}", macro.context, macro.tokens[0]) + parts.append(uops[part.name]) + case parser.CacheEffect(): + parts.append(Skip(part.size)) + case _: + assert_never(part) + assert(parts) + add_instruction(macro.name, parts, instructions) + +def add_family(family: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]): + family = Family( + family.name, + [ instructions[member] for member in family.members ] + ) + for member in family.members: + member.family = family + #The head of the family is an implicit jump target for DEOPTs + instructions[family.name].is_target = True + families[family.name] = family + +def add_pseudo(pseudo: parser.Pseudo, instructions: dict[str, Instruction], pseudos: dict[str, PseudoInstruction]): + pseudos[pseudo.name] = PseudoInstruction( + pseudo.name, + [ instructions[target] for target in pseudo.targets ] + ) + +def analyze_forest(forest: list[parser.AstNode]) -> Analysis: + instructions: dict[str, Instruction] = {} + uops: dict[str, Uop] = {} + families: dict[str, Family] = {} + pseudos: dict[str, PseudoInstruction] = {} + for node in forest: + match node: + case parser.InstDef(name): + if node.kind == "inst": + desugar_inst(node, instructions, uops) + else: + assert node.kind == "op" + add_op(node, uops) + case parser.Macro(): + pass + case parser.Family(): + pass + case parser.Pseudo(): + pass + case _: + assert_never(node) + for node in forest: + if isinstance(node, parser.Macro): + add_macro(node, instructions, uops) + for node in forest: + match node: + case parser.Family(): + add_family(node, instructions, families) + case parser.Pseudo(): + add_pseudo(node, instructions, pseudos) + case _: + pass + for uop in uops.values(): + tkn_iter = iter(uop.body) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text == "GO_TO_INSTRUCTION": + if next(tkn_iter).kind != "LPAREN": + continue + target = next(tkn_iter) + if target.kind != "IDENTIFIER": + continue + if target.text in instructions: + instructions[target.text].is_target = True + #Hack + instructions["BINARY_OP_INPLACE_ADD_UNICODE"].family = families["BINARY_OP"] + return Analysis(instructions, uops, families, pseudos) + +def analyze_file(filename: str) -> Analysis: + return analyze_forest(parser.parse_file(filename)) + +def dump_analysis(analysis: Analysis) -> None: + print("Uops:") + for i in analysis.uops.values(): + i.dump(" ") + print("Instructions:") + for i in analysis.instructions.values(): + i.dump(" ") + print("Families:") + for i in analysis.families.values(): + i.dump(" ") + print("Pseudos:") + for i in analysis.pseudos.values(): + i.dump(" ") + + +if __name__ == "__main__": + import sys + if len(sys.argv) < 2: + print("No input") + else: + filename = sys.argv[1] + dump_analysis(analyze_file(filename)) + + + + + + + + + + + + + diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py new file mode 100644 index 00000000000000..a82da61725cde2 --- /dev/null +++ b/Tools/cases_generator/cwriter.py @@ -0,0 +1,89 @@ + +from lexer import Token + +class CWriter: + 'A writer that understands how to format C code.' + + def __init__(self, out, indent = 0): + self.out = out + self.indent = indent + self.line = 1 + self.column = 0 + self.newline = False + + def set_position(self, tkn): + if self.line < tkn.line and not self.newline: + self.newline = True + self.out.write("\n") + self.line = tkn.end_line + self.column = tkn.end_column + + def emit_token(self, tkn): + if self.line < tkn.line: + self.line = tkn.line + self.newline = True + self.out.write("\n") + if self.newline: + self.column = 0 + elif self.column > 0 and self.column < tkn.column: + self.out.write(" ") * (tkn.column-self.column) + self.line = tkn.end_line + if tkn.kind == "COMMENT" and "\n" in tkn.text: + # Try to format multi-line comments sensibly + first = True + for line in tkn.text.splitlines(True): + if not first: + adjust = 1 if line[0] == "*" else 3 + self.out.write(" " * (self.column + adjust)) + self.emit_text(line.lstrip(" ")) + first = False + self.column = tkn.end_column + self.newline = False + return + self.column = tkn.end_column + if tkn.kind == "CMACRO": + self.out.write(tkn.text) + else: + self.emit_text(tkn.text) + + def emit_text(self, txt): + parens = txt.count("(") - txt.count(")") + if parens < 0: + self.indent += parens + if "}" in txt or txt.endswith(":"): + self.indent -= 1 + assert self.indent >= 0 + if self.newline: + self.out.write(" "*self.indent) + self.newline = False + txt = txt.lstrip(" ") + self.out.write(txt) + if txt.endswith("\n"): + self.line += 1 + self.newline = True + elif txt.endswith("{") or txt.startswith("}"): + if not self.newline: + self.out.write("\n") + self.line += 1 + self.newline = True + if "{" in txt or txt.endswith(":"): + self.indent += 1 + if parens > 0: + self.indent += parens + + def emit(self, txt): + if isinstance(txt, Token): + self.emit_token(txt) + elif isinstance(txt, str): + if txt.count("\n") > 1: + for line in txt.splitlines(True): + self.emit_text(line) + return + self.emit_text(txt) + else: + assert False + + def start_line(self): + if not self.newline: + self.out.write("\n") + self.newline = True diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 1185c855785939..7e1f6d0cd84f51 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -79,7 +79,7 @@ def choice(*opts: str) -> str: opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" +macro = r"#.*\n" CMACRO = "CMACRO" id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" @@ -112,7 +112,7 @@ def choice(*opts: str) -> str: char = r"\'.\'" # TODO: escape sequence CHARACTER = "CHARACTER" -comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +comment_re = r"(//.*\n)|/\*([^*]|\*[^/])*\*/" COMMENT = "COMMENT" newline = r"\n" diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py new file mode 100644 index 00000000000000..eb485aec5cce5e --- /dev/null +++ b/Tools/cases_generator/parser.py @@ -0,0 +1,41 @@ +from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName +from formatting import prettify_filename + + +BEGIN_MARKER = "// BEGIN BYTECODES //\n" +END_MARKER = "// END BYTECODES //\n" + +AstNode = InstDef | Macro | Pseudo | Family + +def parse_file(filename: str) -> None: + with open(filename) as file: + src = file.read() + + psr = Parser(src, filename=prettify_filename(filename)) + + # Skip until begin marker + while tkn := psr.next(raw=True): + if tkn.text == BEGIN_MARKER: + break + else: + raise psr.make_syntax_error( + f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" + ) + start = psr.getpos() + + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) + thing_first_token = psr.peek() + result: list[AstNode] = [] + while node := psr.definition(): + result.append(node) + if not psr.eof(): + psr.backup() + raise psr.make_syntax_error(f"Extra stuff at the end of {filename}", psr.next(True)) + return result diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py new file mode 100644 index 00000000000000..1bbf629627af33 --- /dev/null +++ b/Tools/cases_generator/stack.py @@ -0,0 +1,77 @@ +import sys +from analyzer import StackItem +from dataclasses import dataclass +from formatting import maybe_parenthesize + +def var_size(var): + if var.condition: + # Special case simplification + if var.condition == "oparg & 1" and var.size == "1": + return f"({var.condition})" + else: + return f"(({var.condition}) ? {var.size} : 0)" + else: + return var.size + +@dataclass +class StackOffset: + "The stack offset of the virtual base of the stack from the physical stack pointer" + + def __init__(self): + self.popped: list[str] = [] + self.pushed: list[str] = [] + + def pop(self, item: StackItem): + self.popped.append(var_size(item)) + + def push(self, item: StackItem): + self.pushed.append(var_size(item)) + + def simplify(self): + if not self.popped or not self.pushed: + return + popped = sorted(self.popped) + pushed = sorted(self.pushed) + self.popped = [] + self.pushed = [] + while popped and pushed: + pop = popped.pop() + push = pushed.pop() + if pop == push: + pass + elif pop > push: + self.popped.append(pop) + pushed.append(push) + else: + self.pushed.append(push) + popped.append(pop) + self.popped.extend(popped) + self.pushed.extend(pushed) + + def to_c(self): + self.simplify() + int_offset = 0 + symbol_offset = "" + for item in self.popped: + try: + int_offset -= int(item) + except ValueError: + symbol_offset += f" - {maybe_parenthesize(item)}" + for item in self.pushed: + try: + int_offset += int(item) + except ValueError: + symbol_offset += f" + {maybe_parenthesize(item)}" + if symbol_offset and not int_offset: + res = symbol_offset + else: + res = f"{int_offset}{symbol_offset}" + if res.startswith(" + "): + res = res[3:] + if res.startswith(" - "): + res = "-" + res[3:] + return res + + def clear(self): + self.popped = [] + self.pushed = [] diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py new file mode 100644 index 00000000000000..be7d2279c596f1 --- /dev/null +++ b/Tools/cases_generator/tier1_generator.py @@ -0,0 +1,312 @@ +import sys +from analyzer import Analysis, Instruction, Uop, Part, analyze_file, Skip, StackItem, analysis_error +from cwriter import CWriter +from typing import Tuple, TextIO, Iterator +from lexer import Token +from dataclasses import dataclass +from stack import StackOffset + +def write_header(filename: str, outfile: TextIO) -> None: + outfile.write(f"""// This file is generated by {__file__} +// from: +// {filename} +// Do not edit! + +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 +""") + +FOOTER = "#undef TIER_ONE\n" + +class SizeMismatch(Exception): + pass + + +class Stack: + + def __init__(self): + self.top_offset: StackOffset = StackOffset() + self.base_offset: StackOffset = StackOffset() + self.peek_offset: StackOffset = StackOffset() + self.variables: list[StackItem] = [] + self.defined: Set[str] = set() + + def pop(self, var: StackItem) -> str: + self.top_offset.pop(var) + if not var.peek: + self.peek_offset.pop(var) + indirect = "&" if var.is_array() else "" + if self.variables: + popped = self.variables.pop() + if popped.size != var.size: + raise SizeMismatch(f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}") + if popped.name == var.name: + return "" + elif popped.name == "unused": + self.defined.add(var.name) + return f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + elif var.name == "unused": + return "" + else: + self.defined.add(var.name) + return f"{var.name} = {popped.name};\n" + self.base_offset.pop(var) + if var.name == "unused": + return "" + else: + self.defined.add(var.name) + assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];" + if var.condition: + return f"if ({var.condition}) {{ {assign} }}\n" + return f"{assign}\n" + + def push(self, var: StackItem) -> None: + self.variables.append(var) + if var.is_array() and var.name not in self.defined and var.name != "unused": + c_offset = self.top_offset.to_c() + self.top_offset.push(var) + self.defined.add(var.name) + return f"{var.name} = &stack_pointer[{c_offset}];\n" + else: + self.top_offset.push(var) + return "" + + def flush(self, out: CWriter) -> None: + for var in self.variables: + if not var.peek: + if var.name != "unused" and not var.is_array(): + if var.condition: + out.emit(f" if ({var.condition}) ") + out.emit(f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n") + self.base_offset.push(var) + if self.base_offset.to_c() != self.top_offset.to_c(): + print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) + assert False + number = self.base_offset.to_c() + if number != "0": + out.emit(f"stack_pointer += {number};\n") + self.variables = [] + self.base_offset.clear() + self.top_offset.clear() + self.peek_offset.clear() + + def as_comment(self): + return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + + +def declare_variables(inst: Instruction, out: CWriter)->None: + variables = {"unused"} + for uop in inst.uops: + if isinstance(uop, Uop): + for var in reversed(uop.stack.inputs): + if var.name not in variables: + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type = var.type if var.type else "PyObject *" + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + out.emit(tkn) + +def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): + parens = 0 + out.emit("DEOPT_IF") + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(", ") + out.emit(inst.family.name) + out.emit(");") + +def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): + out.emit("if ") + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label) + out.emit(close) + +def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size != "1": + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit("}\n") + elif var.condition: + out.emit(f"Py_XDECREF({var.name});\n") + else: + out.emit(f"Py_DECREF({var.name});\n") + +def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + stack.flush(out) + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + +def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();") + + + +REPLACEMENT_FUNCTIONS = { + "DEOPT_IF" : replace_deopt, + "ERROR_IF" : replace_error, + "DECREF_INPUTS": replace_decrefs, + "CHECK_EVAL_BREAKER": replace_check_eval_breaker, + "STORE_SP": replace_store_sp, +} + +#Move this to formatter +def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None: + tkn_iter = iter(uop.body[1:-1]) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: + out.set_position(tkn) + REPLACEMENT_FUNCTIONS[tkn.text](out, tkn_iter, uop, stack, inst) + else: + out.emit(tkn) + +def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces:bool) -> int: + try: + out.start_line() + # out.emit(stack.as_comment() + "\n") + if isinstance(uop, Skip): + entries = "entries" if uop.size > 1 else "entry" + out.emit(f"/* Skip {uop.size} cache {entries} */\n") + return offset + uop.size + if braces: + out.emit(f"// {uop.name}\n") + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if braces: + out.emit("{") + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + reader = "read_obj" + else: + type = f"uint{cache.size*16}_t " + reader = f"read_u{cache.size*16}" + out.emit(f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n") + offset += cache.size + emit_tokens(out, uop, stack, inst) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + if braces: + out.start_line() + out.emit("}") + # out.emit(stack.as_comment() + "\n") + return offset + except SizeMismatch as ex: + raise analysis_error(ex, uop.body[0]) + +def uses_this(inst: Instruction) -> bool: + if inst.properties.needs_this: + return True + for uop in inst.uops: + if isinstance(uop, Skip): + continue + for cache in uop.caches: + if cache.name != "unused": + return True + return False + +def generate_tier1(filename: str, analysis: Analysis, outfile: TextIO) -> None: + write_header(filename, outfile) + out = CWriter(outfile, 2) + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + needs_this = uses_this(inst) + out.emit(f"TARGET({name}) {{") + if needs_this and not inst.is_target: + out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") + else: + out.emit(f"frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED({name});\n") + if needs_this: + out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") + #out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) + + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for uop in inst.uops: + #Only emit braces if more than one uop + offset = write_uop(uop, out, offset, stack, inst, len(inst.uops) > 1) + out.start_line() + if not inst.uops[-1].properties.always_exits: + stack.flush(out) + if inst.uops[-1].properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("DISPATCH();\n") + out.emit("}") + out.emit("\n") + outfile.write(FOOTER) + + +if __name__ == "__main__": + import sys + if len(sys.argv) not in (2, 3): + print(__file__, "INFILE", "OUTFILE") + else: + filename = sys.argv[1] + data = analyze_file(filename) + if len(sys.argv) == 3: + outfilename = sys.argv[2] + with open(outfilename, "w") as outfile: + generate_tier1(filename, data, outfile) + else: + generate_tier1(filename, data, sys.stdout) From 56e5001c3fce1b9cf56aef67dcc571928dbc0ee9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 00:49:10 +0000 Subject: [PATCH 03/40] Add type annotations --- Python/generated_cases.c.h | 60 ++++++++++++++++---------------- Tools/cases_generator/cwriter.py | 16 +++++---- Tools/cases_generator/lexer.py | 2 +- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dcc2cb05ac9ba7..b9666a15dabed5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -61,8 +61,8 @@ PyObject *res; mgr = stack_pointer[-1]; /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + * value returned from calling its __enter__ + */PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -245,16 +245,16 @@ DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */assert(Py_REFCNT(left) >= 2); + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); @@ -847,10 +847,10 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); callable = stack_pointer[-2 - oparg]; /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */_PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */_PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -888,8 +888,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); frame = tstate->current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */tstate->py_recursion_remaining--; + * We don't check recursion depth here, + * as it will be checked after start_frame */tstate->py_recursion_remaining--; goto start_frame; } @@ -1068,7 +1068,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. - */ + */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -2394,7 +2394,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *res; value = stack_pointer[-1]; /* If value is a unicode object, then we know the result - * of format(value) is value itself. */if (!PyUnicode_CheckExact(value)) { + * of format(value) is value itself. */if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -2917,7 +2917,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } receiver = stack_pointer[-2]; TIER_ONE_ONLY /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */if (PyGen_Check(receiver)) { + * to conform to PEP 380 */if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { GOTO_ERROR(error); @@ -3254,7 +3254,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } #if ENABLE_SPECIALIZATION this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */uint16_t ucounter = this_instr[1].cache + (1 << 15); + * do the 2s complement comparison manually */uint16_t ucounter = this_instr[1].cache + (1 << 15); uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); // Double-check that the opcode isn't instrumented or something: if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { @@ -3289,10 +3289,10 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */JUMPBY(-oparg); + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */JUMPBY(-oparg); DISPATCH(); } @@ -3392,7 +3392,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN - */assert(attr != NULL); // No errors on this branch + */assert(attr != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -3401,7 +3401,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */Py_DECREF(owner); + */Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; } @@ -4088,7 +4088,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */_PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + * an exception if the key doesn't exist */_PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } if (true) goto error; @@ -5783,7 +5783,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */PyObject *exc, *tb; + */PyObject *exc, *tb; assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index a82da61725cde2..fdd5e213c95122 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -1,24 +1,25 @@ from lexer import Token +from typing import TextIO class CWriter: 'A writer that understands how to format C code.' - def __init__(self, out, indent = 0): + def __init__(self, out: TextIO, indent: int = 0): self.out = out self.indent = indent - self.line = 1 + self.line = -1 self.column = 0 self.newline = False - def set_position(self, tkn): + def set_position(self, tkn: Token): if self.line < tkn.line and not self.newline: self.newline = True self.out.write("\n") self.line = tkn.end_line self.column = tkn.end_column - def emit_token(self, tkn): + def emit_token(self, tkn: Token): if self.line < tkn.line: self.line = tkn.line self.newline = True @@ -32,10 +33,11 @@ def emit_token(self, tkn): # Try to format multi-line comments sensibly first = True for line in tkn.text.splitlines(True): + line = line.lstrip(" ") if not first: adjust = 1 if line[0] == "*" else 3 self.out.write(" " * (self.column + adjust)) - self.emit_text(line.lstrip(" ")) + self.emit_text(line) first = False self.column = tkn.end_column self.newline = False @@ -46,7 +48,7 @@ def emit_token(self, tkn): else: self.emit_text(tkn.text) - def emit_text(self, txt): + def emit_text(self, txt: Token): parens = txt.count("(") - txt.count(")") if parens < 0: self.indent += parens @@ -71,7 +73,7 @@ def emit_text(self, txt): if parens > 0: self.indent += parens - def emit(self, txt): + def emit(self, txt: str | Token): if isinstance(txt, Token): self.emit_token(txt) elif isinstance(txt, str): diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 7e1f6d0cd84f51..7300b4fbb4da8e 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -359,4 +359,4 @@ def to_text(tkns: list[Token], dedent: int = 0) -> str: src = open(filename).read() # print(to_text(tokenize(src))) for tkn in tokenize(src, filename=filename): - print(tkn) + pass #print(tkn) From 3c35ba0ca90e6d214255e21ca1b06b1ed89b91e3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 03:06:02 +0000 Subject: [PATCH 04/40] Various improvements to formatting --- Python/generated_cases.c.h | 727 ++++++++++++----------- Tools/cases_generator/cwriter.py | 99 ++- Tools/cases_generator/lexer.py | 4 +- Tools/cases_generator/tier1_generator.py | 2 +- 4 files changed, 422 insertions(+), 410 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b9666a15dabed5..1411bd597b48b4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,7 +7,7 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 - + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -60,9 +60,10 @@ PyObject *exit; PyObject *res; mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ + */ + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -113,7 +114,7 @@ uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); @@ -121,7 +122,7 @@ } STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); } @@ -150,7 +151,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_FLOAT @@ -178,7 +179,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_INT @@ -234,7 +235,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_INPLACE_ADD_UNICODE @@ -245,22 +246,23 @@ DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */assert(Py_REFCNT(left) >= 2); + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); } stack_pointer += -2; @@ -278,7 +280,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_FLOAT @@ -306,7 +308,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_INT @@ -363,7 +365,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_INT @@ -393,9 +395,9 @@ container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { res = NULL; } else { @@ -424,7 +426,7 @@ { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); @@ -432,10 +434,11 @@ } STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _BINARY_SUBSCR { + res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); @@ -465,7 +468,7 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error; - // not found or error + // not found or error stack_pointer[-2] = res; stack_pointer += -1; @@ -480,7 +483,7 @@ PyObject *container; sub = stack_pointer[-1]; container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -512,10 +515,10 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) -DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -538,13 +541,13 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; @@ -564,10 +567,10 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) -DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -595,8 +598,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + GOTO_ERROR(error); // Pop the keys and values. + } map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -617,7 +620,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject **values; PyObject *list; values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); + list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { stack_pointer += -oparg; goto error; } stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -652,7 +655,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject **values; PyObject *set; values = &stack_pointer[-oparg]; - set = PySet_New(NULL); + set = PySet_New(NULL); if (set == NULL) GOTO_ERROR(error); int err = 0; @@ -700,7 +703,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject **pieces; PyObject *str; pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } @@ -717,7 +720,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject **values; PyObject *tup; values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); + tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { stack_pointer += -oparg; goto error; } stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -750,7 +753,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); @@ -758,13 +761,13 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _CALL { - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg counts all of the args, but *not* self: + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -779,8 +782,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(callable); callable = method; } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { @@ -790,11 +793,11 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -828,6 +831,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(args[i]); } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -850,7 +854,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack - * */_PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + * */ + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -889,7 +894,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. * We don't check recursion depth here, - * as it will be checked after start_frame */tstate->py_recursion_remaining--; + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; goto start_frame; } @@ -913,7 +919,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS @@ -921,16 +927,17 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = self; callable = func; { uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -949,7 +956,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; + int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; @@ -964,18 +971,18 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); // _SAVE_RETURN_OFFSET { -#if TIER_ONE + #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); -#endif -#if TIER_TWO + #endif + #if TIER_TWO frame->return_offset = oparg; -#endif + #endif } // _PUSH_FRAME { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; @@ -984,13 +991,12 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif - + #endif } stack_pointer += (((0) ? 1 : 0)); DISPATCH(); @@ -1024,6 +1030,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); } Py_DECREF(tp); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1065,10 +1072,11 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1086,7 +1094,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1107,6 +1115,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1124,7 +1133,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ + /* Builtin METH_O functions */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1135,9 +1144,9 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } PyObject *arg = args[0]; @@ -1147,6 +1156,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1167,9 +1177,9 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); callargs = stack_pointer[-1 - ((oparg & 1))]; func = stack_pointer[-3 - ((oparg & 1))]; - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { GOTO_ERROR(error); @@ -1217,8 +1227,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { GOTO_ERROR(error); } @@ -1233,6 +1243,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_XDECREF(kwargs); assert(PEEK(2 + (oparg & 1)) == NULL); if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + stack_pointer[-3 - ((oparg & 1))] = result; stack_pointer += -2 - ((oparg & 1)); CHECK_EVAL_BREAKER(); @@ -1246,7 +1257,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject *value; PyObject *res; value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); + assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -1263,7 +1274,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject *res; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); + assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); Py_DECREF(value1); @@ -1328,8 +1339,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg counts all of the args, but *not* self: + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -1345,8 +1356,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); callable = method; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { @@ -1357,11 +1368,11 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args, positional_args, kwnames ); Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } assert(next_instr - this_instr == 1); @@ -1397,6 +1408,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(args[i]); } if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; CHECK_EVAL_BREAKER(); @@ -1414,7 +1426,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* len(o) */ + /* len(o) */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1457,13 +1469,13 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + goto pop_1_error; // Since arg is DECREF'ed already + } Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); DISPATCH(); } @@ -1504,6 +1516,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1521,7 +1534,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -1545,6 +1558,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1562,7 +1576,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); + assert(oparg == 0 || oparg == 1); int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1577,9 +1591,9 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); @@ -1588,6 +1602,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1605,7 +1620,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -1620,9 +1635,9 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); @@ -1632,6 +1647,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1674,7 +1690,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; + int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; @@ -1689,18 +1705,18 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); // _SAVE_RETURN_OFFSET { -#if TIER_ONE + #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); -#endif -#if TIER_TWO + #endif + #if TIER_TWO frame->return_offset = oparg; -#endif + #endif } // _PUSH_FRAME { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; @@ -1709,13 +1725,12 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif - + #endif } stack_pointer += (((0) ? 1 : 0)); DISPATCH(); @@ -1732,6 +1747,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(tstate->interp->eval_frame, CALL); int argcount = oparg; if (self_or_null != NULL) { @@ -1759,8 +1775,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } @@ -1784,8 +1800,9 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable -if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1803,15 +1820,16 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); + assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple -if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1829,14 +1847,14 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); + assert(oparg == 1); DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable + Py_DECREF(&PyType_Type); // I.e., callable stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1853,12 +1871,11 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *match; match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); if (true) goto pop_2_error; } - match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, @@ -1887,12 +1904,11 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); + assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); if (true) goto pop_1_error; } - int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -1912,7 +1928,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } exc_value = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY + TIER_ONE_ONLY assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1949,7 +1965,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); @@ -1957,10 +1973,11 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _COMPARE_OP { + assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); @@ -1987,17 +2004,17 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; @@ -2013,7 +2030,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); @@ -2022,12 +2039,12 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } _PyLong_DigitCount((PyLongObject *)right) <= 1); Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; @@ -2043,7 +2060,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); @@ -2054,7 +2071,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; @@ -2070,7 +2087,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); + int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); if (res < 0) goto pop_2_error; @@ -2088,7 +2105,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *result; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); @@ -2151,9 +2168,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); GOTO_ERROR(error); } @@ -2179,8 +2196,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { + // Can't use ERROR_IF here. + if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -2203,8 +2220,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { + // Can't use ERROR_IF here. + if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -2259,7 +2276,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *dict; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", @@ -2393,8 +2410,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *value; PyObject *res; value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */if (!PyUnicode_CheckExact(value)) { + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -2415,7 +2433,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); + res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; @@ -2437,7 +2455,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); @@ -2445,10 +2463,11 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _FOR_ITER { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next == NULL) { @@ -2468,7 +2487,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } JUMPBY(oparg + 1); DISPATCH(); } - // Common case: no jump, leave it to the code generator + // Common case: no jump, leave it to the code generator } stack_pointer[0] = next; @@ -2510,7 +2529,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -2569,8 +2588,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); DISPATCH(); } } @@ -2601,7 +2620,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -2644,12 +2663,11 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *obj; PyObject *iter; obj = stack_pointer[-1]; - unaryfunc getter = NULL; + unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " @@ -2658,14 +2676,12 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } Py_DECREF(obj); if (true) goto pop_1_error; } - iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -2684,7 +2700,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *aiter; PyObject *awaitable; aiter = stack_pointer[-1]; - unaryfunc getter = NULL; + unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { @@ -2692,12 +2708,10 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (awaitable == NULL) { GOTO_ERROR(error); } - } - else { + } else { if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -2711,7 +2725,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } type->tp_name); GOTO_ERROR(error); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -2721,8 +2734,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); GOTO_ERROR(error); - } - else { + } else { Py_DECREF(next_iter); } } @@ -2738,7 +2750,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); + iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } @@ -2749,7 +2761,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable - being awaited on. */Py_DECREF(yf); + being awaited on. */ + Py_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); @@ -2784,8 +2797,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; @@ -2807,7 +2820,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a - regular generator. */_PyErr_SetString(tstate, PyExc_TypeError, + regular generator. */ + _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); GOTO_ERROR(error); @@ -2836,7 +2850,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *from; PyObject *res; from = stack_pointer[-1]; - TIER_ONE_ONLY + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2854,7 +2868,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; - TIER_ONE_ONLY + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2917,7 +2931,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } receiver = stack_pointer[-2]; TIER_ONE_ONLY /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */if (PyGen_Check(receiver)) { + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { GOTO_ERROR(error); @@ -2938,7 +2953,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - TIER_ONE_ONLY + TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { @@ -3005,6 +3020,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); @@ -3022,9 +3038,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -3037,9 +3053,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); int offset = flag * oparg; -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3058,9 +3074,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } Py_DECREF(value); offset = 0; } -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3079,9 +3095,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } Py_DECREF(value); offset = oparg; } -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; -#endif + #endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3094,9 +3110,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); int offset = flag * oparg; -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3146,8 +3162,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -3170,8 +3186,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -3248,22 +3264,24 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */uint16_t ucounter = this_instr[1].cache + (1 << 15); + * do the 2s complement comparison manually */ + uint16_t ucounter = this_instr[1].cache + (1 << 15); uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); next_instr = this_instr; this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } @@ -3279,7 +3297,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; } } -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } @@ -3292,7 +3310,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). - */JUMPBY(-oparg); + */ + JUMPBY(-oparg); DISPATCH(); } @@ -3312,7 +3331,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *list; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; stack_pointer += -1; DISPATCH(); } @@ -3350,7 +3369,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; - value = Py_NewRef(PyExc_AssertionError); + value = Py_NewRef(PyExc_AssertionError); stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); @@ -3371,7 +3390,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; @@ -3380,28 +3399,31 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_ATTR { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } else { /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - NULL | meth | arg1 | ... | argN - */Py_DECREF(owner); + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ + Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; } @@ -3414,7 +3436,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; + if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3431,6 +3453,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); @@ -3446,7 +3469,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } Py_DECREF(owner); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; + if (oparg & 1) stack_pointer[0] = null; stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3477,8 +3500,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -3522,7 +3545,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; + if (oparg & 1) stack_pointer[0] = null; stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3564,7 +3587,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; + if (1) stack_pointer[0] = self; stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3599,7 +3622,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; + if (1) stack_pointer[0] = self; stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3646,7 +3669,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } self = owner; } stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; + if (1) stack_pointer[0] = self; stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3684,7 +3707,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; + if (oparg & 1) stack_pointer[0] = null; stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3789,8 +3812,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); new_frame->localsplus[0] = owner; frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -3826,7 +3849,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; + if (oparg & 1) stack_pointer[0] = null; stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3882,7 +3905,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; + if (oparg & 1) stack_pointer[0] = null; stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3892,7 +3915,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -3908,7 +3931,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); + value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; @@ -3938,7 +3961,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); PyObject *value; - value = GETLOCAL(oparg); + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); stack_pointer[0] = value; @@ -3951,9 +3974,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); @@ -3964,7 +3987,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; - value = GETLOCAL(oparg); + value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); stack_pointer[0] = value; @@ -4027,7 +4050,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } PyObject *mod_or_class_dict; PyObject *v; mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { GOTO_ERROR(error); } @@ -4065,7 +4088,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; @@ -4074,10 +4097,11 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_GLOBAL { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -4088,7 +4112,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */_PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } if (true) goto error; @@ -4113,7 +4138,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } null = NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; + if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4154,7 +4179,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } null = NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; + if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4188,7 +4213,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } null = NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; + if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4198,7 +4223,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); PyObject *locals; - locals = LOCALS(); + locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); @@ -4265,7 +4290,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -4274,12 +4299,12 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } STAT_INC(LOAD_SUPER_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR self = stack_pointer[-1]; { - TIER_ONE_ONLY + TIER_ONE_ONLY if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4287,11 +4312,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } frame, this_instr, global_super, arg); if (err) goto pop_3_error; } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = { - class, self} - ; + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -4320,7 +4343,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } null = NULL; } stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; + if (oparg & 1) stack_pointer[-2] = null; stack_pointer += -2 + ((oparg & 1)); DISPATCH(); } @@ -4336,7 +4359,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; - assert(!(oparg & 1)); + assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); @@ -4363,7 +4386,7 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; - assert(oparg & 1); + assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); @@ -4379,9 +4402,8 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (true) goto pop_3_error; } if (method_found) { - self_or_null = self; // transfer ownership - } - else { + self_or_null = self; // transfer ownership + } else { Py_DECREF(self); self_or_null = NULL; } @@ -4395,9 +4417,9 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { GOTO_ERROR(error); @@ -4420,7 +4442,6 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } if (func_obj == NULL) { GOTO_ERROR(error); } - _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; @@ -4438,10 +4459,10 @@ if (res == NULL) { stack_pointer += -2 - oparg; goto error; } value = stack_pointer[-1]; key = stack_pointer[-2]; dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); + assert(PyDict_CheckExact(dict)); /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references -if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; stack_pointer += -2; DISPATCH(); } @@ -4458,21 +4479,21 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err type = stack_pointer[-2]; subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } + assert(PyTuple_CheckExact(attrs)); // Success! + } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = Py_None; // Failure! - } + // Error! + attrs = Py_None; // Failure! + } stack_pointer[-3] = attrs; stack_pointer += -2; DISPATCH(); @@ -4487,8 +4508,8 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -4502,7 +4523,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *subject; PyObject *res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; stack_pointer[0] = res; stack_pointer += 1; @@ -4516,7 +4537,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *subject; PyObject *res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; stack_pointer[0] = res; stack_pointer += 1; @@ -4551,9 +4572,9 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif JUMPBY(oparg * flag); stack_pointer += -1; DISPATCH(); @@ -4569,7 +4590,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err // _IS_NONE value = stack_pointer[-1]; { - if (Py_IsNone(value)) { + if (Py_IsNone(value)) { b = Py_True; } else { @@ -4582,9 +4603,9 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err { assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif JUMPBY(oparg * flag); } stack_pointer += -1; @@ -4601,7 +4622,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err // _IS_NONE value = stack_pointer[-1]; { - if (Py_IsNone(value)) { + if (Py_IsNone(value)) { b = Py_True; } else { @@ -4614,9 +4635,9 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err { assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif JUMPBY(oparg * flag); } stack_pointer += -1; @@ -4631,9 +4652,9 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err cond = stack_pointer[-1]; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; -#endif + #endif JUMPBY(oparg * flag); stack_pointer += -1; DISPATCH(); @@ -4678,7 +4699,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); PyObject *res; - res = NULL; + res = NULL; stack_pointer[0] = res; stack_pointer += 1; DISPATCH(); @@ -4787,10 +4808,10 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); -#if defined(__EMSCRIPTEN__) + #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif + #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); @@ -4814,25 +4835,25 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err retval = value; { -#if TIER_ONE + #if TIER_ONE assert(frame != &entry_frame); -#endif + #endif _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif } DISPATCH(); } @@ -4872,26 +4893,26 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; -#if TIER_ONE + #if TIER_ONE assert(frame != &entry_frame); -#endif + #endif stack_pointer += -1; _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif DISPATCH(); } @@ -4910,7 +4931,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); @@ -4918,12 +4939,12 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err } STAT_INC(SEND, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _SEND v = stack_pointer[-1]; { - assert(frame != &entry_frame); + assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) @@ -4972,7 +4993,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *receiver; v = stack_pointer[-1]; receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); @@ -5024,7 +5045,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *set; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); + int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -5077,7 +5098,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *set; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); + int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; @@ -5097,7 +5118,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -5106,7 +5127,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err } STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _STORE_ATTR v = stack_pointer[-2]; @@ -5280,7 +5301,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err PyObject *value1; PyObject *value2; value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; + uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); value2 = GETLOCAL(oparg2); @@ -5389,7 +5410,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); @@ -5397,7 +5418,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err } STAT_INC(STORE_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SUBSCR v = stack_pointer[-3]; @@ -5423,6 +5444,7 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -5444,11 +5466,11 @@ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_err value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. -DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) -DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -5488,7 +5510,7 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); { uint16_t counter = read_u16(&this_instr[1].cache); TIER_ONE_ONLY -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); @@ -5496,10 +5518,11 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); } STAT_INC(TO_BOOL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ } // _TO_BOOL { + int err = PyObject_IsTrue(value); Py_DECREF(value); if (err < 0) goto pop_1_error; @@ -5518,8 +5541,8 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); value = stack_pointer[-1]; uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); + // This one is a bit weird, because we expect *some* failures: + assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); STAT_INC(TO_BOOL, hit); Py_DECREF(value); @@ -5534,7 +5557,7 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); INSTRUCTION_STATS(TO_BOOL_BOOL); PyObject *value; value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -5546,7 +5569,7 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -5567,7 +5590,7 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -5582,8 +5605,8 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); PyObject *value; PyObject *res; value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: -DEOPT_IF(!Py_IsNone(value), TO_BOOL); + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -5597,7 +5620,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -5634,7 +5657,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); PyObject *value; PyObject *res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); + res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; @@ -5648,7 +5671,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); PyObject *value; PyObject *res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); + assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; DISPATCH(); @@ -5681,7 +5704,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); -#if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -5689,7 +5712,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); } STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); -#endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION */ (void)seq; (void)counter; } @@ -5713,6 +5736,7 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); PyObject **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5783,7 +5807,8 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */PyObject *exc, *tb; + */ + PyObject *exc, *tb; assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -5794,10 +5819,8 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); Py_DECREF(tb); } assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = { - NULL, exc, val, tb} - ; + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; @@ -5812,10 +5835,10 @@ DEOPT_IF(!Py_IsNone(value), TO_BOOL); INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index fdd5e213c95122..7e10954d2788ed 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -3,89 +3,78 @@ from typing import TextIO class CWriter: - 'A writer that understands how to format C code.' + 'A writer that understands tokens and how to format C code' def __init__(self, out: TextIO, indent: int = 0): self.out = out - self.indent = indent + self.initial_indent = self.indent = indent self.line = -1 self.column = 0 - self.newline = False def set_position(self, tkn: Token): - if self.line < tkn.line and not self.newline: - self.newline = True - self.out.write("\n") - self.line = tkn.end_line - self.column = tkn.end_column - - def emit_token(self, tkn: Token): if self.line < tkn.line: - self.line = tkn.line - self.newline = True - self.out.write("\n") - if self.newline: self.column = 0 - elif self.column > 0 and self.column < tkn.column: - self.out.write(" ") * (tkn.column-self.column) - self.line = tkn.end_line - if tkn.kind == "COMMENT" and "\n" in tkn.text: - # Try to format multi-line comments sensibly - first = True - for line in tkn.text.splitlines(True): - line = line.lstrip(" ") - if not first: - adjust = 1 if line[0] == "*" else 3 - self.out.write(" " * (self.column + adjust)) - self.emit_text(line) - first = False - self.column = tkn.end_column - self.newline = False - return - self.column = tkn.end_column - if tkn.kind == "CMACRO": - self.out.write(tkn.text) + self.out.write("\n") + if self.column == 0: + self.out.write(" " * self.indent) else: - self.emit_text(tkn.text) + #Token colums are 1 based + column = tkn.column - 1 + if self.column < column: + self.out.write(" " * (column-self.column)) + self.line = tkn.end_line + self.column = tkn.end_column - 1 - def emit_text(self, txt: Token): + def maybe_dedent(self, txt: str): parens = txt.count("(") - txt.count(")") if parens < 0: self.indent += parens - if "}" in txt or txt.endswith(":"): + elif "}" in txt or txt.endswith(":"): self.indent -= 1 - assert self.indent >= 0 - if self.newline: - self.out.write(" "*self.indent) - self.newline = False - txt = txt.lstrip(" ") + + def maybe_indent(self, txt: str): + parens = txt.count("(") - txt.count(")") + if parens > 0: + self.indent += parens + elif "{" in txt or txt.endswith(":"): + self.indent += 1 + + def emit_token(self, tkn: Token): + self.maybe_dedent(tkn.text) + self.set_position(tkn) + self.emit_text(tkn.text) + self.maybe_indent(tkn.text) + + def emit_text(self, txt: str): + if self.column == 0: + self.out.write(" " * self.indent) + self.column = self.initial_indent * 4 self.out.write(txt) + + def emit_str(self, txt: str): + self.maybe_dedent(txt) + self.emit_text(txt) if txt.endswith("\n"): + self.column = 0 self.line += 1 - self.newline = True elif txt.endswith("{") or txt.startswith("}"): - if not self.newline: + if self.column: self.out.write("\n") + self.column = 0 self.line += 1 - self.newline = True - if "{" in txt or txt.endswith(":"): - self.indent += 1 - if parens > 0: - self.indent += parens + else: + self.column += len(txt) + self.maybe_indent(txt) def emit(self, txt: str | Token): if isinstance(txt, Token): self.emit_token(txt) elif isinstance(txt, str): - if txt.count("\n") > 1: - for line in txt.splitlines(True): - self.emit_text(line) - return - self.emit_text(txt) + self.emit_str(txt) else: assert False def start_line(self): - if not self.newline: + if self.column: self.out.write("\n") - self.newline = True + self.column = 0 diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 7300b4fbb4da8e..66adbb64d11ac0 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -79,7 +79,7 @@ def choice(*opts: str) -> str: opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r"#.*\n" +macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" CMACRO = "CMACRO" id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" @@ -359,4 +359,4 @@ def to_text(tkns: list[Token], dedent: int = 0) -> str: src = open(filename).read() # print(to_text(tokenize(src))) for tkn in tokenize(src, filename=filename): - pass #print(tkn) + print(tkn) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index be7d2279c596f1..5ddb51a60cfa75 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -213,12 +213,12 @@ def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces:bool) -> int: try: - out.start_line() # out.emit(stack.as_comment() + "\n") if isinstance(uop, Skip): entries = "entries" if uop.size > 1 else "entry" out.emit(f"/* Skip {uop.size} cache {entries} */\n") return offset + uop.size + out.start_line() if braces: out.emit(f"// {uop.name}\n") for var in reversed(uop.stack.inputs): From 59bdeca18dfa5c072c3044ebc339078678fbb4f9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 04:12:28 +0000 Subject: [PATCH 05/40] Revert to declaration ordering of instructions --- Python/generated_cases.c.h | 9595 +++++++++++----------- Tools/cases_generator/analyzer.py | 5 +- Tools/cases_generator/tier1_generator.py | 8 +- 3 files changed, 4783 insertions(+), 4825 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1411bd597b48b4..115232fcca2a23 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,1245 +8,705 @@ #endif #define TIER_ONE 1 - TARGET(BEFORE_ASYNC_WITH) { + TARGET(NOP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BEFORE_ASYNC_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + + TIER_ONE_ONLY + assert(frame == tstate->current_frame); + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; + this_instr->op.code = RESUME_CHECK; } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; DISPATCH(); } - TARGET(BEFORE_WITH) { + TARGET(RESUME_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BEFORE_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); + INSTRUCTION_STATS(RESUME_CHECK); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { + GOTO_ERROR(error); } - GOTO_ERROR(error); + next_instr = this_instr; } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = this_instr; + DISPATCH(); } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_OP) { + TARGET(LOAD_FAST_CHECK) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *rhs; - PyObject *lhs; - PyObject *res; - // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - // _BINARY_OP - { - - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_OP_ADD_FLOAT) { + TARGET(LOAD_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_OP_ADD_INT) { + TARGET(LOAD_FAST_AND_CLEAR) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_OP_ADD_UNICODE) { + TARGET(LOAD_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_UNICODE - { - - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + TARGET(LOAD_CONST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - PyObject *right; - PyObject *left; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_INPLACE_ADD_UNICODE - { - - assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - } - stack_pointer += -2; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_FLOAT) { + TARGET(STORE_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_INT) { + TARGET(STORE_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_FLOAT) { + TARGET(STORE_FAST_STORE_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_INT) { + TARGET(POP_TOP) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SLICE) { + TARGET(PUSH_NULL) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; + INSTRUCTION_STATS(PUSH_NULL); PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { res = NULL; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); - } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - stack_pointer[-3] = res; + Py_DECREF(receiver); + Py_DECREF(value); stack_pointer += -2; DISPATCH(); } - TARGET(BINARY_SUBSCR) { + TARGET(END_SEND) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *res; - // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SUBSCR - { - - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + Py_DECREF(receiver); + stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error; - // not found or error - - stack_pointer[-2] = res; + Py_DECREF(receiver); + stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); - STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(BINARY_SUBSCR_LIST_INT) { + TARGET(UNARY_NEGATIVE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - PyObject *sub; - PyObject *list; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + PyObject *value; PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer[-2] = res; - stack_pointer += -1; + value = stack_pointer[-1]; + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BINARY_SUBSCR_STR_INT) { + TARGET(UNARY_NOT) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - PyObject *sub; - PyObject *str; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + PyObject *value; PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - stack_pointer[-2] = res; - stack_pointer += -1; + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BINARY_SUBSCR_TUPLE_INT) { + TARGET(TO_BOOL_BOOL) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + PyObject *value; + value = stack_pointer[-1]; + + DEOPT_IF(!PyBool_Check(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); DISPATCH(); } - TARGET(BUILD_CONST_KEY_MAP) { + TARGET(TO_BOOL_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortal(value)); + res = Py_False; } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error; } - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; + else { + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BUILD_LIST) { + TARGET(TO_BOOL_LIST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; - values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BUILD_MAP) { + TARGET(TO_BOOL_NONE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; - values = &stack_pointer[-oparg*2]; - - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); - } - if (map == NULL) { stack_pointer += -oparg*2; goto error; } - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BUILD_SET) { + TARGET(TO_BOOL_STR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; - values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value)); + res = Py_False; } - if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error; } + else { + assert(Py_SIZE(value)); + Py_DECREF(value); + res = Py_True; } - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BUILD_SLICE) { + TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + uint32_t version = read_u32(&this_instr[2].cache); + // This one is a bit weird, because we expect *some* failures: + assert(version); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + STAT_INC(TO_BOOL, hit); + Py_DECREF(value); + res = Py_True; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_INVERT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; + INSTRUCTION_STATS(UNARY_INVERT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Invert(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); PyObject *stop; PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } - stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; - start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; + PyObject *container; + PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } - stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; - stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res = NULL; + } + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); + } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } - TARGET(BUILD_STRING) { + TARGET(STORE_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; - pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + INSTRUCTION_STATS(STORE_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; } - if (str == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); + } + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + stack_pointer += -4; DISPATCH(); } - TARGET(BUILD_TUPLE) { + TARGET(BINARY_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; - values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + PyObject *sub; + PyObject *list; + PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(CACHE) { + TARGET(BINARY_SUBSCR_STR_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); } - TARGET(CALL) { + TARGET(BINARY_SUBSCR_TUPLE_INT) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + PyObject *sub; + PyObject *tuple; PyObject *res; - // _SPECIALIZE_CALL - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _CALL - { - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - PyObject **args; - PyObject *null; - PyObject *callable; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; - } - - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - PyObject *null; - PyObject *callable; - PyObject *func; - PyObject *self; - PyObject *self_or_null; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } + TARGET(BINARY_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + PyObject *sub; + PyObject *dict; + PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); } - // _SAVE_RETURN_OFFSET - { + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + // not found or error - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - stack_pointer += (((0) ? 1 : 0)); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_BUILTIN_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *cached = ht->_spec_cache.getitem; + DEOPT_IF(cached == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + uint32_t cached_version = ht->_spec_cache.getitem_version; + DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + Py_INCREF(getitem); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); + STACK_SHRINK(2); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(CALL_BUILTIN_FAST) { + TARGET(LIST_APPEND) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + TARGET(SET_ADD) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_BUILTIN_O) { + TARGET(STORE_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; DISPATCH(); } - TARGET(CALL_FUNCTION_EX) { + TARGET(STORE_SUBSCR_DICT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } - callargs = stack_pointer[-1 - ((oparg & 1))]; - func = stack_pointer[-3 - ((oparg & 1))]; - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - GOTO_ERROR(error); - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - result = PyObject_Call(func, callargs, kwargs); - } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } - - stack_pointer[-3 - ((oparg & 1))] = result; - stack_pointer += -2 - ((oparg & 1)); - CHECK_EVAL_BREAKER(); + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + stack_pointer += -3; + DISPATCH(); + } + + TARGET(DELETE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_2_error; + stack_pointer += -2; DISPATCH(); } @@ -1284,849 +744,783 @@ DISPATCH(); } - TARGET(CALL_ISINSTANCE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); + TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); + TIER_ONE_ONLY + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = args[1]; + /* fall through */ + case 1: + exc = args[0]; + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (true) { stack_pointer += -oparg; goto error; } + stack_pointer += -oparg; DISPATCH(); } - TARGET(CALL_KW) { + TARGET(INTERPRETER_EXIT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + INSTRUCTION_STATS(INTERPRETER_EXIT); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + if (true) goto pop_1_error; } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + if (true) goto pop_1_error; } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_ANEXT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + GOTO_ERROR(error); } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + GOTO_ERROR(error); } } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + GOTO_ERROR(error); + } + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + Py_DECREF(next_iter); + GOTO_ERROR(error); + } else { + Py_DECREF(next_iter); + } } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -3 - oparg; goto error; } - - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - CHECK_EVAL_BREAKER(); + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } - TARGET(CALL_LEN) { + TARGET(GET_AWAITABLE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(CALL_LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - PyObject **args; - PyObject *self; - PyObject *callable; - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - DISPATCH(); + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame != &entry_frame); frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + TIER_ONE_ONLY + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + GOTO_ERROR(error); + } } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; + TIER_ONE_ONLY + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + Py_DECREF(awaitable); + Py_DECREF(exc); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + stack_pointer += -2; DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; + PyObject *none; + PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + TIER_ONE_ONLY + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); + none = Py_None; } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + else { + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_O) { + TARGET(LOAD_ASSERTION_ERROR) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - - TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - PyObject *self_or_null; - PyObject *callable; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + + TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + stack_pointer[0] = bc; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) goto pop_1_error; } - stack_pointer += (((0) ? 1 : 0)); + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + GOTO_ERROR(error); } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + err = PyObject_DelItem(ns, name); + // Can't use ERROR_IF here. + if (err != 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + GOTO_ERROR(error); } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); } - TARGET(CALL_STR_1) { + TARGET(UNPACK_SEQUENCE_LIST) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; DISPATCH(); } - TARGET(CALL_TUPLE_1) { + TARGET(UNPACK_EX) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + PyObject *seq; + seq = stack_pointer[-1]; + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } - TARGET(CALL_TYPE_1) { + TARGET(DELETE_ATTR) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + PyObject *owner; + owner = stack_pointer[-1]; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(CHECK_EG_MATCH) { + TARGET(STORE_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + INSTRUCTION_STATS(STORE_GLOBAL); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err; + err = PyDict_DelItem(GLOBALS(), name); + // Can't use ERROR_IF here. + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + GOTO_ERROR(error); } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; DISPATCH(); } - TARGET(CHECK_EXC_MATCH) { + TARGET(LOAD_LOCALS) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + INSTRUCTION_STATS(LOAD_LOCALS); + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; + Py_INCREF(locals); + stack_pointer[0] = locals; + stack_pointer += 1; DISPATCH(); } - TARGET(CLEANUP_THROW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); } - else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; DISPATCH(); } - TARGET(COMPARE_OP) { + TARGET(LOAD_NAME) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *right; - PyObject *left; - PyObject *res; - // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + PyObject *v; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } - // _COMPARE_OP - { - - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } } } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } - TARGET(COMPARE_OP_FLOAT) { + TARGET(DELETE_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); - stack_pointer[-2] = res; - stack_pointer += -1; + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); DISPATCH(); } - TARGET(COMPARE_OP_INT) { + TARGET(MAKE_CELL) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + GOTO_ERROR(error); + } + SETLOCAL(oparg, cell); DISPATCH(); } - TARGET(COMPARE_OP_STR) { + TARGET(DELETE_DEREF) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); DISPATCH(); } - TARGET(CONTAINS_OP) { + TARGET(LOAD_FROM_DICT_OR_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CONTAINS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + GOTO_ERROR(error); + } + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + Py_INCREF(value); + } + Py_DECREF(class_dict); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(CONVERT_VALUE) { + TARGET(LOAD_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); + INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; - PyObject *result; - value = stack_pointer[-1]; - - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; - stack_pointer[-1] = result; + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; + } + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(COPY) { + TARGET(STORE_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - - assert(oparg > 0); - top = Py_NewRef(bottom); - stack_pointer[0] = top; - stack_pointer += 1; + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + stack_pointer += -1; DISPATCH(); } @@ -2147,124 +1541,191 @@ DISPATCH(); } - TARGET(DELETE_ATTR) { + TARGET(BUILD_STRING) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - stack_pointer += -1; + INSTRUCTION_STATS(BUILD_STRING); + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); + } + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(DELETE_DEREF) { + TARGET(BUILD_TUPLE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); + INSTRUCTION_STATS(BUILD_TUPLE); + PyObject **values; + PyObject *tup; + values = &stack_pointer[-oparg]; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(DELETE_FAST) { + TARGET(BUILD_LIST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); + INSTRUCTION_STATS(BUILD_LIST); + PyObject **values; + PyObject *list; + values = &stack_pointer[-oparg]; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(DELETE_GLOBAL) { + TARGET(LIST_EXTEND) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + INSTRUCTION_STATS(LIST_EXTEND); + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } - GOTO_ERROR(error); + Py_DECREF(iterable); + if (true) goto pop_1_error; + } + assert(Py_IsNone(none_val)); + Py_DECREF(iterable); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + PyObject **values; + PyObject *set; + values = &stack_pointer[-oparg]; + set = PySet_New(NULL); + if (set == NULL) + GOTO_ERROR(error); + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set); + if (true) { stack_pointer += -oparg; goto error; } + } + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); } + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } - TARGET(DELETE_NAME) { + TARGET(SETUP_ANNOTATIONS) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); + INSTRUCTION_STATS(SETUP_ANNOTATIONS); int err; - if (ns == NULL) { + PyObject *ann_dict; + if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - GOTO_ERROR(error); + "no locals found when setting up annotations"); + if (true) goto error; } - err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - GOTO_ERROR(error); + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); } DISPATCH(); } - TARGET(DELETE_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(DICT_MERGE) { + TARGET(BUILD_CONST_KEY_MAP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; - PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + GOTO_ERROR(error); // Pop the keys and values. + } + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); } - Py_DECREF(update); - stack_pointer += -1; + Py_DECREF(keys); + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; DISPATCH(); } @@ -2290,503 +1751,534 @@ DISPATCH(); } - TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; - TIER_ONE_ONLY - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + INSTRUCTION_STATS(DICT_MERGE); + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; } - stack_pointer += -2; + Py_DECREF(update); + stack_pointer += -1; DISPATCH(); } - TARGET(END_FOR) { + TARGET(MAP_ADD) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(END_FOR); + INSTRUCTION_STATS(MAP_ADD); PyObject *value; - // _POP_TOP + PyObject *key; + PyObject *dict; value = stack_pointer[-1]; - { - Py_DECREF(value); - } - // _POP_TOP - value = stack_pointer[-2]; - { - Py_DECREF(value); - } + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; stack_pointer += -2; DISPATCH(); } - TARGET(END_SEND) { + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) goto pop_3_error; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); DISPATCH(); } - TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - - TIER_ONE_ONLY - CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); + TARGET(LOAD_SUPER_ATTR_METHOD) { frame->instr_ptr = next_instr; - Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { - current_executor = (_PyUOpExecutorObject *)executor; - GOTO_TIER_TWO(); - } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; - goto resume_with_error; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *self_or_null; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; } - goto enter_tier_one; - } - - TARGET(EXIT_INIT_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; } + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; stack_pointer += -1; DISPATCH(); } - TARGET(EXTENDED_ARG) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); + TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(type_version != 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(FORMAT_SIMPLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(type_version != 0); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + Py_INCREF(f); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = Py_NewRef(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + PyObject *owner; PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); + + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } else { - res = value; + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } - stack_pointer[-1] = res; + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); + stack_pointer += -2; DISPATCH(); } - TARGET(FORMAT_WITH_SPEC) { + TARGET(COMPARE_OP_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; - PyObject *value; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + PyObject *right; + PyObject *left; PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); } - TARGET(FOR_ITER) { + TARGET(COMPARE_OP_INT) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *iter; - PyObject *next; - // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - // Common case: no jump, leave it to the code generator + INSTRUCTION_STATS(COMPARE_OP_INT); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. - } - stack_pointer[0] = next; - stack_pointer += 1; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - PyObject *iter; - iter = stack_pointer[-1]; - - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); } - TARGET(FOR_ITER_LIST) { + TARGET(CONTAINS_OP) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - } - // _ITER_JUMP_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; + next_instr += 1; + INSTRUCTION_STATS(CONTAINS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } - TARGET(FOR_ITER_RANGE) { + TARGET(CHECK_EG_MATCH) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); - } + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; } - // _ITER_NEXT_RANGE - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; + + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; + + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); } - stack_pointer[0] = next; - stack_pointer += 1; + stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } - TARGET(FOR_ITER_TUPLE) { + TARGET(CHECK_EXC_MATCH) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; } - stack_pointer[0] = next; - stack_pointer += 1; + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; DISPATCH(); } - TARGET(GET_AITER) { + TARGET(IMPORT_NAME) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; - } - stack_pointer[-1] = iter; + INSTRUCTION_STATS(IMPORT_NAME); + PyObject *fromlist; + PyObject *level; + PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(GET_ANEXT) { + TARGET(IMPORT_FROM) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } + INSTRUCTION_STATS(IMPORT_FROM); + PyObject *from; + PyObject *res; + from = stack_pointer[-1]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #if ENABLE_SPECIALIZATION + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + /* We are using unsigned values, but we really want signed values, so + * do the 2s complement comparison manually */ + uint16_t ucounter = this_instr[1].cache + (1 << 15); + uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + OPT_STAT_INC(attempts); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + if (optimized < 0) goto error; + if (optimized) { + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - GOTO_ERROR(error); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); + int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + if (backoff < MINIMUM_TIER2_BACKOFF) { + backoff = MINIMUM_TIER2_BACKOFF; + } + else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { + backoff++; + } + assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); + this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; } } - stack_pointer[0] = awaitable; - stack_pointer += 1; + #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } - TARGET(GET_AWAITABLE) { + TARGET(ENTER_EXECUTOR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); + INSTRUCTION_STATS(ENTER_EXECUTOR); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } + TIER_ONE_ONLY + CHECK_EVAL_BREAKER(); + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); + frame->instr_ptr = next_instr; + Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); } - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; - DISPATCH(); + frame = executor->execute(executor, frame, stack_pointer); + if (frame == NULL) { + frame = tstate->current_frame; + goto resume_with_error; + } + goto enter_tier_one; } - TARGET(GET_ITER) { + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); DISPATCH(); } @@ -2807,6 +2299,97 @@ DISPATCH(); } + TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! + } + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = Py_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; + stack_pointer[0] = values_or_none; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; + DISPATCH(); + } + TARGET(GET_YIELD_FROM_ITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2814,8 +2397,7 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - - /* before: [obj]; after [getiter(obj)] */ + /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { @@ -2843,2355 +2425,2923 @@ DISPATCH(); } - TARGET(IMPORT_FROM) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; - from = stack_pointer[-1]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(IMPORT_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + oparg + 1; + } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(INSTRUMENTED_CALL) { + TARGET(FOR_ITER_GEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(CALL); + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + PyObject *iter; + iter = stack_pointer[-1]; + + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + PyGenObject *gen = (PyGenObject *)iter; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyFrame_StackPush(gen_frame, Py_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); } - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - } - - TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); } - TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(BEFORE_WITH) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + INSTRUCTION_STATS(BEFORE_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ + */ + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); } - PyErr_SetRaisedException(NULL); + GOTO_ERROR(error); } - Py_DECREF(receiver); - Py_DECREF(value); - stack_pointer += -2; + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); + INSTRUCTION_STATS(WITH_EXCEPT_START); + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_func: FOURTH = the context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; } - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; + else { + Py_DECREF(tb); + } + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr; + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; } else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + prev_exc = Py_None; } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; DISPATCH(); } - TARGET(INSTRUMENTED_INSTRUCTION) { + TARGET(INSTRUMENTED_CALL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - if (next_opcode < 0) goto error; - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); + int is_meth = PEEK(oparg + 1) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 2); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(CALL); } - TARGET(INSTRUMENTED_JUMP_BACKWARD) { + TARGET(CALL_PY_WITH_DEFAULTS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(tstate->interp->eval_frame, CALL); + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; + DEOPT_IF(argcount > code->co_argcount, CALL); + DEOPT_IF(argcount < min_args, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); + new_frame->localsplus[i] = Py_NewRef(def); + } + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + TARGET(CALL_TUPLE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + TARGET(CALL_ALLOC_AND_ENTER_INIT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + PyObject **args; + PyObject *null; + PyObject *callable; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(tp); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + goto start_frame; } - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + GOTO_ERROR(error); + } + stack_pointer += -1; DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyObject *value = POP(); - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; + TARGET(CALL_BUILTIN_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - else { - Py_DECREF(value); - offset = 0; + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyObject *value = POP(); - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; + TARGET(CALL_BUILTIN_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - else { - Py_DECREF(value); - offset = oparg; + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); - } - next_instr = this_instr; + TARGET(CALL_BUILTIN_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = this_instr; - DISPATCH(); - } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - DISPATCH(); - } - - TARGET(INSTRUMENTED_RETURN_CONST) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); } - TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - assert(frame != &entry_frame); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + GOTO_ERROR(error); + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); } - TARGET(INTERPRETER_EXIT) { + TARGET(CALL_ISINSTANCE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + GOTO_ERROR(error); + } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); } - TARGET(IS_OP) { + TARGET(CALL_LIST_APPEND) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + PyObject **args; + PyObject *self; + PyObject *callable; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); + DEOPT_IF(!PyList_Check(self), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } - TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); - if (optimized < 0) goto error; - if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - } - else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; - } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; - } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; - } + TARGET(CALL_METHOD_DESCRIPTOR_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); } - #endif /* ENABLE_SPECIALIZATION */ + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(JUMP_FORWARD) { + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LIST_APPEND) { + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - stack_pointer += -1; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LIST_EXTEND) { + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = PEEK(oparg + 2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 3); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PEEK(total_args + 1); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); + } + + TARGET(CALL_KW) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwnames; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames + ); + Py_DECREF(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } } - Py_DECREF(iterable); - if (true) goto pop_1_error; } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - stack_pointer += -1; + Py_DECREF(kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LOAD_ASSERTION_ERROR) { + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - TARGET(LOAD_ATTR) { + TARGET(CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; + PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + GOTO_ERROR(error); } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + GOTO_ERROR(error); + } + Py_SETREF(callargs, tuple); } - // _LOAD_ATTR - { - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; - } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) GOTO_ERROR(error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); } else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } } } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyObject_Call(func, callargs, kwargs); + } + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + assert(PEEK(2 + (oparg & 1)) == NULL); + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + Py_DECREF(codeobj); + if (func_obj == NULL) { + GOTO_ERROR(error); } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = (PyObject *)func_obj; + stack_pointer[-1] = func; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - PyObject *owner; + TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + PyObject *func; PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); + stack_pointer[-2] = func; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); DISPATCH(); } - TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; + convertion_func_ptr conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = CONVERSION_FUNCTIONS[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + else { + res = value; } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); + TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + PyObject *fmt_spec; + PyObject *value; + PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = Py_NewRef(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + DISPATCH(); } - TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; DISPATCH(); } - TARGET(LOAD_ATTR_WITH_HINT) { + TARGET(INSTRUMENTED_INSTRUCTION) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(LOAD_BUILD_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error; - } - stack_pointer[0] = bc; - stack_pointer += 1; + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(LOAD_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + PyObject *value = POP(); + int flag = Py_IsNone(value); + int offset; + if (flag) { + offset = oparg; } - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; + else { + Py_DECREF(value); + offset = 0; + } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + PyObject *value = POP(); + int offset; + int nflag = Py_IsNone(value); + if (nflag) { + offset = 0; + } + else { + Py_DECREF(value); + offset = oparg; + } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(LOAD_FAST_AND_CLEAR) { + TARGET(EXTENDED_ARG) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); + INSTRUCTION_STATS(EXTENDED_ARG); + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); } - TARGET(LOAD_FAST_CHECK) { + TARGET(CACHE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); } - TARGET(LOAD_FAST_LOAD_FAST) { + TARGET(RESERVED) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - DISPATCH(); + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); } - TARGET(LOAD_FROM_DICT_OR_DEREF) { + TARGET(END_FOR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; + INSTRUCTION_STATS(END_FOR); PyObject *value; - class_dict = stack_pointer[-1]; - - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + // _POP_TOP + value = stack_pointer[-1]; + { + Py_DECREF(value); } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - Py_INCREF(value); + // _POP_TOP + value = stack_pointer[-2]; + { + Py_DECREF(value); } - Py_DECREF(class_dict); - stack_pointer[-1] = value; + stack_pointer += -2; DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + TARGET(TO_BOOL) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject *value; + PyObject *res; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); } + STAT_INC(TO_BOOL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _TO_BOOL + { + + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; + // _BINARY_OP_MULTIPLY_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_GLOBAL) { + TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *this_instr = next_instr - 5; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + PyObject *right; + PyObject *left; PyObject *res; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_GLOBAL + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } - // _LOAD_GLOBAL + // _BINARY_OP_ADD_INT { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } - } - } - null = NULL; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + TARGET(BINARY_OP_SUBTRACT_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + PyObject *right; + PyObject *left; PyObject *res; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } - // _GUARD_BUILTINS_VERSION + // _BINARY_OP_SUBTRACT_INT { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; } - // _LOAD_GLOBAL_BUILTINS + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - uint16_t index = read_u16(&this_instr[4].cache); - - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); + // _BINARY_OP_MULTIPLY_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + TARGET(BINARY_OP_ADD_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + PyObject *right; + PyObject *left; PyObject *res; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE + // _BINARY_OP_ADD_FLOAT { - uint16_t index = read_u16(&this_instr[4].cache); - - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_LOCALS) { + TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } - Py_INCREF(locals); - stack_pointer[0] = locals; - stack_pointer += 1; + // _BINARY_OP_SUBTRACT_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_NAME) { + TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; - - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + // _BINARY_OP_ADD_UNICODE + { + + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } - stack_pointer[0] = v; - stack_pointer += 1; + // _BINARY_OP_INPLACE_ADD_UNICODE + { + + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } + stack_pointer += -2; DISPATCH(); } - TARGET(LOAD_SUPER_ATTR) { + TARGET(BINARY_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *class; - PyObject *global_super; - PyObject *self; - PyObject *attr; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *res; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - int load_method = oparg & 1; if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } - STAT_INC(LOAD_SUPER_ATTR, deferred); + STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _LOAD_SUPER_ATTR - self = stack_pointer[-1]; + // _BINARY_SUBSCR { - TIER_ONE_ONLY - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - if (err) goto pop_3_error; - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } - } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; + + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; } - stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; - stack_pointer += -2 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - stack_pointer[-3] = attr; - stack_pointer += -2 + (((0) ? 1 : 0)); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_SUPER_ATTR_METHOD) { + TARGET(STORE_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; + PyObject *container; + PyObject *v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; + // _STORE_SUBSCR + v = stack_pointer[-3]; + { + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; } - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; + stack_pointer += -3; DISPATCH(); } - TARGET(MAKE_CELL) { + TARGET(RETURN_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - GOTO_ERROR(error); + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; } - SETLOCAL(oparg, cell); + #endif DISPATCH(); } - TARGET(MAKE_FUNCTION) { + TARGET(RETURN_CONST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; - - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); + INSTRUCTION_STATS(RETURN_CONST); + PyObject *value; + PyObject *retval; + // _LOAD_CONST + { + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + } + // _POP_FRAME + retval = value; + { + + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; DISPATCH(); } - TARGET(MAP_ADD) { + TARGET(SEND) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - stack_pointer += -2; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *receiver; + PyObject *v; + PyObject *retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _SEND + v = stack_pointer[-1]; + { + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + GOTO_ERROR(error); + } + } + Py_DECREF(v); + } + stack_pointer[-1] = retval; DISPATCH(); } - TARGET(MATCH_CLASS) { + TARGET(UNPACK_SEQUENCE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = Py_None; // Failure! - } - stack_pointer[-3] = attrs; - stack_pointer += -2; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; DISPATCH(); } - TARGET(MATCH_KEYS) { + TARGET(STORE_ATTR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - stack_pointer[0] = values_or_none; - stack_pointer += 1; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; + PyObject *owner; + PyObject *v; + // _SPECIALIZE_STORE_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_ATTR + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; DISPATCH(); } - TARGET(MATCH_MAPPING) { + TARGET(LOAD_GLOBAL) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_GLOBAL + { + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) goto error; + } + Py_INCREF(res); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } + } + null = NULL; + } stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - TARGET(MATCH_SEQUENCE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; + TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(NOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); - } - - TARGET(POP_EXCEPT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - stack_pointer += -1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - TARGET(POP_JUMP_IF_FALSE) { + TARGET(LOAD_GLOBAL_BUILTIN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; - cond = stack_pointer[-1]; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + PyObject *res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _GUARD_BUILTINS_VERSION + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t index = read_u16(&this_instr[4].cache); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - stack_pointer += -1; + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *class; + PyObject *global_super; + PyObject *self; + PyObject *attr; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; { - if (Py_IsNone(value)) { - b = Py_True; + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); } - else { - b = Py_False; - Py_DECREF(value); + STAT_INC(LOAD_SUPER_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_SUPER_ATTR + self = stack_pointer[-1]; + { + TIER_ONE_ONLY + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; } - // _POP_JUMP_IF_TRUE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - stack_pointer += -1; + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + ((oparg & 1)); DISPATCH(); } - TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR + owner = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - // _POP_JUMP_IF_FALSE - cond = b; + // _LOAD_ATTR { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } + } + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + } } - stack_pointer += -1; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(POP_JUMP_IF_TRUE) { + TARGET(LOAD_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_TOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(PUSH_EXC_INFO) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - prev_exc = Py_None; + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(PUSH_NULL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - PyObject *res; - res = NULL; - stack_pointer[0] = res; - stack_pointer += 1; + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(RAISE_VARARGS) { + TARGET(LOAD_ATTR_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - PyObject **args; - args = &stack_pointer[-oparg]; - - TIER_ONE_ONLY - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); } - if (true) { stack_pointer += -oparg; goto error; } - stack_pointer += -oparg; + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(RERAISE) { + TARGET(LOAD_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - - TIER_ONE_ONLY - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; } else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - - TARGET(RESERVED) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); } - TARGET(RESUME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; - TIER_ONE_ONLY - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; + TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - this_instr->op.code = RESUME_CHECK; + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(RESUME_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } - - TARGET(RETURN_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_CONST); - PyObject *value; - PyObject *retval; - // _LOAD_CONST + TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + owner = stack_pointer[-1]; { - - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } - // _POP_FRAME - retval = value; + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS { - - #if TIER_ONE - assert(frame != &entry_frame); - #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(RETURN_GENERATOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); + TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - LOAD_IP(frame->return_offset); - goto resume_frame; + // _GUARD_DORV_VALUES + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + } + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + } + stack_pointer += -2; + DISPATCH(); } - TARGET(RETURN_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != &entry_frame); - #endif - stack_pointer += -1; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - #endif + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); + } + stack_pointer += -2; DISPATCH(); } - TARGET(SEND) { + TARGET(COMPARE_OP) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *receiver; - PyObject *v; - PyObject *retval; - // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_Send(receiver, next_instr); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } - STAT_INC(SEND, deferred); + STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _SEND - v = stack_pointer[-1]; + // _COMPARE_OP { - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); - } - else { - GOTO_ERROR(error); - } + + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; } - Py_DECREF(v); } - stack_pointer[-1] = retval; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(SEND_GEN) { + TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - PyObject *v; - PyObject *receiver; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(SETUP_ANNOTATIONS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + PyObject *cond; + cond = stack_pointer[-1]; - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error; - } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; DISPATCH(); } - TARGET(SET_ADD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; + TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + PyObject *cond; + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); stack_pointer += -1; DISPATCH(); } - TARGET(SET_FUNCTION_ATTRIBUTE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); + TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } + } + // _POP_JUMP_IF_TRUE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - stack_pointer[-2] = func; stack_pointer += -1; DISPATCH(); } - TARGET(SET_UPDATE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; + TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } + } + // _POP_JUMP_IF_FALSE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + } stack_pointer += -1; DISPATCH(); } - TARGET(STORE_ATTR) { + TARGET(FOR_ITER) { frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; - PyObject *owner; - PyObject *v; - // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *iter; + PyObject *next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); + _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } - STAT_INC(STORE_ATTR, deferred); + STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _STORE_ATTR - v = stack_pointer[-2]; + // _FOR_ITER { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; + + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + // Common case: no jump, leave it to the code generator + } - stack_pointer += -2; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - TARGET(STORE_ATTR_INSTANCE_VALUE) { + TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; + { + + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; + { + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + Py_DECREF(r); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; - PyObject *value; + PyObject *attr; + PyObject *self = NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _GUARD_DORV_VALUES + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; + // _GUARD_KEYS_VERSION { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); - } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } - stack_pointer += -2; + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - TARGET(STORE_ATTR_SLOT) { + TARGET(LOAD_ATTR_METHOD_NO_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; - PyObject *value; + PyObject *attr; + PyObject *self = NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; @@ -5199,213 +5349,159 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _STORE_ATTR_SLOT - value = stack_pointer[-2]; + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT { - uint16_t index = read_u16(&this_instr[4].cache); + PyObject *descr = read_obj(&this_instr[6].cache); - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + assert(oparg & 1); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; } - stack_pointer += -2; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - TARGET(STORE_ATTR_WITH_HINT) { + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; - PyObject *value; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_FAST_LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); - stack_pointer[-1] = value2; - DISPATCH(); - } - - TARGET(STORE_FAST_STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; - v = stack_pointer[-1]; + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(STORE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(STORE_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + // _CHECK_ATTR_METHOD_LAZY_DICT + { + + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - stack_pointer += -4; + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - TARGET(STORE_SUBSCR) { + TARGET(CALL) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *v; - // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + // _SPECIALIZE_CALL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; { uint16_t counter = read_u16(&this_instr[1].cache); @@ -5413,449 +5509,314 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); DISPATCH_SAME_OPARG(); } - STAT_INC(STORE_SUBSCR, deferred); + STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _STORE_SUBSCR - v = stack_pointer[-3]; + // _CALL { - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } - stack_pointer += -3; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - stack_pointer += -3; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer += -3; - DISPATCH(); - } - - TARGET(SWAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(TO_BOOL) { - frame->instr_ptr = next_instr; + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject *value; - PyObject *res; - // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + PyObject *null; + PyObject *callable; + PyObject *func; + PyObject *self; + PyObject *self_or_null; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + DEOPT_IF(tstate->interp->eval_frame, CALL); } - // _TO_BOOL + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - PyObject *value; - value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); - } - - TARGET(TO_BOOL_INT) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } - else { - Py_DECREF(value); - res = Py_True; + // _CHECK_STACK_SPACE + { + + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_LIST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_NONE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; - stack_pointer[-1] = res; + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(TO_BOOL_STR) { - frame->instr_ptr = next_instr; + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + PyObject *self_or_null; + PyObject *callable; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_INVERT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NEGATIVE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NOT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNPACK_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; - seq = stack_pointer[-1]; - - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - stack_pointer += (oparg >> 8) + (oparg & 0xFF); + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(UNPACK_SEQUENCE) { + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *seq; - // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; + PyObject *rhs; + PyObject *lhs; + PyObject *res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } - STAT_INC(UNPACK_SEQUENCE, deferred); + STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ - (void)seq; - (void)counter; + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); } - // _UNPACK_SEQUENCE + // _BINARY_OP { - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - } - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(WITH_EXCEPT_START) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(YIELD_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } - #undef TIER_ONE diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 7bab92d1de6db6..11b69c1d3264e1 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -2,7 +2,6 @@ from dataclasses import dataclass import lexer import parser -from _typing_backports import assert_never @dataclass class Properties: @@ -302,7 +301,7 @@ def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: d case parser.CacheEffect(): parts.append(Skip(part.size)) case _: - assert_never(part) + assert False assert(parts) add_instruction(macro.name, parts, instructions) @@ -343,7 +342,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: case parser.Pseudo(): pass case _: - assert_never(node) + assert False for node in forest: if isinstance(node, parser.Macro): add_macro(node, instructions, uops) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 5ddb51a60cfa75..3a5f48ba007b06 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -1,9 +1,8 @@ import sys from analyzer import Analysis, Instruction, Uop, Part, analyze_file, Skip, StackItem, analysis_error from cwriter import CWriter -from typing import Tuple, TextIO, Iterator +from typing import TextIO, Iterator from lexer import Token -from dataclasses import dataclass from stack import StackOffset def write_header(filename: str, outfile: TextIO) -> None: @@ -129,8 +128,7 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): parens -= 1 out.emit(tkn) -def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): - parens = 0 +def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack, inst: Instruction): out.emit("DEOPT_IF") out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") @@ -265,7 +263,7 @@ def generate_tier1(filename: str, analysis: Analysis, outfile: TextIO) -> None: write_header(filename, outfile) out = CWriter(outfile, 2) out.emit("\n") - for name, inst in sorted(analysis.instructions.items()): + for name, inst in analysis.instructions.items(): needs_this = uses_this(inst) out.emit(f"TARGET({name}) {{") if needs_this and not inst.is_target: From d41dfa577df540052b684a39a981eef099357d8c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 04:16:25 +0000 Subject: [PATCH 06/40] Tweak formatting --- Python/generated_cases.c.h | 722 +++++++++++++++---------------- Tools/cases_generator/cwriter.py | 2 +- 2 files changed, 362 insertions(+), 362 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 115232fcca2a23..59a19a368eaef6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,14 +7,14 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 - + TARGET(NOP) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); DISPATCH(); } - + TARGET(RESUME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -42,7 +42,7 @@ } DISPATCH(); } - + TARGET(RESUME_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; @@ -57,7 +57,7 @@ DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } - + TARGET(INSTRUMENTED_RESUME) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -87,53 +87,53 @@ } DISPATCH(); } - + TARGET(LOAD_FAST_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; - value = GETLOCAL(oparg); + value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); PyObject *value; - value = GETLOCAL(oparg); + value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST_AND_CLEAR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyObject *value; - value = GETLOCAL(oparg); + value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; - uint32_t oparg1 = oparg >> 4; + uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; value1 = GETLOCAL(oparg1); value2 = GETLOCAL(oparg2); @@ -144,19 +144,19 @@ stack_pointer += 2; DISPATCH(); } - + TARGET(LOAD_CONST) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); + value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); } - + TARGET(STORE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -167,7 +167,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -175,7 +175,7 @@ PyObject *value1; PyObject *value2; value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; + uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); value2 = GETLOCAL(oparg2); @@ -183,7 +183,7 @@ stack_pointer[-1] = value2; DISPATCH(); } - + TARGET(STORE_FAST_STORE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -199,7 +199,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(POP_TOP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -210,18 +210,18 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(PUSH_NULL) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); PyObject *res; - res = NULL; + res = NULL; stack_pointer[0] = res; stack_pointer += 1; DISPATCH(); } - + TARGET(INSTRUMENTED_END_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -245,7 +245,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(END_SEND) { frame->instr_ptr = next_instr; next_instr += 1; @@ -254,12 +254,12 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + Py_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); } - + TARGET(INSTRUMENTED_END_SEND) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -268,7 +268,7 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - TIER_ONE_ONLY + TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { @@ -281,7 +281,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(UNARY_NEGATIVE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -289,13 +289,13 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); + res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(UNARY_NOT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -303,24 +303,24 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); + assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_BOOL) { frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); PyObject *value; value = stack_pointer[-1]; - + DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } - + TARGET(TO_BOOL_INT) { frame->instr_ptr = next_instr; next_instr += 4; @@ -328,7 +328,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -341,7 +341,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_LIST) { frame->instr_ptr = next_instr; next_instr += 4; @@ -349,14 +349,14 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_NONE) { frame->instr_ptr = next_instr; next_instr += 4; @@ -364,14 +364,14 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: + // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_STR) { frame->instr_ptr = next_instr; next_instr += 4; @@ -379,7 +379,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -393,7 +393,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(TO_BOOL_ALWAYS_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -411,7 +411,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(UNARY_INVERT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -419,13 +419,13 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); + res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(BINARY_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -437,7 +437,7 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -454,7 +454,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -482,7 +482,7 @@ stack_pointer += -4; DISPATCH(); } - + TARGET(BINARY_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -492,7 +492,7 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); @@ -508,7 +508,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_STR_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -518,7 +518,7 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -534,7 +534,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_TUPLE_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -544,7 +544,7 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); @@ -560,7 +560,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_DICT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -570,7 +570,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -585,7 +585,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_SUBSCR_GETITEM) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -594,7 +594,7 @@ PyObject *container; sub = stack_pointer[-1]; container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -616,7 +616,7 @@ frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(LIST_APPEND) { frame->instr_ptr = next_instr; next_instr += 1; @@ -625,11 +625,11 @@ PyObject *list; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; stack_pointer += -1; DISPATCH(); } - + TARGET(SET_ADD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -638,13 +638,13 @@ PyObject *set; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); + int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -673,7 +673,7 @@ stack_pointer += -3; DISPATCH(); } - + TARGET(STORE_SUBSCR_DICT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -692,7 +692,7 @@ stack_pointer += -3; DISPATCH(); } - + TARGET(DELETE_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -709,7 +709,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(CALL_INTRINSIC_1) { frame->instr_ptr = next_instr; next_instr += 1; @@ -717,14 +717,14 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); + assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); } - + TARGET(CALL_INTRINSIC_2) { frame->instr_ptr = next_instr; next_instr += 1; @@ -734,7 +734,7 @@ PyObject *res; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); + assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); Py_DECREF(value1); @@ -743,7 +743,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(RAISE_VARARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -775,7 +775,7 @@ stack_pointer += -oparg; DISPATCH(); } - + TARGET(INTERPRETER_EXIT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -790,7 +790,7 @@ tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } - + TARGET(INSTRUMENTED_RETURN_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -815,7 +815,7 @@ LOAD_IP(frame->return_offset); goto resume_frame; } - + TARGET(INSTRUMENTED_RETURN_CONST) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -839,7 +839,7 @@ LOAD_IP(frame->return_offset); goto resume_frame; } - + TARGET(GET_AITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -847,7 +847,7 @@ PyObject *obj; PyObject *iter; obj = stack_pointer[-1]; - unaryfunc getter = NULL; + unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; @@ -876,7 +876,7 @@ stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(GET_ANEXT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -884,7 +884,7 @@ PyObject *aiter; PyObject *awaitable; aiter = stack_pointer[-1]; - unaryfunc getter = NULL; + unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { @@ -926,7 +926,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(GET_AWAITABLE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -934,7 +934,7 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); + iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } @@ -957,7 +957,7 @@ stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(SEND_GEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -966,7 +966,7 @@ PyObject *receiver; v = stack_pointer[-1]; receiver = stack_pointer[-2]; - + DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -982,14 +982,14 @@ frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - + TARGET(INSTRUMENTED_YIELD_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - assert(frame != &entry_frame); + assert(frame != &entry_frame); frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -1012,14 +1012,14 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - + TARGET(YIELD_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! + // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. assert(frame != &entry_frame); @@ -1041,7 +1041,7 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - + TARGET(POP_EXCEPT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1053,7 +1053,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(RERAISE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -1062,7 +1062,7 @@ PyObject **values; exc = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - TIER_ONE_ONLY + TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1082,7 +1082,7 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - + TARGET(END_ASYNC_FOR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -1106,7 +1106,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(CLEANUP_THROW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -1119,7 +1119,7 @@ exc_value = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY + TIER_ONE_ONLY assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1139,24 +1139,24 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(LOAD_ASSERTION_ERROR) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; - value = Py_NewRef(PyExc_AssertionError); + value = Py_NewRef(PyExc_AssertionError); stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_BUILD_CLASS) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -1166,7 +1166,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(STORE_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1191,7 +1191,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(DELETE_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1214,7 +1214,7 @@ } DISPATCH(); } - + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1234,7 +1234,7 @@ stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(UNPACK_SEQUENCE_TUPLE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1254,7 +1254,7 @@ stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(UNPACK_SEQUENCE_LIST) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1274,14 +1274,14 @@ stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(UNPACK_EX) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); PyObject *seq; seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); @@ -1289,7 +1289,7 @@ stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } - + TARGET(DELETE_ATTR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1304,7 +1304,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1318,7 +1318,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(DELETE_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1336,13 +1336,13 @@ } DISPATCH(); } - + TARGET(LOAD_LOCALS) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); PyObject *locals; - locals = LOCALS(); + locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); @@ -1353,7 +1353,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1361,7 +1361,7 @@ PyObject *mod_or_class_dict; PyObject *v; mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { GOTO_ERROR(error); } @@ -1385,13 +1385,13 @@ stack_pointer[-1] = v; DISPATCH(); } - + TARGET(LOAD_NAME) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); + PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); @@ -1421,7 +1421,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(DELETE_FAST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1432,7 +1432,7 @@ SETLOCAL(oparg, NULL); DISPATCH(); } - + TARGET(MAKE_CELL) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1447,7 +1447,7 @@ SETLOCAL(oparg, cell); DISPATCH(); } - + TARGET(DELETE_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1464,7 +1464,7 @@ Py_DECREF(oldobj); DISPATCH(); } - + TARGET(LOAD_FROM_DICT_OR_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1472,7 +1472,7 @@ PyObject *class_dict; PyObject *value; class_dict = stack_pointer[-1]; - PyObject *name; + PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); @@ -1492,13 +1492,13 @@ stack_pointer[-1] = value; DISPATCH(); } - + TARGET(LOAD_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; - PyObject *cell = GETLOCAL(oparg); + PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); @@ -1509,7 +1509,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(STORE_DEREF) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1523,7 +1523,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(COPY_FREE_VARS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1540,7 +1540,7 @@ } DISPATCH(); } - + TARGET(BUILD_STRING) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1548,7 +1548,7 @@ PyObject **pieces; PyObject *str; pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } @@ -1557,7 +1557,7 @@ stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(BUILD_TUPLE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1565,13 +1565,13 @@ PyObject **values; PyObject *tup; values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); + tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { stack_pointer += -oparg; goto error; } stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(BUILD_LIST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1579,13 +1579,13 @@ PyObject **values; PyObject *list; values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); + list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { stack_pointer += -oparg; goto error; } stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(LIST_EXTEND) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1594,7 +1594,7 @@ PyObject *list; iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) @@ -1612,7 +1612,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(SET_UPDATE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1621,13 +1621,13 @@ PyObject *set; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); + int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; DISPATCH(); } - + TARGET(BUILD_SET) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1635,7 +1635,7 @@ PyObject **values; PyObject *set; values = &stack_pointer[-oparg]; - set = PySet_New(NULL); + set = PySet_New(NULL); if (set == NULL) GOTO_ERROR(error); int err = 0; @@ -1653,7 +1653,7 @@ stack_pointer += 1 - oparg; DISPATCH(); } - + TARGET(BUILD_MAP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1661,7 +1661,7 @@ PyObject **values; PyObject *map; values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( + map = _PyDict_FromItems( values, 2, values+1, 2, oparg); @@ -1673,7 +1673,7 @@ stack_pointer += 1 - oparg*2; DISPATCH(); } - + TARGET(SETUP_ANNOTATIONS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1700,7 +1700,7 @@ } DISPATCH(); } - + TARGET(BUILD_CONST_KEY_MAP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1710,7 +1710,7 @@ PyObject *map; keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || + if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); @@ -1728,7 +1728,7 @@ stack_pointer += -oparg; DISPATCH(); } - + TARGET(DICT_UPDATE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1737,7 +1737,7 @@ PyObject *dict; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", @@ -1750,7 +1750,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(DICT_MERGE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1761,7 +1761,7 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { + if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; @@ -1770,7 +1770,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(MAP_ADD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -1781,24 +1781,24 @@ value = stack_pointer[-1]; key = stack_pointer[-2]; dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); + assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; stack_pointer += -2; DISPATCH(); } - + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } - + TARGET(LOAD_SUPER_ATTR_ATTR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1810,7 +1810,7 @@ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; - + assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1825,7 +1825,7 @@ stack_pointer += -2 + (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_SUPER_ATTR_METHOD) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1838,7 +1838,7 @@ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; - assert(oparg & 1); + assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); @@ -1864,7 +1864,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(LOAD_ATTR_PROPERTY) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -1896,7 +1896,7 @@ frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -1929,7 +1929,7 @@ frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(STORE_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -1982,7 +1982,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(COMPARE_OP_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -1992,7 +1992,7 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2009,7 +2009,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(COMPARE_OP_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -2019,7 +2019,7 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); @@ -2039,7 +2039,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(COMPARE_OP_STR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -2049,7 +2049,7 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); @@ -2066,7 +2066,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(IS_OP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2076,7 +2076,7 @@ PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; + int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -2084,7 +2084,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(CONTAINS_OP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2094,7 +2094,7 @@ PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); + int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); if (res < 0) goto pop_2_error; @@ -2103,7 +2103,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(CHECK_EG_MATCH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2114,7 +2114,7 @@ PyObject *match; match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); if (true) goto pop_2_error; @@ -2137,7 +2137,7 @@ stack_pointer[-1] = match; DISPATCH(); } - + TARGET(CHECK_EXC_MATCH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2147,7 +2147,7 @@ PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); + assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); if (true) goto pop_1_error; @@ -2158,7 +2158,7 @@ stack_pointer[-1] = b; DISPATCH(); } - + TARGET(IMPORT_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2168,7 +2168,7 @@ PyObject *res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; - TIER_ONE_ONLY + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2178,7 +2178,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(IMPORT_FROM) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2186,7 +2186,7 @@ PyObject *from; PyObject *res; from = stack_pointer[-1]; - TIER_ONE_ONLY + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2194,7 +2194,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(JUMP_FORWARD) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2202,7 +2202,7 @@ JUMPBY(oparg); DISPATCH(); } - + TARGET(JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -2242,7 +2242,7 @@ #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } - + TARGET(ENTER_EXECUTOR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2267,7 +2267,7 @@ } goto enter_tier_one; } - + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2281,7 +2281,7 @@ JUMPBY(-oparg); DISPATCH(); } - + TARGET(GET_LEN) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2289,7 +2289,7 @@ PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; - // PUSH(len(TOS)) + // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); @@ -2298,7 +2298,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(MATCH_CLASS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2310,7 +2310,7 @@ names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); @@ -2329,7 +2329,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(MATCH_MAPPING) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2337,13 +2337,13 @@ PyObject *subject; PyObject *res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; stack_pointer[0] = res; stack_pointer += 1; DISPATCH(); } - + TARGET(MATCH_SEQUENCE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2351,13 +2351,13 @@ PyObject *subject; PyObject *res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; stack_pointer[0] = res; stack_pointer += 1; DISPATCH(); } - + TARGET(MATCH_KEYS) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2367,14 +2367,14 @@ PyObject *values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). + // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; stack_pointer[0] = values_or_none; stack_pointer += 1; DISPATCH(); } - + TARGET(GET_ITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2382,14 +2382,14 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ + /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(GET_YIELD_FROM_ITER) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2397,7 +2397,7 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ + /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { @@ -2424,7 +2424,7 @@ stack_pointer[-1] = iter; DISPATCH(); } - + TARGET(INSTRUMENTED_FOR_ITER) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -2456,14 +2456,14 @@ INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(FOR_ITER_GEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; iter = stack_pointer[-1]; - + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -2480,7 +2480,7 @@ frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2489,7 +2489,7 @@ PyObject *exit; PyObject *res; mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2523,7 +2523,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(BEFORE_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2532,7 +2532,7 @@ PyObject *exit; PyObject *res; mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the + /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); @@ -2569,7 +2569,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(WITH_EXCEPT_START) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2581,7 +2581,7 @@ val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_func = stack_pointer[-4]; - /* At the top of the stack are 4 values: + /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() @@ -2609,7 +2609,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(PUSH_EXC_INFO) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2617,7 +2617,7 @@ PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; - + _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -2632,7 +2632,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(INSTRUMENTED_CALL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -2650,7 +2650,7 @@ INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); } - + TARGET(CALL_PY_WITH_DEFAULTS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -2695,7 +2695,7 @@ frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } - + TARGET(CALL_TYPE_1) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2707,7 +2707,7 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); + assert(oparg == 1); DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); @@ -2720,7 +2720,7 @@ stack_pointer += -1 - oparg; DISPATCH(); } - + TARGET(CALL_STR_1) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2732,7 +2732,7 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); + assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); @@ -2747,7 +2747,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_TUPLE_1) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2759,7 +2759,7 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); + assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); @@ -2774,7 +2774,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_ALLOC_AND_ENTER_INIT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -2785,7 +2785,7 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* This instruction does the following: + /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack @@ -2833,7 +2833,7 @@ tstate->py_recursion_remaining--; goto start_frame; } - + TARGET(EXIT_INIT_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; @@ -2850,7 +2850,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(CALL_BUILTIN_CLASS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2862,7 +2862,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -2884,7 +2884,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BUILTIN_O) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2896,7 +2896,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ + /* Builtin METH_O functions */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -2925,7 +2925,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BUILTIN_FAST) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2937,7 +2937,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ + /* Builtin METH_FASTCALL functions, without keywords */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -2970,7 +2970,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -2982,7 +2982,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3009,7 +3009,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3021,7 +3021,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* len(o) */ + /* len(o) */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3045,7 +3045,7 @@ stack_pointer += -1 - oparg; DISPATCH(); } - + TARGET(CALL_ISINSTANCE) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3057,7 +3057,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ + /* isinstance(o, o2) */ int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3083,7 +3083,7 @@ stack_pointer += -1 - oparg; DISPATCH(); } - + TARGET(CALL_LIST_APPEND) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3094,7 +3094,7 @@ args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); + assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); assert(self != NULL); @@ -3111,7 +3111,7 @@ SKIP_OVER(1); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_O) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3123,7 +3123,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -3156,7 +3156,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3168,7 +3168,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -3198,7 +3198,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3210,7 +3210,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); + assert(oparg == 0 || oparg == 1); int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3242,7 +3242,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3254,7 +3254,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -3284,7 +3284,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(INSTRUMENTED_CALL_KW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -3300,7 +3300,7 @@ if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } - + TARGET(CALL_KW) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3316,7 +3316,7 @@ args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: + // oparg counts all of the args, but *not* self: int total_args = oparg; if (self_or_null != NULL) { args--; @@ -3391,14 +3391,14 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - + TARGET(CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3412,7 +3412,7 @@ if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } callargs = stack_pointer[-1 - ((oparg & 1))]; func = stack_pointer[-3 - ((oparg & 1))]; - // DICT_MERGE is called before this opcode if there are kwargs. + // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { @@ -3484,7 +3484,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(MAKE_FUNCTION) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3492,7 +3492,7 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { @@ -3504,7 +3504,7 @@ stack_pointer[-1] = func; DISPATCH(); } - + TARGET(SET_FUNCTION_ATTRIBUTE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3513,7 +3513,7 @@ PyObject *attr; func = stack_pointer[-1]; attr = stack_pointer[-2]; - assert(PyFunction_Check(func)); + assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: @@ -3541,7 +3541,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(RETURN_GENERATOR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3569,7 +3569,7 @@ LOAD_IP(frame->return_offset); goto resume_frame; } - + TARGET(BUILD_SLICE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3581,7 +3581,7 @@ if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - slice = PySlice_New(start, stop, step); + slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); @@ -3590,7 +3590,7 @@ stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); DISPATCH(); } - + TARGET(CONVERT_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3598,7 +3598,7 @@ PyObject *value; PyObject *result; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; + convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); @@ -3607,7 +3607,7 @@ stack_pointer[-1] = result; DISPATCH(); } - + TARGET(FORMAT_SIMPLE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3615,7 +3615,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result + /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); @@ -3628,7 +3628,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(FORMAT_WITH_SPEC) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3638,7 +3638,7 @@ PyObject *res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); + res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; @@ -3646,7 +3646,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(COPY) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3654,13 +3654,13 @@ PyObject *bottom; PyObject *top; bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); + assert(oparg > 0); top = Py_NewRef(bottom); stack_pointer[0] = top; stack_pointer += 1; DISPATCH(); } - + TARGET(SWAP) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3669,13 +3669,13 @@ PyObject *bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - + assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; DISPATCH(); } - + TARGET(INSTRUMENTED_INSTRUCTION) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -3691,7 +3691,7 @@ opcode = next_opcode; DISPATCH_GOTO(); } - + TARGET(INSTRUMENTED_JUMP_FORWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; @@ -3699,7 +3699,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - + TARGET(INSTRUMENTED_JUMP_BACKWARD) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3708,7 +3708,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3723,7 +3723,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3738,7 +3738,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3759,7 +3759,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -3780,7 +3780,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - + TARGET(EXTENDED_ARG) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3791,7 +3791,7 @@ PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } - + TARGET(CACHE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3799,7 +3799,7 @@ assert(0 && "Executing a cache."); Py_UNREACHABLE(); } - + TARGET(RESERVED) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3807,7 +3807,7 @@ assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } - + TARGET(END_FOR) { frame->instr_ptr = next_instr; next_instr += 1; @@ -3826,7 +3826,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(TO_BOOL) { frame->instr_ptr = next_instr; next_instr += 4; @@ -3853,7 +3853,7 @@ } // _TO_BOOL { - + int err = PyObject_IsTrue(value); Py_DECREF(value); if (err < 0) goto pop_1_error; @@ -3862,7 +3862,7 @@ stack_pointer[-1] = res; DISPATCH(); } - + TARGET(BINARY_OP_MULTIPLY_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -3874,13 +3874,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_INT { - + STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -3891,7 +3891,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -3903,12 +3903,12 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_INT { - + STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -3919,7 +3919,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_SUBTRACT_INT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -3931,12 +3931,12 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_INT { - + STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -3947,7 +3947,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_MULTIPLY_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -3959,12 +3959,12 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_FLOAT { - + STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -3975,7 +3975,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_ADD_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -3987,12 +3987,12 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_FLOAT { - + STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -4003,7 +4003,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4015,12 +4015,12 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { - + STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -4031,7 +4031,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4043,12 +4043,12 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { - + STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -4059,7 +4059,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4070,7 +4070,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_INPLACE_ADD_UNICODE @@ -4103,7 +4103,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(BINARY_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4131,7 +4131,7 @@ } // _BINARY_SUBSCR { - + res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); @@ -4141,7 +4141,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(STORE_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4181,7 +4181,7 @@ stack_pointer += -3; DISPATCH(); } - + TARGET(RETURN_VALUE) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4211,7 +4211,7 @@ #endif DISPATCH(); } - + TARGET(RETURN_CONST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -4220,7 +4220,7 @@ PyObject *retval; // _LOAD_CONST { - value = GETITEM(FRAME_CO_CONSTS, oparg); + value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); } // _POP_FRAME @@ -4249,7 +4249,7 @@ } DISPATCH(); } - + TARGET(SEND) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4278,7 +4278,7 @@ // _SEND v = stack_pointer[-1]; { - assert(frame != &entry_frame); + assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) @@ -4318,7 +4318,7 @@ stack_pointer[-1] = retval; DISPATCH(); } - + TARGET(UNPACK_SEQUENCE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4345,7 +4345,7 @@ } // _UNPACK_SEQUENCE { - + PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); @@ -4354,7 +4354,7 @@ stack_pointer += -1 + oparg; DISPATCH(); } - + TARGET(STORE_ATTR) { frame->instr_ptr = next_instr; next_instr += 5; @@ -4392,7 +4392,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(LOAD_GLOBAL) { frame->instr_ptr = next_instr; next_instr += 5; @@ -4419,7 +4419,7 @@ } // _LOAD_GLOBAL { - + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -4460,7 +4460,7 @@ stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_GLOBAL_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -4494,7 +4494,7 @@ stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_GLOBAL_BUILTIN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -4535,7 +4535,7 @@ stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_SUPER_ATTR) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4568,7 +4568,7 @@ // _LOAD_SUPER_ATTR self = stack_pointer[-1]; { - TIER_ONE_ONLY + TIER_ONE_ONLY if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4611,7 +4611,7 @@ stack_pointer += -2 + ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR) { frame->instr_ptr = next_instr; next_instr += 10; @@ -4640,7 +4640,7 @@ } // _LOAD_ATTR { - + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ @@ -4677,7 +4677,7 @@ stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -4696,7 +4696,7 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { - + assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); @@ -4719,7 +4719,7 @@ stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -4757,7 +4757,7 @@ stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -4776,7 +4776,7 @@ } // _CHECK_ATTR_WITH_HINT { - + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); @@ -4813,7 +4813,7 @@ stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_SLOT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -4848,7 +4848,7 @@ stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(LOAD_ATTR_CLASS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -4880,7 +4880,7 @@ stack_pointer += ((oparg & 1)); DISPATCH(); } - + TARGET(STORE_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -4898,7 +4898,7 @@ } // _GUARD_DORV_VALUES { - + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); @@ -4923,7 +4923,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(STORE_ATTR_SLOT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; @@ -4954,7 +4954,7 @@ stack_pointer += -2; DISPATCH(); } - + TARGET(COMPARE_OP) { frame->instr_ptr = next_instr; next_instr += 2; @@ -4982,7 +4982,7 @@ } // _COMPARE_OP { - + assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); @@ -4999,7 +4999,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -5016,7 +5016,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_FALSE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -5032,7 +5032,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -5043,7 +5043,7 @@ // _IS_NONE value = stack_pointer[-1]; { - if (Py_IsNone(value)) { + if (Py_IsNone(value)) { b = Py_True; } else { @@ -5064,7 +5064,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(POP_JUMP_IF_NOT_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; @@ -5075,7 +5075,7 @@ // _IS_NONE value = stack_pointer[-1]; { - if (Py_IsNone(value)) { + if (Py_IsNone(value)) { b = Py_True; } else { @@ -5096,7 +5096,7 @@ stack_pointer += -1; DISPATCH(); } - + TARGET(FOR_ITER) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5123,7 +5123,7 @@ } // _FOR_ITER { - + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next == NULL) { @@ -5150,7 +5150,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(FOR_ITER_LIST) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5161,12 +5161,12 @@ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { - + _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); @@ -5185,7 +5185,7 @@ } // _ITER_NEXT_LIST { - + _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -5197,7 +5197,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(FOR_ITER_TUPLE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5208,11 +5208,11 @@ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { - + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); @@ -5231,7 +5231,7 @@ } // _ITER_NEXT_TUPLE { - + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -5243,7 +5243,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(FOR_ITER_RANGE) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5254,12 +5254,12 @@ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { - + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); @@ -5273,7 +5273,7 @@ } // _ITER_NEXT_RANGE { - + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -5287,7 +5287,7 @@ stack_pointer += 1; DISPATCH(); } - + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -5306,7 +5306,7 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); @@ -5334,7 +5334,7 @@ stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_METHOD_NO_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -5369,7 +5369,7 @@ stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -5387,7 +5387,7 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); @@ -5413,7 +5413,7 @@ stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -5445,7 +5445,7 @@ stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; @@ -5464,7 +5464,7 @@ } // _CHECK_ATTR_METHOD_LAZY_DICT { - + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); @@ -5487,7 +5487,7 @@ stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - + TARGET(CALL) { frame->instr_ptr = next_instr; next_instr += 4; @@ -5518,7 +5518,7 @@ } // _CALL { - + // oparg counts all of the args, but *not* self: int total_args = oparg; if (self_or_null != NULL) { @@ -5591,7 +5591,7 @@ CHECK_EVAL_BREAKER(); DISPATCH(); } - + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -5612,12 +5612,12 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { - + STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS @@ -5639,7 +5639,7 @@ } // _CHECK_STACK_SPACE { - + PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); @@ -5649,7 +5649,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; + int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; @@ -5694,7 +5694,7 @@ stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(CALL_PY_EXACT_ARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -5721,7 +5721,7 @@ } // _CHECK_STACK_SPACE { - + PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); @@ -5731,7 +5731,7 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int argcount = oparg; + int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; @@ -5776,7 +5776,7 @@ stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; next_instr += 2; @@ -5807,7 +5807,7 @@ } // _BINARY_OP { - + assert(_PyEval_BinaryOps[oparg]); res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); @@ -5818,5 +5818,5 @@ stack_pointer += -1; DISPATCH(); } - + #undef TIER_ONE diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 7e10954d2788ed..0fb9cfe4a3ca40 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -46,7 +46,7 @@ def emit_token(self, tkn: Token): self.maybe_indent(tkn.text) def emit_text(self, txt: str): - if self.column == 0: + if self.column == 0 and txt.strip(): self.out.write(" " * self.indent) self.column = self.initial_indent * 4 self.out.write(txt) From efdcda127918376efb1038762c2ca4025113d89f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 04:20:09 +0000 Subject: [PATCH 07/40] Don't treat newline at end of '//' comment as part of the comment --- Python/generated_cases.c.h | 294 +++++++++++++++--------------- Tools/cases_generator/analysis.py | 4 +- Tools/cases_generator/lexer.py | 2 +- Tools/cases_generator/parser.py | 4 +- 4 files changed, 149 insertions(+), 155 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 59a19a368eaef6..2f55e6f0d67dd3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -120,8 +120,8 @@ INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyObject *value; value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); @@ -364,8 +364,8 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -402,8 +402,8 @@ PyObject *res; value = stack_pointer[-1]; uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); + // This one is a bit weird, because we expect *some* failures: + assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); STAT_INC(TO_BOOL, hit); Py_DECREF(value); @@ -439,9 +439,9 @@ container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { res = NULL; } else { @@ -494,8 +494,8 @@ list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -523,8 +523,8 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; @@ -546,8 +546,8 @@ tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -579,8 +579,7 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error; - // not found or error - + // not found or error stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -658,11 +657,11 @@ DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -807,8 +806,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -831,8 +830,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -1019,10 +1018,10 @@ INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -1205,8 +1204,8 @@ GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { + // Can't use ERROR_IF here. + if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1326,8 +1325,8 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { + // Can't use ERROR_IF here. + if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1437,9 +1436,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { GOTO_ERROR(error); @@ -1454,9 +1453,9 @@ INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); GOTO_ERROR(error); } @@ -1714,8 +1713,8 @@ PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + GOTO_ERROR(error); // Pop the keys and values. + } map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -1783,8 +1782,8 @@ dict = stack_pointer[-3 - (oparg - 1)]; assert(PyDict_CheckExact(dict)); /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; stack_pointer += -2; DISPATCH(); } @@ -1793,9 +1792,9 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -1854,8 +1853,8 @@ if (true) goto pop_3_error; } if (method_found) { - self_or_null = self; // transfer ownership - } else { + self_or_null = self; // transfer ownership + } else { Py_DECREF(self); self_or_null = NULL; } @@ -1890,8 +1889,8 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); new_frame->localsplus[0] = owner; frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -1922,8 +1921,8 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -1998,13 +1997,12 @@ STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - + // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2028,13 +2026,12 @@ _PyLong_DigitCount((PyLongObject *)right) <= 1); Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - + // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2060,8 +2057,7 @@ assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - + // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -2216,14 +2212,14 @@ * do the 2s complement comparison manually */ uint16_t ucounter = this_instr[1].cache + (1 << 15); uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); next_instr = this_instr; this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } @@ -2289,8 +2285,8 @@ PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; @@ -2310,21 +2306,21 @@ names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } + assert(PyTuple_CheckExact(attrs)); // Success! + } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = Py_None; // Failure! - } + // Error! + attrs = Py_None; // Failure! + } stack_pointer[-3] = attrs; stack_pointer += -2; DISPATCH(); @@ -2367,8 +2363,8 @@ PyObject *values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -2600,8 +2596,8 @@ Py_DECREF(tb); } assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; @@ -2690,8 +2686,8 @@ PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } @@ -2714,8 +2710,7 @@ STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - + Py_DECREF(&PyType_Type); // I.e., callable stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -2739,8 +2734,8 @@ PyObject *arg = args[0]; res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2766,8 +2761,8 @@ PyObject *arg = args[0]; res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2907,9 +2902,9 @@ DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } PyObject *arg = args[0]; @@ -3101,13 +3096,13 @@ DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + goto pop_1_error; // Since arg is DECREF'ed already + } Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); DISPATCH(); } @@ -3138,9 +3133,9 @@ DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); @@ -3225,9 +3220,9 @@ DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); @@ -3316,8 +3311,8 @@ args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg counts all of the args, but *not* self: + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -3333,8 +3328,8 @@ callable = method; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { @@ -3345,11 +3340,11 @@ args, positional_args, kwnames ); Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } assert(next_instr - this_instr == 1); @@ -3412,9 +3407,9 @@ if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } callargs = stack_pointer[-1 - ((oparg & 1))]; func = stack_pointer[-3 - ((oparg & 1))]; - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { GOTO_ERROR(error); @@ -3462,8 +3457,8 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { GOTO_ERROR(error); } @@ -4096,8 +4091,8 @@ PyUnicode_Append(target_local, right); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); } stack_pointer += -2; @@ -4196,8 +4191,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -4233,8 +4228,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); @@ -4576,9 +4571,9 @@ frame, this_instr, global_super, arg); if (err) goto pop_3_error; } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -4650,9 +4645,9 @@ meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } else { /* meth is not an unbound method (but a regular attr, or something was returned by a descriptor protocol). Set @@ -5143,8 +5138,7 @@ JUMPBY(oparg + 1); DISPATCH(); } - // Common case: no jump, leave it to the code generator - + // Common case: no jump, leave it to the code generator } stack_pointer[0] = next; stack_pointer += 1; @@ -5266,8 +5260,8 @@ if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); DISPATCH(); } } @@ -5519,8 +5513,8 @@ // _CALL { - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg counts all of the args, but *not* self: + int total_args = oparg; if (self_or_null != NULL) { args--; total_args++; @@ -5535,8 +5529,8 @@ Py_DECREF(callable); callable = method; } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { @@ -5546,11 +5540,11 @@ tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -5620,10 +5614,10 @@ STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = self; @@ -5673,9 +5667,9 @@ } // _PUSH_FRAME { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; @@ -5755,9 +5749,9 @@ } // _PUSH_FRAME { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 76e625a42ea5cc..603b15596f16de 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -17,8 +17,8 @@ import parsing from parsing import StackEffect -BEGIN_MARKER = "// BEGIN BYTECODES //\n" -END_MARKER = "// END BYTECODES //\n" +BEGIN_MARKER = "// BEGIN BYTECODES //" +END_MARKER = "// END BYTECODES //" RESERVED_WORDS = { "co_consts": "Use FRAME_CO_CONSTS.", diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 66adbb64d11ac0..856bd678ba8908 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -112,7 +112,7 @@ def choice(*opts: str) -> str: char = r"\'.\'" # TODO: escape sequence CHARACTER = "CHARACTER" -comment_re = r"(//.*\n)|/\*([^*]|\*[^/])*\*/" +comment_re = r"(//.*)|/\*([^*]|\*[^/])*\*/" COMMENT = "COMMENT" newline = r"\n" diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index eb485aec5cce5e..55a679f80df38b 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -2,8 +2,8 @@ from formatting import prettify_filename -BEGIN_MARKER = "// BEGIN BYTECODES //\n" -END_MARKER = "// END BYTECODES //\n" +BEGIN_MARKER = "// BEGIN BYTECODES //" +END_MARKER = "// END BYTECODES //" AstNode = InstDef | Macro | Pseudo | Family From f2d6a21722014bfd6642b54587a815b75505ea71 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 04:37:42 +0000 Subject: [PATCH 08/40] Add back static asserts --- Python/generated_cases.c.h | 70 ++++++++++++++++++++++++ Tools/cases_generator/analyzer.py | 2 + Tools/cases_generator/tier1_generator.py | 6 +- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2f55e6f0d67dd3..ad194586424a32 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -47,6 +47,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -313,6 +314,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; value = stack_pointer[-1]; @@ -325,6 +327,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -346,6 +349,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -361,6 +365,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -376,6 +381,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -398,6 +404,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -487,6 +494,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *res; @@ -513,6 +521,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *str; PyObject *res; @@ -539,6 +548,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *tuple; PyObject *res; @@ -565,6 +575,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *res; @@ -589,6 +600,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -648,6 +660,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *value; @@ -677,6 +690,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *value; @@ -961,6 +975,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; @@ -1218,6 +1233,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1238,6 +1254,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1258,6 +1275,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1802,6 +1820,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; @@ -1829,6 +1848,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; @@ -1868,6 +1888,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -1900,6 +1921,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -1933,6 +1955,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; @@ -1986,6 +2009,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2012,6 +2036,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2041,6 +2066,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2457,6 +2483,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; iter = stack_pointer[-1]; @@ -2651,6 +2678,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -2696,6 +2724,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; @@ -2720,6 +2749,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; @@ -2747,6 +2777,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; @@ -2774,6 +2805,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; @@ -2850,6 +2882,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -2884,6 +2917,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -2925,6 +2959,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -2970,6 +3005,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3009,6 +3045,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3045,6 +3082,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3083,6 +3121,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self; PyObject *callable; @@ -3111,6 +3150,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3156,6 +3196,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3198,6 +3239,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3242,6 +3284,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -3862,6 +3905,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -3891,6 +3935,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -3919,6 +3964,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -3947,6 +3993,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -3975,6 +4022,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -4003,6 +4051,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -4031,6 +4080,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -4059,6 +4109,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -4460,6 +4511,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; /* Skip 1 cache entry */ @@ -4494,6 +4546,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; /* Skip 1 cache entry */ @@ -4677,6 +4730,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -4719,6 +4773,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -4757,6 +4812,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -4813,6 +4869,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -4848,6 +4905,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -4880,6 +4938,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; /* Skip 1 cache entry */ @@ -4923,6 +4982,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; /* Skip 1 cache entry */ @@ -5149,6 +5209,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; /* Skip 1 cache entry */ @@ -5196,6 +5257,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; /* Skip 1 cache entry */ @@ -5242,6 +5304,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; /* Skip 1 cache entry */ @@ -5286,6 +5349,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self = NULL; @@ -5333,6 +5397,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self = NULL; @@ -5368,6 +5433,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; /* Skip 1 cache entry */ @@ -5412,6 +5478,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; /* Skip 1 cache entry */ @@ -5444,6 +5511,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self = NULL; @@ -5590,6 +5658,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *null; PyObject *callable; PyObject *func; @@ -5693,6 +5762,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *self_or_null; PyObject *callable; PyObject **args; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 11b69c1d3264e1..d9716eade7acc5 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -159,6 +159,7 @@ def dump(self, indent): @dataclass class Family: name: str + size: str members: list[Instruction] def dump(self, indent): @@ -308,6 +309,7 @@ def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: d def add_family(family: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]): family = Family( family.name, + family.size, [ instructions[member] for member in family.members ] ) for member in family.members: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 3a5f48ba007b06..1a6eb1964873d2 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -276,8 +276,10 @@ def generate_tier1(filename: str, analysis: Analysis, outfile: TextIO) -> None: out.emit(f"PREDICTED({name});\n") if needs_this: out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n") - #out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) - + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n') declare_variables(inst, out) offset = 1 # The instruction itself stack = Stack() From 2b593dca88f4f8a2a0c67076d19e4f184b7a7f26 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 04:50:19 +0000 Subject: [PATCH 09/40] Fix type annotation --- Tools/cases_generator/tier1_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 1a6eb1964873d2..58e53c0a000c96 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -30,7 +30,7 @@ def __init__(self): self.base_offset: StackOffset = StackOffset() self.peek_offset: StackOffset = StackOffset() self.variables: list[StackItem] = [] - self.defined: Set[str] = set() + self.defined: set[str] = set() def pop(self, var: StackItem) -> str: self.top_offset.pop(var) @@ -263,7 +263,7 @@ def generate_tier1(filename: str, analysis: Analysis, outfile: TextIO) -> None: write_header(filename, outfile) out = CWriter(outfile, 2) out.emit("\n") - for name, inst in analysis.instructions.items(): + for name, inst in sorted(analysis.instructions.items()): needs_this = uses_this(inst) out.emit(f"TARGET({name}) {{") if needs_this and not inst.is_target: From 0a329a8e437f400ab0d6a0c33117f5718b0d31ac Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 05:10:46 +0000 Subject: [PATCH 10/40] Use argparse and support multiple input files --- Python/generated_cases.c.h | 9325 +++++++++++----------- Tools/cases_generator/analyzer.py | 4 +- Tools/cases_generator/parser.py | 61 +- Tools/cases_generator/tier1_generator.py | 65 +- 4 files changed, 4764 insertions(+), 4691 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ad194586424a32..7bc871b8495a69 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,6 +1,6 @@ -// This file is generated by /home/mark/cpython/Tools/cases_generator/tier1_generator.py +// This file is generated by Tools/cases_generator/tier1_generator.py // from: -// Python/bytecodes.c +// ['Python/bytecodes.c'] // Do not edit! #ifdef TIER_TWO @@ -8,428 +8,385 @@ #endif #define TIER_ONE 1 - TARGET(NOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); - } - - TARGET(RESUME) { + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; - TIER_ONE_ONLY - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); } - this_instr->op.code = RESUME_CHECK; + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(RESUME_CHECK) { + TARGET(BEFORE_WITH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } - - TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + INSTRUCTION_STATS(BEFORE_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ + */ + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); } - next_instr = this_instr; + GOTO_ERROR(error); } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = this_instr; - DISPATCH(); + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(LOAD_FAST_CHECK) { + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *rhs; + PyObject *lhs; + PyObject *res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST) { + TARGET(BINARY_OP_ADD_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST_AND_CLEAR) { + TARGET(BINARY_OP_ADD_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - stack_pointer[0] = value; - stack_pointer += 1; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_FAST_LOAD_FAST) { + TARGET(BINARY_OP_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - DISPATCH(); - } + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { - TARGET(LOAD_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_UNICODE + { - TARGET(STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); } - TARGET(STORE_FAST_LOAD_FAST) { + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); - stack_pointer[-1] = value2; - DISPATCH(); - } + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_INPLACE_ADD_UNICODE + { - TARGET(STORE_FAST_STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } stack_pointer += -2; DISPATCH(); } - TARGET(POP_TOP) { + TARGET(BINARY_OP_MULTIPLY_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); } - TARGET(PUSH_NULL) { + TARGET(BINARY_OP_MULTIPLY_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; PyObject *res; - res = NULL; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_INT + { - TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - /* Need to create a fake StopIteration error here, - * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; } - Py_DECREF(receiver); - Py_DECREF(value); - stack_pointer += -2; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(END_SEND) { + TARGET(BINARY_OP_SUBTRACT_FLOAT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { - TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } - Py_DECREF(receiver); - stack_pointer[-2] = value; + // _BINARY_OP_SUBTRACT_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); } - TARGET(UNARY_NEGATIVE) { + TARGET(BINARY_OP_SUBTRACT_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NOT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - value = stack_pointer[-1]; - - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); - } - - TARGET(TO_BOOL_INT) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - Py_DECREF(value); - res = Py_True; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_LIST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_NONE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } + // _BINARY_OP_SUBTRACT_INT + { - TARGET(TO_BOOL_STR) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_INVERT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -462,147 +419,77 @@ DISPATCH(); } - TARGET(STORE_SLICE) { + TARGET(BINARY_SUBSCR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; + PyObject *res; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + // _BINARY_SUBSCR + { + + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - stack_pointer += -4; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SUBSCR_LIST_INT) { + TARGET(BINARY_SUBSCR_DICT) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; - PyObject *list; + PyObject *dict; PyObject *res; sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + dict = stack_pointer[-2]; + + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + // not found or error stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); } - TARGET(BINARY_SUBSCR_STR_INT) { - frame->instr_ptr = next_instr; + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_TUPLE_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error; - // not found or error - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; + PyObject *container; sub = stack_pointer[-1]; container = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); @@ -628,1380 +515,1512 @@ DISPATCH_INLINED(new_frame); } - TARGET(LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(SET_ADD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_LIST_INT) { + TARGET(BINARY_SUBSCR_LIST_INT) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; - PyObject *value; + PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - value = stack_pointer[-3]; - - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - stack_pointer += -3; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(STORE_SUBSCR_DICT) { + TARGET(BINARY_SUBSCR_STR_INT) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; - PyObject *dict; - PyObject *value; + PyObject *str; + PyObject *res; sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - stack_pointer += -3; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(DELETE_SUBSCR) { + TARGET(BINARY_SUBSCR_TUPLE_INT) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; - PyObject *container; + PyObject *tuple; + PyObject *res; sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; - stack_pointer += -2; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_INTRINSIC_1) { + TARGET(BUILD_CONST_KEY_MAP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + GOTO_ERROR(error); // Pop the keys and values. + } + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); + } + Py_DECREF(keys); + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; DISPATCH(); } - TARGET(CALL_INTRINSIC_2) { + TARGET(BUILD_LIST) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; + INSTRUCTION_STATS(BUILD_LIST); + PyObject **values; + PyObject *list; + values = &stack_pointer[-oparg]; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(RAISE_VARARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - PyObject **args; - args = &stack_pointer[-oparg]; - TIER_ONE_ONLY - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; + INSTRUCTION_STATS(BUILD_MAP); + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); } - if (true) { stack_pointer += -oparg; goto error; } - stack_pointer += -oparg; + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } - TARGET(INTERPRETER_EXIT) { + TARGET(BUILD_SET) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; - } - - TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + INSTRUCTION_STATS(BUILD_SET); + PyObject **values; + PyObject *set; + values = &stack_pointer[-oparg]; + set = PySet_New(NULL); + if (set == NULL) + GOTO_ERROR(error); + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set); + if (true) { stack_pointer += -oparg; goto error; } + } + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + DISPATCH(); } - TARGET(INSTRUMENTED_RETURN_CONST) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + INSTRUCTION_STATS(BUILD_SLICE); + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); + DISPATCH(); } - TARGET(GET_AITER) { + TARGET(BUILD_STRING) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; + INSTRUCTION_STATS(BUILD_STRING); + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); } - stack_pointer[-1] = iter; + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(GET_ANEXT) { + TARGET(BUILD_TUPLE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - GOTO_ERROR(error); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } - } - stack_pointer[0] = awaitable; - stack_pointer += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + PyObject **values; + PyObject *tup; + values = &stack_pointer[-oparg]; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; DISPATCH(); } - TARGET(GET_AWAITABLE) { + TARGET(CACHE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); + INSTRUCTION_STATS(CACHE); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } - } - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; - PyObject *receiver; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } - - TARGET(YIELD_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); } - TARGET(POP_EXCEPT) { + TARGET(CALL) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(RERAISE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - TIER_ONE_ONLY - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + // _SPECIALIZE_CALL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + DISPATCH_SAME_OPARG(); } + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } + // _CALL + { - TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; - TIER_ONE_ONLY - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } - stack_pointer += -2; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(CLEANUP_THROW) { + TARGET(CALL_ALLOC_AND_ENTER_INIT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; - } - else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(LOAD_ASSERTION_ERROR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_BUILD_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error; - } - stack_pointer[0] = bc; - stack_pointer += 1; - DISPATCH(); - } + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; - TARGET(STORE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + GOTO_ERROR(error); } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); + Py_DECREF(tp); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + goto start_frame; } - TARGET(DELETE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - GOTO_ERROR(error); + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *func; + PyObject *self; + PyObject *self_or_null; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); } - err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - GOTO_ERROR(error); + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } - DISPATCH(); - } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { - TARGET(UNPACK_SEQUENCE_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { - TARGET(UNPACK_SEQUENCE_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - Py_DECREF(seq); - stack_pointer += -1 + oparg; + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(UNPACK_EX) { + TARGET(CALL_BUILTIN_CLASS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; - seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - stack_pointer += (oparg >> 8) + (oparg & 0xFF); + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(DELETE_ATTR) { + TARGET(CALL_BUILTIN_FAST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; - owner = stack_pointer[-1]; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - stack_pointer += -1; + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(STORE_GLOBAL) { + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(DELETE_GLOBAL) { + TARGET(CALL_BUILTIN_O) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { GOTO_ERROR(error); } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LOAD_LOCALS) { + TARGET(CALL_FUNCTION_EX) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - stack_pointer[0] = locals; - stack_pointer += 1; - DISPATCH(); - } + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; + PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { GOTO_ERROR(error); } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + GOTO_ERROR(error); } + Py_SETREF(callargs, tuple); } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - DISPATCH(); - } - - TARGET(LOAD_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) GOTO_ERROR(error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { GOTO_ERROR(error); } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); } + result = PyObject_Call(func, callargs, kwargs); } - stack_pointer[0] = v; - stack_pointer += 1; + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + assert(PEEK(2 + (oparg & 1)) == NULL); + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(DELETE_FAST) { + TARGET(CALL_INTRINSIC_1) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); + INSTRUCTION_STATS(CALL_INTRINSIC_1); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(MAKE_CELL) { + TARGET(CALL_INTRINSIC_2) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - GOTO_ERROR(error); - } - SETLOCAL(oparg, cell); + INSTRUCTION_STATS(CALL_INTRINSIC_2); + PyObject *value1; + PyObject *value2; + PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(DELETE_DEREF) { + TARGET(CALL_ISINSTANCE) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { GOTO_ERROR(error); } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - TARGET(LOAD_FROM_DICT_OR_DEREF) { + TARGET(CALL_KW) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwnames; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames + ); + Py_DECREF(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { GOTO_ERROR(error); } - Py_INCREF(value); + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); } - Py_DECREF(class_dict); - stack_pointer[-1] = value; + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + Py_DECREF(kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LOAD_DEREF) { + TARGET(CALL_LEN) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + GOTO_ERROR(error); + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - TARGET(STORE_DEREF) { + TARGET(CALL_LIST_APPEND) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - stack_pointer += -1; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self; + PyObject *callable; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); + DEOPT_IF(!PyList_Check(self), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } - TARGET(COPY_FREE_VARS) { + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BUILD_STRING) { + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; - pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - if (str == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BUILD_TUPLE) { + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; - values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(BUILD_LIST) { + TARGET(CALL_METHOD_DESCRIPTOR_O) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; - values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(LIST_EXTEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - } - Py_DECREF(iterable); - if (true) goto pop_1_error; + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *self_or_null; + PyObject *callable; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(SET_UPDATE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } + // _CHECK_STACK_SPACE + { - TARGET(BUILD_SET) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; - values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } - if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error; } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } } - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - DISPATCH(); - } + // _SAVE_RETURN_OFFSET + { - TARGET(BUILD_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; - values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - if (map == NULL) { stack_pointer += -oparg*2; goto error; } - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(SETUP_ANNOTATIONS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error; + TARGET(CALL_PY_WITH_DEFAULTS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(tstate->interp->eval_frame, CALL); + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; + DEOPT_IF(argcount > code->co_argcount, CALL); + DEOPT_IF(argcount < min_args, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; } - else { - Py_DECREF(ann_dict); + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); + new_frame->localsplus[i] = Py_NewRef(def); } - DISPATCH(); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(BUILD_CONST_KEY_MAP) { + TARGET(CALL_STR_1) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error; } - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(DICT_UPDATE) { + TARGET(CALL_TUPLE_1) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - PyObject *update; - PyObject *dict; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); - } - Py_DECREF(update); - if (true) goto pop_1_error; - } - Py_DECREF(update); - stack_pointer += -1; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(DICT_MERGE) { + TARGET(CALL_TYPE_1) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; - } - Py_DECREF(update); - stack_pointer += -1; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } - TARGET(MAP_ADD) { + TARGET(CHECK_EG_MATCH) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - } + INSTRUCTION_STATS(CHECK_EG_MATCH); + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; + } + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; - TARGET(LOAD_SUPER_ATTR_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - stack_pointer[-3] = attr; - stack_pointer += -2 + (((0) ? 1 : 0)); + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } + stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } - TARGET(LOAD_SUPER_ATTR_METHOD) { + TARGET(CHECK_EXC_MATCH) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; - } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; } - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; DISPATCH(); } - TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + TARGET(CLEANUP_THROW) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(type_version != 0); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; + PyObject *none; + PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + TIER_ONE_ONLY + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); + none = Py_None; + } + else { + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); } - TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); + TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *right; + PyObject *left; + PyObject *res; + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); + // _COMPARE_OP + { + + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; + } } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - stack_pointer += -2; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2015,7 +2034,6 @@ PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2032,236 +2050,330 @@ DISPATCH(); } - TARGET(COMPARE_OP_INT) { + TARGET(COMPARE_OP_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONTAINS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; + + convertion_func_ptr conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = CONVERSION_FUNCTIONS[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; + DISPATCH(); + } + + TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; + + assert(oparg > 0); + top = Py_NewRef(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); + } + DISPATCH(); + } + + TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + PyObject *owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); + DISPATCH(); + } + + TARGET(DELETE_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); DISPATCH(); } - TARGET(COMPARE_OP_STR) { + TARGET(DELETE_GLOBAL) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err; + err = PyDict_DelItem(GLOBALS(), name); + // Can't use ERROR_IF here. + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + GOTO_ERROR(error); + } DISPATCH(); } - TARGET(IS_OP) { + TARGET(DELETE_NAME) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + GOTO_ERROR(error); + } + err = PyObject_DelItem(ns, name); + // Can't use ERROR_IF here. + if (err != 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + GOTO_ERROR(error); + } DISPATCH(); } - TARGET(CONTAINS_OP) { + TARGET(DELETE_SUBSCR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CONTAINS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; + INSTRUCTION_STATS(DELETE_SUBSCR); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_2_error; + stack_pointer += -2; DISPATCH(); } - TARGET(CHECK_EG_MATCH) { + TARGET(DICT_MERGE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; + INSTRUCTION_STATS(DICT_MERGE); + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + Py_DECREF(update); + stack_pointer += -1; DISPATCH(); } - TARGET(CHECK_EXC_MATCH) { + TARGET(DICT_UPDATE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + INSTRUCTION_STATS(DICT_UPDATE); + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); if (true) goto pop_1_error; } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; + Py_DECREF(update); + stack_pointer += -1; DISPATCH(); } - TARGET(IMPORT_NAME) { - frame->instr_ptr = next_instr; + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; + INSTRUCTION_STATS(END_ASYNC_FOR); + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + Py_DECREF(awaitable); + Py_DECREF(exc); + } + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer += -2; DISPATCH(); } - TARGET(IMPORT_FROM) { + TARGET(END_FOR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; - from = stack_pointer[-1]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; + INSTRUCTION_STATS(END_FOR); + PyObject *value; + // _POP_TOP + value = stack_pointer[-1]; + { + Py_DECREF(value); + } + // _POP_TOP + value = stack_pointer[-2]; + { + Py_DECREF(value); + } + stack_pointer += -2; DISPATCH(); } - TARGET(JUMP_FORWARD) { + TARGET(END_SEND) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); - } + INSTRUCTION_STATS(END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; - TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so - * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); - if (optimized < 0) goto error; - if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - } - else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; - } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; - } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; - } - } - #endif /* ENABLE_SPECIALIZATION */ + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -2290,192 +2402,124 @@ goto enter_tier_one; } - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + TARGET(EXIT_INIT_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + INSTRUCTION_STATS(EXIT_INIT_CHECK); + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; - /* This bytecode is used in the `yield from` or `await` loop. - * If there is an interrupt, we want it handled in the innermost - * generator or coroutine, so we deliberately do not check it here. - * (see bpo-30039). - */ - JUMPBY(-oparg); + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + GOTO_ERROR(error); + } + stack_pointer += -1; DISPATCH(); } - TARGET(GET_LEN) { + TARGET(EXTENDED_ARG) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - PyObject *obj; - PyObject *len_o; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - stack_pointer[0] = len_o; - stack_pointer += 1; - DISPATCH(); + INSTRUCTION_STATS(EXTENDED_ARG); + + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); } - TARGET(MATCH_CLASS) { + TARGET(FORMAT_SIMPLE) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + INSTRUCTION_STATS(FORMAT_SIMPLE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = Py_None; // Failure! + res = value; } - stack_pointer[-3] = attrs; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(MATCH_MAPPING) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-1] = res; DISPATCH(); } - TARGET(MATCH_SEQUENCE) { + TARGET(FORMAT_WITH_SPEC) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + PyObject *fmt_spec; + PyObject *value; PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(MATCH_KEYS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - stack_pointer[0] = values_or_none; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_ITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { + TARGET(FOR_ITER) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyObject *iterable; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - GOTO_ERROR(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; - } - else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - GOTO_ERROR(error); + PyObject *next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ForIter(iter, next_instr, oparg); + DISPATCH_SAME_OPARG(); } - Py_DECREF(iterable); - } - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr; + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + // _FOR_ITER + { + + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; + // Common case: no jump, leave it to the code generator } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2504,1938 +2548,1574 @@ DISPATCH_INLINED(gen_frame); } - TARGET(BEFORE_ASYNC_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_ASYNC_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(BEFORE_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(WITH_EXCEPT_START) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); - } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(PUSH_EXC_INFO) { + TARGET(FOR_ITER_LIST) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; - } - else { - prev_exc = Py_None; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(CALL); - } - - TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); + // _ITER_JUMP_LIST + { - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } + // _ITER_NEXT_LIST + { - TARGET(CALL_TYPE_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - TARGET(CALL_STR_1) { + TARGET(FOR_ITER_RANGE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + Py_DECREF(r); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; + } + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - TARGET(CALL_TUPLE_1) { + TARGET(FOR_ITER_TUPLE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } - TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* This instruction does the following: - * 1. Creates the object (by calling ``object.__new__``) - * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) - * 3. Pushes the frame for ``__init__`` to the frame stack - * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - GOTO_ERROR(error); + TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + if (true) goto pop_1_error; + } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + if (true) goto pop_1_error; } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. - * We don't check recursion depth here, - * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; + stack_pointer[-1] = iter; + DISPATCH(); } - TARGET(EXIT_INIT_CHECK) { + TARGET(GET_ANEXT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + INSTRUCTION_STATS(GET_ANEXT); + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + GOTO_ERROR(error); + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + GOTO_ERROR(error); + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + GOTO_ERROR(error); + } + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + Py_DECREF(next_iter); + GOTO_ERROR(error); + } else { + Py_DECREF(next_iter); + } } - stack_pointer += -1; + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } - TARGET(CALL_BUILTIN_CLASS) { + TARGET(GET_AWAITABLE) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(CALL_BUILTIN_O) { + TARGET(GET_ITER) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(CALL_BUILTIN_FAST) { + TARGET(GET_LEN) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + PyObject *obj; + PyObject *len_o; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + stack_pointer[0] = len_o; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + GOTO_ERROR(error); + } + iter = iterable; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + else if (PyGen_CheckExact(iterable)) { + iter = iterable; } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + else { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + if (iter == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(iterable); + } + stack_pointer[-1] = iter; DISPATCH(); } - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + TARGET(IMPORT_FROM) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + PyObject *from; PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + from = stack_pointer[-1]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } - TARGET(CALL_LEN) { + TARGET(IMPORT_NAME) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + PyObject *fromlist; + PyObject *level; PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); - } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_ISINSTANCE) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + + int is_meth = PEEK(oparg + 1) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 2); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(CALL); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = PEEK(oparg + 2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 3); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PEEK(total_args + 1); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); + } + + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); + Py_DECREF(receiver); + Py_DECREF(value); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } - TARGET(CALL_LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self; - PyObject *callable; - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + oparg + 1; } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_O) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + PyObject *value = POP(); + int flag = Py_IsNone(value); + int offset; + if (flag) { + offset = oparg; } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + else { + Py_DECREF(value); + offset = 0; } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + PyObject *value = POP(); + int offset; + int nflag = Py_IsNone(value); + if (nflag) { + offset = 0; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + else { + Py_DECREF(value); + offset = oparg; } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_KW) { + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); } - TARGET(CALL_KW) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { GOTO_ERROR(error); } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + next_instr = this_instr; } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = this_instr; + DISPATCH(); } } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -3 - oparg; goto error; } - - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - CHECK_EVAL_BREAKER(); DISPATCH(); } - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; + TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } - callargs = stack_pointer[-1 - ((oparg & 1))]; - func = stack_pointer[-3 - ((oparg & 1))]; - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - GOTO_ERROR(error); - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - result = PyObject_Call(func, callargs, kwargs); - } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } - - stack_pointer[-3 - ((oparg & 1))] = result; - stack_pointer += -2 - ((oparg & 1)); - CHECK_EVAL_BREAKER(); - DISPATCH(); + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; } - TARGET(MAKE_FUNCTION) { + TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + TARGET(INTERPRETER_EXIT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; - DISPATCH(); + INSTRUCTION_STATS(INTERPRETER_EXIT); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; } - TARGET(SET_FUNCTION_ATTRIBUTE) { + TARGET(IS_OP) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func; + INSTRUCTION_STATS(IS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-2] = b; stack_pointer += -1; DISPATCH(); } - TARGET(RETURN_GENERATOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); + TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #if ENABLE_SPECIALIZATION + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + /* We are using unsigned values, but we really want signed values, so + * do the 2s complement comparison manually */ + uint16_t ucounter = this_instr[1].cache + (1 << 15); + uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + OPT_STAT_INC(attempts); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + if (optimized < 0) goto error; + if (optimized) { + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + } + else { + int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + if (backoff < MINIMUM_TIER2_BACKOFF) { + backoff = MINIMUM_TIER2_BACKOFF; + } + else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { + backoff++; + } + assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); + this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; + } } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - LOAD_IP(frame->return_offset); - goto resume_frame; + #endif /* ENABLE_SPECIALIZATION */ + DISPATCH(); } - TARGET(BUILD_SLICE) { + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } - stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; - start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } - stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; - stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); DISPATCH(); } - TARGET(CONVERT_VALUE) { + TARGET(JUMP_FORWARD) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - PyObject *value; - PyObject *result; - value = stack_pointer[-1]; - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; - stack_pointer[-1] = result; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); DISPATCH(); } - TARGET(FORMAT_SIMPLE) { + TARGET(LIST_APPEND) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - } - else { - res = value; - } - stack_pointer[-1] = res; + INSTRUCTION_STATS(LIST_APPEND); + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(FORMAT_WITH_SPEC) { + TARGET(LIST_EXTEND) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; - PyObject *value; - PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; + INSTRUCTION_STATS(LIST_EXTEND); + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } + Py_DECREF(iterable); + if (true) goto pop_1_error; + } + assert(Py_IsNone(none_val)); + Py_DECREF(iterable); stack_pointer += -1; DISPATCH(); } - TARGET(COPY) { + TARGET(LOAD_ASSERTION_ERROR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = Py_NewRef(bottom); - stack_pointer[0] = top; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + stack_pointer[0] = value; stack_pointer += 1; DISPATCH(); } - TARGET(SWAP) { + TARGET(LOAD_ATTR) { frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; - DISPATCH(); - } + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); - TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - if (next_opcode < 0) goto error; - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } + // _LOAD_ATTR + { - TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } + } + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + } + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(INSTRUMENTED_JUMP_BACKWARD) { + TARGET(LOAD_ATTR_CLASS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(type_version != 0); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + Py_INCREF(f); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = Py_NewRef(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + TARGET(LOAD_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyObject *value = POP(); - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - else { - Py_DECREF(value); - offset = 0; + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyObject *value = POP(); - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; - } - else { - Py_DECREF(value); - offset = oparg; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(EXTENDED_ARG) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TARGET(CACHE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); - } - - TARGET(RESERVED) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); - } - - TARGET(END_FOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - PyObject *value; - // _POP_TOP - value = stack_pointer[-1]; + // _CHECK_ATTR_METHOD_LAZY_DICT { - Py_DECREF(value); + + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); } - // _POP_TOP - value = stack_pointer[-2]; + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_LAZY_DICT { - Py_DECREF(value); + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; } - stack_pointer += -2; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - TARGET(TO_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject *value; - PyObject *res; - // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _TO_BOOL + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT { + PyObject *descr = read_obj(&this_instr[6].cache); - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; + assert(oparg & 1); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; } - stack_pointer[-1] = res; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - TARGET(BINARY_OP_MULTIPLY_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _BINARY_OP_MULTIPLY_INT + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - stack_pointer[-2] = res; - stack_pointer += -1; + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } - TARGET(BINARY_OP_ADD_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; + TARGET(LOAD_ATTR_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _BINARY_OP_ADD_INT + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { + PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _BINARY_OP_SUBTRACT_INT + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_MULTIPLY_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; + // _GUARD_KEYS_VERSION { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } - // _BINARY_OP_MULTIPLY_FLOAT + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { + PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } - TARGET(BINARY_OP_ADD_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; + TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(type_version != 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _BINARY_OP_ADD_FLOAT + // _LOAD_ATTR_SLOT { + uint16_t index = read_u16(&this_instr[4].cache); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - stack_pointer[-2] = res; - stack_pointer += -1; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(BINARY_OP_SUBTRACT_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; + TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } - // _BINARY_OP_SUBTRACT_FLOAT + // _CHECK_ATTR_WITH_HINT { - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); } - stack_pointer[-2] = res; - stack_pointer += -1; + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } - TARGET(BINARY_OP_ADD_UNICODE) { + TARGET(LOAD_BUILD_CLASS) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_UNICODE - { - - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = bc; + stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + TARGET(LOAD_CONST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_INPLACE_ADD_UNICODE - { - - assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - } - stack_pointer += -2; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(BINARY_SUBSCR) { + TARGET(LOAD_DEREF) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *res; - // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SUBSCR - { + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + PyObject *value; - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; } - stack_pointer[-2] = res; - stack_pointer += -1; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(STORE_SUBSCR) { + TARGET(LOAD_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *v; - // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SUBSCR - v = stack_pointer[-3]; - { - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; - } - stack_pointer += -3; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(RETURN_VALUE) { + TARGET(LOAD_FAST_AND_CLEAR) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - - #if TIER_ONE - assert(frame != &entry_frame); - #endif - stack_pointer += -1; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(RETURN_CONST) { + TARGET(LOAD_FAST_CHECK) { frame->instr_ptr = next_instr; next_instr += 1; - INSTRUCTION_STATS(RETURN_CONST); + INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; - PyObject *retval; - // _LOAD_CONST - { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - } - // _POP_FRAME - retval = value; - { - - #if TIER_ONE - assert(frame != &entry_frame); - #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } - TARGET(SEND) { + TARGET(LOAD_FAST_LOAD_FAST) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *receiver; - PyObject *v; - PyObject *retval; - // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Send(receiver, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _SEND - v = stack_pointer[-1]; - { - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); - } - else { - GOTO_ERROR(error); - } - } - Py_DECREF(v); - } - stack_pointer[-1] = retval; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } - TARGET(UNPACK_SEQUENCE) { + TARGET(LOAD_FROM_DICT_OR_DEREF) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *seq; - // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - (void)seq; - (void)counter; + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + GOTO_ERROR(error); } - // _UNPACK_SEQUENCE - { - - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + Py_INCREF(value); } - stack_pointer += -1 + oparg; + Py_DECREF(class_dict); + stack_pointer[-1] = value; DISPATCH(); } - TARGET(STORE_ATTR) { + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; - PyObject *owner; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + PyObject *mod_or_class_dict; PyObject *v; - // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); } - // _STORE_ATTR - v = stack_pointer[-2]; - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } } - stack_pointer += -2; + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; DISPATCH(); } @@ -4507,10 +4187,10 @@ DISPATCH(); } - TARGET(LOAD_GLOBAL_MODULE) { + TARGET(LOAD_GLOBAL_BUILTIN) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; @@ -4523,13 +4203,20 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE + // _GUARD_BUILTINS_VERSION + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _LOAD_GLOBAL_BUILTINS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); @@ -4542,10 +4229,10 @@ DISPATCH(); } - TARGET(LOAD_GLOBAL_BUILTIN) { + TARGET(LOAD_GLOBAL_MODULE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; @@ -4558,29 +4245,76 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } - // _GUARD_BUILTINS_VERSION - { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - // _LOAD_GLOBAL_BUILTINS + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + Py_INCREF(locals); + stack_pointer[0] = locals; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + PyObject *v; + + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } + } + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } @@ -4660,397 +4394,300 @@ DISPATCH(); } - TARGET(LOAD_ATTR) { + TARGET(LOAD_SUPER_ATTR_ATTR) { frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_ATTR - { - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; - } - } - else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - } - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) goto pop_3_error; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); DISPATCH(); } - TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; + TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + PyObject *self_or_null; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; + } + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + GOTO_ERROR(error); } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + SETLOCAL(oparg, cell); DISPATCH(); } - TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + Py_DECREF(codeobj); + if (func_obj == NULL) { + GOTO_ERROR(error); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = (PyObject *)func_obj; + stack_pointer[-1] = func; DISPATCH(); } - TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; + TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); PyObject *value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _GUARD_DORV_VALUES - { + PyObject *key; + PyObject *dict; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + stack_pointer += -2; + DISPATCH(); + } - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! } - // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); - } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = Py_None; // Failure! } + stack_pointer[-3] = attrs; stack_pointer += -2; DISPATCH(); } - TARGET(STORE_ATTR_SLOT) { + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; + stack_pointer[0] = values_or_none; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + PyObject *cond; + cond = stack_pointer[-1]; + + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyObject *value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } } - // _STORE_ATTR_SLOT - value = stack_pointer[-2]; + // _POP_JUMP_IF_TRUE + cond = b; { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - stack_pointer += -2; + stack_pointer += -1; DISPATCH(); } - TARGET(COMPARE_OP) { - frame->instr_ptr = next_instr; + TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *right; - PyObject *left; - PyObject *res; - // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ } - // _COMPARE_OP + // _POP_JUMP_IF_FALSE + cond = b; { - - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; - } + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); } - stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); } @@ -5061,7 +4698,6 @@ INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION @@ -5072,96 +4708,274 @@ DISPATCH(); } - TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; + TARGET(POP_TOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; + } + else { + prev_exc = Py_None; + } + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + PyObject *res; + res = NULL; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + PyObject **args; + args = &stack_pointer[-oparg]; + + TIER_ONE_ONLY + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = args[1]; + /* fall through */ + case 1: + exc = args[0]; + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; + } + if (true) { stack_pointer += -oparg; goto error; } + stack_pointer += -oparg; + DISPATCH(); + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + TIER_ONE_ONLY + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + GOTO_ERROR(error); + } + } + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + + TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); + } + + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + TIER_ONE_ONLY + assert(frame == tstate->current_frame); + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + } + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; + } + DISPATCH(); + } + + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif - JUMPBY(oparg * flag); - stack_pointer += -1; + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } - TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); + TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; + PyObject *retval; + // _LOAD_CONST { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } + + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); } - // _POP_JUMP_IF_TRUE - cond = b; + // _POP_FRAME + retval = value; { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; + + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } #endif - JUMPBY(oparg * flag); } - stack_pointer += -1; DISPATCH(); } - TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; - { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } - } - // _POP_JUMP_IF_FALSE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != &entry_frame); + #endif stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif DISPATCH(); } - TARGET(FOR_ITER) { + TARGET(SEND) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *iter; - PyObject *next; - // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; + PyObject *receiver; + PyObject *v; + PyObject *retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); @@ -5169,273 +4983,267 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } - STAT_INC(FOR_ITER, deferred); + STAT_INC(SEND, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _FOR_ITER + // _SEND + v = stack_pointer[-1]; { - - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + GOTO_ERROR(error); + } } - // Common case: no jump, leave it to the code generator + Py_DECREF(v); } - stack_pointer[0] = next; - stack_pointer += 1; + stack_pointer[-1] = retval; DISPATCH(); } - TARGET(FOR_ITER_LIST) { + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(SETUP_ANNOTATIONS) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - iter = stack_pointer[-1]; - { + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) goto error; } - // _ITER_JUMP_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; } - // _ITER_NEXT_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + else { + Py_DECREF(ann_dict); } - stack_pointer[0] = next; - stack_pointer += 1; DISPATCH(); } - TARGET(FOR_ITER_TUPLE) { + TARGET(SET_ADD) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(FOR_ITER_RANGE) { + TARGET(SET_FUNCTION_ATTRIBUTE) { frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; - { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); - } + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); } - // _ITER_NEXT_RANGE - { + stack_pointer[-2] = func; + stack_pointer += -1; + DISPATCH(); + } - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; - } - stack_pointer[0] = next; - stack_pointer += 1; + TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION + PyObject *v; + // _SPECIALIZE_STORE_ATTR owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - // _LOAD_ATTR_METHOD_WITH_VALUES + // _STORE_ATTR + v = stack_pointer[-2]; { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); + stack_pointer += -2; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_NO_DICT) { + TARGET(STORE_ATTR_INSTANCE_VALUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + PyObject *value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT + // _GUARD_DORV_VALUES { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + } + stack_pointer += -2; DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + TARGET(STORE_ATTR_SLOT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; - PyObject *attr; + PyObject *value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; @@ -5443,127 +5251,214 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; { - PyObject *descr = read_obj(&this_instr[6].cache); + uint16_t index = read_u16(&this_instr[4].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); Py_DECREF(owner); - attr = Py_NewRef(descr); } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); + stack_pointer += -2; DISPATCH(); } - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + TARGET(STORE_ATTR_WITH_HINT) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION + PyObject *value; owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); + stack_pointer += -2; + DISPATCH(); + } - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; + DISPATCH(); + } + + TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + PyObject *v; + v = stack_pointer[-1]; + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) goto pop_1_error; } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; DISPATCH(); } - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + stack_pointer += -4; DISPATCH(); } - TARGET(CALL) { + TARGET(STORE_SUBSCR) { frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - // _SPECIALIZE_CALL - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; + PyObject *container; + PyObject *v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); @@ -5571,316 +5466,460 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } - STAT_INC(CALL, deferred); + STAT_INC(STORE_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ } - // _CALL + // _STORE_SUBSCR + v = stack_pointer[-3]; { - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); + stack_pointer += -3; DISPATCH(); } - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *null; - PyObject *callable; - PyObject *func; - PyObject *self; - PyObject *self_or_null; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { + TARGET(STORE_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; + + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + stack_pointer += -3; + DISPATCH(); + } + + TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; + DISPATCH(); + } - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; - { - uint32_t func_version = read_u32(&this_instr[2].cache); + TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + DISPATCH(); + } - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject *value; + PyObject *res; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); } + STAT_INC(TO_BOOL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - // _SAVE_RETURN_OFFSET + // _TO_BOOL { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; } - stack_pointer += (((0) ? 1 : 0)); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(CALL_PY_EXACT_ARGS) { + TARGET(TO_BOOL_ALWAYS_TRUE) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *self_or_null; - PyObject *callable; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + uint32_t version = read_u32(&this_instr[2].cache); - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + // This one is a bit weird, because we expect *some* failures: + assert(version); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + STAT_INC(TO_BOOL, hit); + Py_DECREF(value); + res = Py_True; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); + } + + TARGET(TO_BOOL_INT) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortal(value)); + res = Py_False; } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } + else { + Py_DECREF(value); + res = Py_True; } - // _SAVE_RETURN_OFFSET - { + stack_pointer[-1] = res; + DISPATCH(); + } - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + TARGET(TO_BOOL_LIST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_NONE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_STR) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value)); + res = Py_False; } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif + else { + assert(Py_SIZE(value)); + Py_DECREF(value); + res = Py_True; } - stack_pointer += (((0) ? 1 : 0)); + stack_pointer[-1] = res; DISPATCH(); } - TARGET(BINARY_OP) { + TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + + res = PyNumber_Invert(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + PyObject *seq; + seq = stack_pointer[-1]; + + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *rhs; - PyObject *lhs; - PyObject *res; - // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; + PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } - STAT_INC(BINARY_OP, deferred); + STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); + (void)seq; + (void)counter; } - // _BINARY_OP + // _UNPACK_SEQUENCE { - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_func: FOURTH = the context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } + TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + #undef TIER_ONE diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d9716eade7acc5..b7cf9ab51ead5a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -371,8 +371,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: instructions["BINARY_OP_INPLACE_ADD_UNICODE"].family = families["BINARY_OP"] return Analysis(instructions, uops, families, pseudos) -def analyze_file(filename: str) -> Analysis: - return analyze_forest(parser.parse_file(filename)) +def analyze_files(filenames: list[str]) -> Analysis: + return analyze_forest(parser.parse_files(filenames)) def dump_analysis(analysis: Analysis) -> None: print("Uops:") diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 55a679f80df38b..1b07d756c11993 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -7,35 +7,36 @@ AstNode = InstDef | Macro | Pseudo | Family -def parse_file(filename: str) -> None: - with open(filename) as file: - src = file.read() - - psr = Parser(src, filename=prettify_filename(filename)) - - # Skip until begin marker - while tkn := psr.next(raw=True): - if tkn.text == BEGIN_MARKER: - break - else: - raise psr.make_syntax_error( - f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" - ) - start = psr.getpos() - - # Find end marker, then delete everything after it - while tkn := psr.next(raw=True): - if tkn.text == END_MARKER: - break - del psr.tokens[psr.getpos() - 1 :] - - # Parse from start - psr.setpos(start) - thing_first_token = psr.peek() +def parse_files(filenames: list[str]) -> None: result: list[AstNode] = [] - while node := psr.definition(): - result.append(node) - if not psr.eof(): - psr.backup() - raise psr.make_syntax_error(f"Extra stuff at the end of {filename}", psr.next(True)) + for filename in filenames: + with open(filename) as file: + src = file.read() + + psr = Parser(src, filename=prettify_filename(filename)) + + # Skip until begin marker + while tkn := psr.next(raw=True): + if tkn.text == BEGIN_MARKER: + break + else: + raise psr.make_syntax_error( + f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" + ) + start = psr.getpos() + + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) + thing_first_token = psr.peek() + while node := psr.definition(): + result.append(node) + if not psr.eof(): + psr.backup() + raise psr.make_syntax_error(f"Extra stuff at the end of {filename}", psr.next(True)) return result diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 58e53c0a000c96..e66cf2f9eb3950 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -1,12 +1,27 @@ +"""Generate the main interpreter switch. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import os.path import sys -from analyzer import Analysis, Instruction, Uop, Part, analyze_file, Skip, StackItem, analysis_error + +from analyzer import Analysis, Instruction, Uop, Part, analyze_files, Skip, StackItem, analysis_error from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token from stack import StackOffset +HERE = os.path.dirname(__file__) +ROOT = os.path.join(HERE, "../..") +THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/") + +DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) +DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) + def write_header(filename: str, outfile: TextIO) -> None: - outfile.write(f"""// This file is generated by {__file__} + outfile.write(f"""// This file is generated by {THIS} // from: // {filename} // Do not edit! @@ -259,8 +274,8 @@ def uses_this(inst: Instruction) -> bool: return True return False -def generate_tier1(filename: str, analysis: Analysis, outfile: TextIO) -> None: - write_header(filename, outfile) +def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO) -> None: + write_header(filenames, outfile) out = CWriter(outfile, 2) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): @@ -297,16 +312,34 @@ def generate_tier1(filename: str, analysis: Analysis, outfile: TextIO) -> None: outfile.write(FOOTER) +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-v", + "--verbose", + help="Print list of non-viable uops and exit", + action="store_true", +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + if __name__ == "__main__": - import sys - if len(sys.argv) not in (2, 3): - print(__file__, "INFILE", "OUTFILE") - else: - filename = sys.argv[1] - data = analyze_file(filename) - if len(sys.argv) == 3: - outfilename = sys.argv[2] - with open(outfilename, "w") as outfile: - generate_tier1(filename, data, outfile) - else: - generate_tier1(filename, data, sys.stdout) + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier1(args.input, data, outfile) From 570d07a906f484ed648ae8317db4ff59dcf4053c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 05:50:32 +0000 Subject: [PATCH 11/40] Add support for line directives --- Tools/cases_generator/cwriter.py | 5 ++++- Tools/cases_generator/lexer.py | 5 +++-- Tools/cases_generator/tier1_generator.py | 13 +++---------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 0fb9cfe4a3ca40..ddb166e6e93e0b 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -5,16 +5,19 @@ class CWriter: 'A writer that understands tokens and how to format C code' - def __init__(self, out: TextIO, indent: int = 0): + def __init__(self, out: TextIO, indent: int, line_directives: bool): self.out = out self.initial_indent = self.indent = indent self.line = -1 self.column = 0 + self.line_directives = line_directives def set_position(self, tkn: Token): if self.line < tkn.line: self.column = 0 self.out.write("\n") + if self.line_directives: + self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') if self.column == 0: self.out.write(" " * self.indent) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 856bd678ba8908..068ac909656ed8 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -234,6 +234,7 @@ def make_syntax_error( @dataclass(slots=True) class Token: + filename: str kind: str text: str begin: tuple[int, int] @@ -261,7 +262,7 @@ def width(self) -> int: def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) - return Token(self.kind, txt, self.begin, self.end) + return Token(self.filename, self.kind, txt, self.begin, self.end) def __repr__(self) -> str: b0, b1 = self.begin @@ -323,7 +324,7 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T else: begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start - linestart + len(text))) + yield Token(filename, kind, text, begin, (line, start - linestart + len(text))) def to_text(tkns: list[Token], dedent: int = 0) -> str: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index e66cf2f9eb3950..3259bcce95b7c7 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -274,9 +274,9 @@ def uses_this(inst: Instruction) -> bool: return True return False -def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO) -> None: +def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO, lines: bool) -> None: write_header(filenames, outfile) - out = CWriter(outfile, 2) + out = CWriter(outfile, 2, lines) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): needs_this = uses_this(inst) @@ -317,13 +317,6 @@ def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO) -> None: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) -arg_parser.add_argument( - "-v", - "--verbose", - help="Print list of non-viable uops and exit", - action="store_true", -) - arg_parser.add_argument( "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) @@ -342,4 +335,4 @@ def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO) -> None: args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) with open(args.output, "w") as outfile: - generate_tier1(args.input, data, outfile) + generate_tier1(args.input, data, outfile, args.emit_line_directives) From bb1801023be45282d7aaa3a92bfec5af9bb73751 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 06:26:40 +0000 Subject: [PATCH 12/40] Tier 2 code gen. Work in progress --- Python/executor_cases.c.h | 5192 +++++++++++----------- Tools/cases_generator/tier2_generator.py | 321 ++ 2 files changed, 2950 insertions(+), 2563 deletions(-) create mode 100644 Tools/cases_generator/tier2_generator.py diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 471abae6045210..860b603c2ba6c2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1,6 +1,6 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier2_generator.py // from: -// Python/bytecodes.c +// ['Python/bytecodes.c'] // Do not edit! #ifdef TIER_ONE @@ -8,1481 +8,1753 @@ #endif #define TIER_TWO 2 - case NOP: { - break; - } - - case RESUME_CHECK: { -#if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - break; - } + case _BEFORE_ASYNC_WITH: { + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; - case LOAD_FAST_CHECK: { - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error_tier_two; - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case LOAD_FAST: { - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + case _BEFORE_WITH: { + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the + * value returned from calling its __enter__ + */ + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case LOAD_FAST_AND_CLEAR: { - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + case _BINARY_SLICE: { + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res = NULL; + } + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); + } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; break; } - case LOAD_CONST: { - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; - break; - } + case _BINARY_SUBSCR_DICT: { + PyObject *sub; + PyObject *dict; + PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; - case STORE_FAST: { - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - STACK_SHRINK(1); + DEOPT_IF(!PyDict_CheckExact(dict)); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case POP_TOP: { - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); - STACK_SHRINK(1); - break; - } + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ - case PUSH_NULL: { + case _BINARY_SUBSCR_LIST_INT: { + PyObject *sub; + PyObject *list; PyObject *res; - res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list)); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case END_SEND: { - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + case _BINARY_SUBSCR_STR_INT: { + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyUnicode_CheckExact(str)); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case UNARY_NEGATIVE: { - PyObject *value; + case _BINARY_SUBSCR_TUPLE_INT: { + PyObject *sub; + PyObject *tuple; PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; - stack_pointer[-1] = res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyTuple_CheckExact(tuple)); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case UNARY_NOT: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; + case _BUILD_CONST_KEY_MAP: { + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + GOTO_ERROR(error); // Pop the keys and values. + } + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); + } + Py_DECREF(keys); + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; break; } - case _TO_BOOL: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error_tier_two; - res = err ? Py_True : Py_False; - stack_pointer[-1] = res; + case _BUILD_LIST: { + PyObject **values; + PyObject *list; + values = &stack_pointer[-oparg]; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; break; } - case TO_BOOL_BOOL: { - PyObject *value; - value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); + case _BUILD_MAP: { + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); + } + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; break; } - case TO_BOOL_INT: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + case _BUILD_SET: { + PyObject **values; + PyObject *set; + values = &stack_pointer[-oparg]; + set = PySet_New(NULL); + if (set == NULL) + GOTO_ERROR(error); + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); } - else { - Py_DECREF(value); - res = Py_True; + if (err != 0) { + Py_DECREF(set); + if (true) { stack_pointer += -oparg; goto error; } } - stack_pointer[-1] = res; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; break; } - case TO_BOOL_LIST: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); - stack_pointer[-1] = res; - break; - } + case _BUILD_SLICE: { + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - case TO_BOOL_NONE: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; - stack_pointer[-1] = res; + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); break; } - case TO_BOOL_STR: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + case _BUILD_STRING: { + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); } - stack_pointer[-1] = res; + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; break; } - case TO_BOOL_ALWAYS_TRUE: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = (uint32_t)operand; - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; + case _BUILD_TUPLE: { + PyObject **values; + PyObject *tup; + values = &stack_pointer[-oparg]; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; break; } - case UNARY_INVERT: { - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; - stack_pointer[-1] = res; - break; - } + case _CACHE: { - case _GUARD_BOTH_INT: { - PyObject *right; - PyObject *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); - DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); - break; + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); } - case _BINARY_OP_MULTIPLY_INT: { - PyObject *right; - PyObject *left; + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + + case _CALL_BUILTIN_CLASS: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable)); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case _BINARY_OP_ADD_INT: { - PyObject *right; - PyObject *left; + case _CALL_BUILTIN_FAST: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case _BINARY_OP_SUBTRACT_INT: { - PyObject *right; - PyObject *left; + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case _GUARD_BOTH_FLOAT: { - PyObject *right; - PyObject *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); - DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); + case _CALL_BUILTIN_O: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right; - PyObject *left; + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _CALL_INTRINSIC_1: { + PyObject *value; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; break; } - case _BINARY_OP_ADD_FLOAT: { - PyObject *right; - PyObject *left; + case _CALL_INTRINSIC_2: { + PyObject *value1; + PyObject *value2; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right; - PyObject *left; + case _CALL_ISINSTANCE: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; - case _GUARD_BOTH_UNICODE: { - PyObject *right; - PyObject *left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); - DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + GOTO_ERROR(error); + } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case _BINARY_OP_ADD_UNICODE: { - PyObject *right; - PyObject *left; + /* _CALL_KW is not a viable micro-op for tier 2 */ + + case _CALL_LEN: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case _BINARY_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *res; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case BINARY_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + DEOPT_IF(total_args != 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + GOTO_ERROR(error); } - Py_DECREF(container); - if (res == NULL) goto pop_3_error_tier_two; - STACK_SHRINK(2); - stack_pointer[-1] = res; + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case STORE_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; - } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + case _CALL_LIST_APPEND: { + PyObject **args; + PyObject *self; + PyObject *callable; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append); + assert(self != NULL); + DEOPT_IF(!PyList_Check(self)); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error_tier_two; - STACK_SHRINK(4); - break; + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + DISPATCH(); } - case BINARY_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; + case _CALL_METHOD_DESCRIPTOR_FAST: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case BINARY_SUBSCR_STR_INT: { - PyObject *sub; - PyObject *str; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type)); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub; - PyObject *tuple; + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + DEOPT_IF(meth->ml_flags != METH_NOARGS); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case BINARY_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; + case _CALL_METHOD_DESCRIPTOR_O: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case LIST_APPEND: { - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ + + case _CALL_STR_1: { + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case SET_ADD: { - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + case _CALL_TUPLE_1: { + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); break; } - case _STORE_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *v; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - v = stack_pointer[-3]; - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error_tier_two; - STACK_SHRINK(3); + case _CALL_TYPE_1: { + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; break; } - case STORE_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + case _CHECK_EG_MATCH: { + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; + } + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - STACK_SHRINK(3); + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } + stack_pointer[-2] = rest; + stack_pointer[-1] = match; break; } - case STORE_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error_tier_two; - STACK_SHRINK(3); + case _CHECK_EXC_MATCH: { + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; + } + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; break; } - case DELETE_SUBSCR: { - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error_tier_two; - STACK_SHRINK(2); - break; - } + /* _CLEANUP_THROW is not a viable micro-op for tier 2 */ - case CALL_INTRINSIC_1: { - PyObject *value; + case _COMPARE_OP_FLOAT: { + PyObject *right; + PyObject *left; PyObject *res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; - stack_pointer[-1] = res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case CALL_INTRINSIC_2: { - PyObject *value1; - PyObject *value2; + case _COMPARE_OP_INT: { + PyObject *right; + PyObject *left; PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } - - case _POP_FRAME: { - PyObject *retval; - retval = stack_pointer[-1]; - STACK_SHRINK(1); - #if TIER_ONE - assert(frame != &entry_frame); - #endif - STORE_SP(); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case GET_AITER: { - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error_tier_two; - } - - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error_tier_two; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error_tier_two; - } - stack_pointer[-1] = iter; + case _COMPARE_OP_STR: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case GET_ANEXT: { - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - GOTO_ERROR(error); - } + case _CONTAINS_OP: { + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + break; + } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); + case _CONVERT_VALUE: { + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } - } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + convertion_func_ptr conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = CONVERSION_FUNCTIONS[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; break; } - case GET_AWAITABLE: { - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } + case _COPY: { + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; - Py_DECREF(iterable); + assert(oparg > 0); + top = Py_NewRef(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + break; + } - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } + case _COPY_FREE_VARS: { + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); } - - if (iter == NULL) goto pop_1_error_tier_two; - stack_pointer[-1] = iter; break; } - case POP_EXCEPT: { - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + case _DELETE_ATTR: { + PyObject *owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + stack_pointer += -1; break; } - case LOAD_ASSERTION_ERROR: { - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + case _DELETE_DEREF: { + + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); break; } - case LOAD_BUILD_CLASS: { - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error_tier_two; - } - STACK_GROW(1); - stack_pointer[-1] = bc; + case _DELETE_FAST: { + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); break; } - case STORE_NAME: { - PyObject *v; - v = stack_pointer[-1]; + case _DELETE_GLOBAL: { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error_tier_two; + err = PyDict_DelItem(GLOBALS(), name); + // Can't use ERROR_IF here. + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + GOTO_ERROR(error); } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); break; } - case DELETE_NAME: { + case _DELETE_NAME: { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); + "no locals when deleting %R", name); GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } break; } - case _UNPACK_SEQUENCE: { - PyObject *seq; - seq = stack_pointer[-1]; - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); - STACK_GROW(oparg); + case _DELETE_SUBSCR: { + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_2_error; + stack_pointer += -2; break; } - case UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + case _DICT_MERGE: { + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; break; } - case UNPACK_SEQUENCE_TUPLE: { - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + case _DICT_UPDATE: { + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + if (true) goto pop_1_error; } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + Py_DECREF(update); + stack_pointer += -1; break; } - case UNPACK_SEQUENCE_LIST: { - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = stack_pointer - 1; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + /* _END_ASYNC_FOR is not a viable micro-op for tier 2 */ + + case _END_SEND: { + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; break; } - case UNPACK_EX: { - PyObject *seq; - seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + case _ENTER_EXECUTOR: { + + TIER_ONE_ONLY + CHECK_EVAL_BREAKER(); + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); + frame->instr_ptr = next_instr; + Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } + frame = executor->execute(executor, frame, stack_pointer); + if (frame == NULL) { + frame = tstate->current_frame; + goto resume_with_error; + } + goto enter_tier_one; + } + + case _EXIT_INIT_CHECK: { + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; + + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + GOTO_ERROR(error); + } + stack_pointer += -1; break; } - case _STORE_ATTR: { - PyObject *owner; - PyObject *v; - owner = stack_pointer[-1]; - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error_tier_two; - STACK_SHRINK(2); + case _EXTENDED_ARG: { + + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + case _FORMAT_SIMPLE: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result + * of format(value) is value itself. */ + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + } + else { + res = value; + } + stack_pointer[-1] = res; break; } - case DELETE_ATTR: { - PyObject *owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); + case _FORMAT_WITH_SPEC: { + PyObject *fmt_spec; + PyObject *value; + PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case STORE_GLOBAL: { - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error_tier_two; - STACK_SHRINK(1); - break; - } + /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ - case DELETE_GLOBAL: { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - GOTO_ERROR(error); + case _GET_AITER: { + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; } - break; - } + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + if (true) goto pop_1_error; + } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; - case LOAD_LOCALS: { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error_tier_two; + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + if (true) goto pop_1_error; } - Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[-1] = iter; break; } - case LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + case _GET_ANEXT: { + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { GOTO_ERROR(error); } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { GOTO_ERROR(error); } } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - break; - } - - case LOAD_NAME: { - PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error_tier_two; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); GOTO_ERROR(error); } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + Py_DECREF(next_iter); + GOTO_ERROR(error); + } else { + Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = awaitable; + stack_pointer += 1; break; } - case _LOAD_GLOBAL: { - PyObject *res; - PyObject *null = NULL; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error_tier_two; - } - Py_INCREF(res); + case _GET_AWAITABLE: { + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error_tier_two; - } + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ } } - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - break; - } - - case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)operand; - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); - assert(DK_IS_UNICODE(dict->ma_keys)); - break; - } - - case _GUARD_BUILTINS_VERSION: { - uint16_t version = (uint16_t)operand; - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); - DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); - assert(DK_IS_UNICODE(dict->ma_keys)); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; break; } - case _LOAD_GLOBAL_MODULE: { - PyObject *res; - PyObject *null = NULL; - uint16_t index = (uint16_t)operand; - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - break; - } + case _GET_ITER: { + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; - case _LOAD_GLOBAL_BUILTINS: { - PyObject *res; - PyObject *null = NULL; - uint16_t index = (uint16_t)operand; - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; break; } - case DELETE_FAST: { - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error_tier_two; - SETLOCAL(oparg, NULL); + case _GET_LEN: { + PyObject *obj; + PyObject *len_o; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + stack_pointer[0] = len_o; + stack_pointer += 1; break; } - case MAKE_CELL: { - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - GOTO_ERROR(error); - } - SETLOCAL(oparg, cell); - break; - } + case _GET_YIELD_FROM_ITER: { + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; - case DELETE_DEREF: { - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a + regular generator. */ + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + GOTO_ERROR(error); + } + iter = iterable; } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - break; - } - - case LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + else if (PyGen_CheckExact(iterable)) { + iter = iterable; } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + else { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + if (iter == NULL) { GOTO_ERROR(error); } - Py_INCREF(value); + Py_DECREF(iterable); } - Py_DECREF(class_dict); - stack_pointer[-1] = value; + stack_pointer[-1] = iter; break; } - case LOAD_DEREF: { - PyObject *value; - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error_tier_two; - } - Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + case _IMPORT_FROM: { + PyObject *from; + PyObject *res; + from = stack_pointer[-1]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case STORE_DEREF: { - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - STACK_SHRINK(1); + case _IMPORT_NAME: { + PyObject *fromlist; + PyObject *level; + PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case COPY_FREE_VARS: { - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); - } + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_END_FOR is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_END_SEND is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + + case _INTERPRETER_EXIT: { + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; + } + + case _IS_OP: { + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; break; } - case BUILD_STRING: { - PyObject **pieces; - PyObject *str; - pieces = stack_pointer - oparg; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); - } - if (str == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + case _JUMP_BACKWARD_NO_INTERRUPT: { + + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMPBY(-oparg); break; } - case BUILD_TUPLE: { - PyObject **values; - PyObject *tup; - values = stack_pointer - oparg; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + case _JUMP_FORWARD: { + JUMPBY(oparg); break; } - case BUILD_LIST: { - PyObject **values; + case _LIST_APPEND: { + PyObject *v; PyObject *list; - values = stack_pointer - oparg; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error_tier_two; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + stack_pointer += -1; break; } - case LIST_EXTEND: { + case _LIST_EXTEND: { PyObject *iterable; PyObject *list; iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - if (true) goto pop_1_error_tier_two; + if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; break; } - case SET_UPDATE: { - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error_tier_two; - STACK_SHRINK(1); + case _LOAD_ASSERTION_ERROR: { + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + stack_pointer[0] = value; + stack_pointer += 1; break; } - case BUILD_SET: { - PyObject **values; - PyObject *set; - values = stack_pointer - oparg; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error_tier_two; } + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + + case _LOAD_BUILD_CLASS: { + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[0] = bc; + stack_pointer += 1; break; } - case BUILD_MAP: { - PyObject **values; - PyObject *map; - values = stack_pointer - oparg*2; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + case _LOAD_CONST: { + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_DEREF: { + PyObject *value; + + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error_tier_two; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; break; } - case SETUP_ANNOTATIONS: { - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error_tier_two; + case _LOAD_FAST: { + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_AND_CLEAR: { + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_CHECK: { + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + + case _LOAD_FAST_LOAD_FAST: { + PyObject *value1; + PyObject *value2; + + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + break; + } + + case _LOAD_FROM_DICT_OR_DEREF: { + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + GOTO_ERROR(error); } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error_tier_two; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error_tier_two; + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + Py_INCREF(value); } - else { - Py_DECREF(ann_dict); + Py_DECREF(class_dict); + stack_pointer[-1] = value; + break; + } + + case _LOAD_FROM_DICT_OR_GLOBALS: { + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } } + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; break; } - case BUILD_CONST_KEY_MAP: { - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + case _LOAD_LOCALS: { + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + "no locals found"); + if (true) goto error; } - Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error_tier_two; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + Py_INCREF(locals); + stack_pointer[0] = locals; + stack_pointer += 1; break; } - case DICT_UPDATE: { - PyObject *update; - PyObject *dict; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + case _LOAD_NAME: { + PyObject *v; + + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } } - Py_DECREF(update); - if (true) goto pop_1_error_tier_two; - } - Py_DECREF(update); - STACK_SHRINK(1); - break; - } - - case DICT_MERGE: { - PyObject *update; - PyObject *dict; - PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error_tier_two; } - Py_DECREF(update); - STACK_SHRINK(1); - break; - } - - case MAP_ADD: { - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; - STACK_SHRINK(2); + stack_pointer[0] = v; + stack_pointer += 1; break; } - case LOAD_SUPER_ATTR_ATTR: { + case _LOAD_SUPER_ATTR_ATTR: { PyObject *self; PyObject *class; PyObject *global_super; @@ -1490,22 +1762,23 @@ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; + assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (attr == NULL) goto pop_3_error_tier_two; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + if (attr == NULL) goto pop_3_error; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); break; } - case LOAD_SUPER_ATTR_METHOD: { + case _LOAD_SUPER_ATTR_METHOD: { PyObject *self; PyObject *class; PyObject *global_super; @@ -1515,19 +1788,19 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); if (attr == NULL) { Py_DECREF(self); - if (true) goto pop_3_error_tier_two; + if (true) goto pop_3_error; } if (method_found) { self_or_null = self; // transfer ownership @@ -1535,756 +1808,635 @@ Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; break; } - case _LOAD_ATTR: { - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - NULL | meth | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; - self_or_null = NULL; - } + case _MAKE_CELL: { + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + GOTO_ERROR(error); + } + SETLOCAL(oparg, cell); + break; + } + + case _MAKE_FUNCTION: { + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + Py_DECREF(codeobj); + if (func_obj == NULL) { + GOTO_ERROR(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = (PyObject *)func_obj; + stack_pointer[-1] = func; + break; + } + + case _MAP_ADD: { + PyObject *value; + PyObject *key; + PyObject *dict; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + stack_pointer += -2; + break; + } + + case _MATCH_CLASS: { + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! } else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = Py_None; // Failure! } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-3] = attrs; + stack_pointer += -2; break; } - case _GUARD_TYPE_VERSION: { - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); + case _MATCH_KEYS: { + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; + stack_pointer[0] = values_or_none; + stack_pointer += 1; break; } - case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner; - owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); + case _MATCH_MAPPING: { + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case _LOAD_ATTR_INSTANCE_VALUE: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + case _MATCH_SEQUENCE: { + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case _CHECK_ATTR_MODULE: { - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, _CHECK_ATTR_MODULE); + case _NOP: { break; } - case _LOAD_ATTR_MODULE: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, _LOAD_ATTR_MODULE); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + case _POP_EXCEPT: { + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); + stack_pointer += -1; break; } - case _CHECK_ATTR_WITH_HINT: { - PyObject *owner; - owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), _CHECK_ATTR_WITH_HINT); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, _CHECK_ATTR_WITH_HINT); - assert(PyDict_CheckExact((PyObject *)dict)); + case _POP_TOP: { + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); + stack_pointer += -1; break; } - case _LOAD_ATTR_WITH_HINT: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)operand; - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); - attr = ep->me_value; + case _PUSH_EXC_INFO: { + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; } else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, _LOAD_ATTR_WITH_HINT); - attr = ep->me_value; + prev_exc = Py_None; } - DEOPT_IF(attr == NULL, _LOAD_ATTR_WITH_HINT); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; break; } - case _LOAD_ATTR_SLOT: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)operand; - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + case _PUSH_NULL: { + PyObject *res; + res = NULL; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case _CHECK_ATTR_CLASS: { - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)operand; - DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS); - break; + /* _RAISE_VARARGS is not a viable micro-op for tier 2 */ + + /* _RERAISE is not a viable micro-op for tier 2 */ + + case _RESERVED: { + + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); } - case _LOAD_ATTR_CLASS: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* _RESUME is not a viable micro-op for tier 2 */ + + case _RESUME_CHECK: { + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version); break; } - case _GUARD_DORV_VALUES: { - PyObject *owner; - owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); - break; + case _RETURN_GENERATOR: { + + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; } - case _STORE_ATTR_INSTANCE_VALUE: { - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint16_t index = (uint16_t)operand; - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); + /* _SEND_GEN is not a viable micro-op for tier 2 */ + + case _SETUP_ANNOTATIONS: { + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) goto error; + } + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; } else { - Py_DECREF(old_value); + Py_DECREF(ann_dict); } - Py_DECREF(owner); - STACK_SHRINK(2); break; } - case _STORE_ATTR_SLOT: { - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint16_t index = (uint16_t)operand; - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); - STACK_SHRINK(2); - break; - } + case _SET_ADD: { + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + break; + } + + case _SET_FUNCTION_ATTRIBUTE: { + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; - case _COMPARE_OP: { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error_tier_two; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error_tier_two; - res = res_bool ? Py_True : Py_False; + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = func; + stack_pointer += -1; break; } - case COMPARE_OP_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + case _SET_UPDATE: { + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; break; } - case COMPARE_OP_INT: { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + + case _STORE_DEREF: { + PyObject *v; + v = stack_pointer[-1]; + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + stack_pointer += -1; break; } - case COMPARE_OP_STR: { - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + case _STORE_FAST: { + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; break; } - case IS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + case _STORE_FAST_LOAD_FAST: { + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; break; } - case CONTAINS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; - b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + case _STORE_FAST_STORE_FAST: { + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; break; } - case CHECK_EG_MATCH: { - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error_tier_two; - } - - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error_tier_two; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error_tier_two; + case _STORE_GLOBAL: { + PyObject *v; + v = stack_pointer[-1]; - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); - } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; break; } - case CHECK_EXC_MATCH: { - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error_tier_two; + case _STORE_NAME: { + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) goto pop_1_error; } - - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; break; } - case _IS_NONE: { - PyObject *value; - PyObject *b; - value = stack_pointer[-1]; - if (Py_IsNone(value)) { - b = Py_True; + case _STORE_SLICE: { + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; } else { - b = Py_False; - Py_DECREF(value); + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); } - stack_pointer[-1] = b; + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + stack_pointer += -4; break; } - case GET_LEN: { - PyObject *obj; - PyObject *len_o; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error_tier_two; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = len_o; - break; - } + case _STORE_SUBSCR_DICT: { + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; - case MATCH_CLASS: { - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; - attrs = Py_None; // Failure! - } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + DEOPT_IF(!PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + stack_pointer += -3; break; } - case MATCH_MAPPING: { - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + case _STORE_SUBSCR_LIST_INT: { + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list)); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; break; } - case MATCH_SEQUENCE: { - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; - break; - } + case _SWAP: { + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; - case MATCH_KEYS: { - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; break; } - case GET_ITER: { - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error_tier_two; - stack_pointer[-1] = iter; + /* _TO_BOOL_ALWAYS_TRUE is not a viable micro-op for tier 2 */ + + case _TO_BOOL_BOOL: { + PyObject *value; + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(value)); + STAT_INC(TO_BOOL, hit); break; } - case GET_YIELD_FROM_ITER: { - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - GOTO_ERROR(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; + case _TO_BOOL_INT: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value)); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortal(value)); + res = Py_False; } else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(iterable); + Py_DECREF(value); + res = Py_True; } - stack_pointer[-1] = iter; + stack_pointer[-1] = res; break; } - case _ITER_CHECK_LIST: { - PyObject *iter; - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); + case _TO_BOOL_LIST: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value)); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; break; } - case _GUARD_NOT_EXHAUSTED_LIST: { - PyObject *iter; - iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_LIST); - DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_LIST); + case _TO_BOOL_NONE: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value)); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; break; } - case _ITER_NEXT_LIST: { - PyObject *iter; - PyObject *next; - iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + case _TO_BOOL_STR: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value)); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value)); + res = Py_False; + } + else { + assert(Py_SIZE(value)); + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; break; } - case _ITER_CHECK_TUPLE: { - PyObject *iter; - iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); - break; - } + case _UNARY_INVERT: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; - case _GUARD_NOT_EXHAUSTED_TUPLE: { - PyObject *iter; - iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_TUPLE); - DEOPT_IF(it->it_index >= PyTuple_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_TUPLE); + res = PyNumber_Invert(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; break; } - case _ITER_NEXT_TUPLE: { - PyObject *iter; - PyObject *next; - iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - STACK_GROW(1); - stack_pointer[-1] = next; + case _UNARY_NEGATIVE: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; break; } - case _ITER_CHECK_RANGE: { - PyObject *iter; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); + case _UNARY_NOT: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; + stack_pointer[-1] = res; break; } - case _GUARD_NOT_EXHAUSTED_RANGE: { - PyObject *iter; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - DEOPT_IF(r->len <= 0, _GUARD_NOT_EXHAUSTED_RANGE); + case _UNPACK_EX: { + PyObject *seq; + seq = stack_pointer[-1]; + + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; } - case _ITER_NEXT_RANGE: { - PyObject *iter; - PyObject *next; - iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = next; + case _UNPACK_SEQUENCE_LIST: { + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(seq)); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; break; } - case BEFORE_ASYNC_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; + case _UNPACK_SEQUENCE_TUPLE: { + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + Py_DECREF(seq); + stack_pointer += -1 + oparg; break; } - case BEFORE_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; - } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + case _UNPACK_SEQUENCE_TWO_TUPLE: { + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + Py_DECREF(seq); + stack_pointer += -1 + oparg; break; } - case WITH_EXCEPT_START: { + case _WITH_EXCEPT_START: { PyObject *val; PyObject *lasti; PyObject *exit_func; @@ -2292,6 +2444,7 @@ val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_func = stack_pointer[-4]; + /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -2301,7 +2454,6 @@ Then we push the __exit__ return value. */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -2315,144 +2467,246 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error_tier_two; - STACK_GROW(1); - stack_pointer[-1] = res; + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; break; } - case PUSH_EXC_INFO: { - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; - } - else { - prev_exc = Py_None; - } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + case _YIELD_VALUE: { + PyObject *retval; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + case _BINARY_OP: { + PyObject *rhs; + PyObject *lhs; + PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { - PyObject *owner; - owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); + case _BINARY_OP_ADD_FLOAT: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case _GUARD_KEYS_VERSION: { - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)operand; - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); + case _BINARY_OP_ADD_INT: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case _LOAD_ATTR_METHOD_WITH_VALUES: { - PyObject *owner; - PyObject *attr; - PyObject *self; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + case _BINARY_OP_ADD_UNICODE: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case _LOAD_ATTR_METHOD_NO_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + case _BINARY_OP_INPLACE_ADD_UNICODE: { + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + stack_pointer += -2; break; } - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - PyObject *owner; - PyObject *attr; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - stack_pointer[-1] = attr; + case _BINARY_OP_MULTIPLY_FLOAT: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + stack_pointer[-2] = res; + stack_pointer += -1; break; } - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - PyObject *owner; - PyObject *attr; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - stack_pointer[-1] = attr; + case _BINARY_OP_MULTIPLY_INT: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_OP_SUBTRACT_INT: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _BINARY_SUBSCR: { + PyObject *sub; + PyObject *container; + PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; break; } + /* _CALL is not a viable micro-op for tier 2 */ + + /* _CHECK_ATTR_CLASS is not a viable micro-op for tier 2 */ + case _CHECK_ATTR_METHOD_LAZY_DICT: { PyObject *owner; owner = stack_pointer[-1]; + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, _CHECK_ATTR_METHOD_LAZY_DICT); + DEOPT_IF(dict != NULL); break; } - case _LOAD_ATTR_METHOD_LAZY_DICT: { + /* _CHECK_ATTR_MODULE is not a viable micro-op for tier 2 */ + + case _CHECK_ATTR_WITH_HINT: { PyObject *owner; - PyObject *attr; - PyObject *self; owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)operand; - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL); + assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -2461,807 +2715,619 @@ PyObject *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + + DEOPT_IF(null != NULL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); break; } - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *callable; - PyObject *func; - PyObject *self; - callable = stack_pointer[-2 - oparg]; - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = self; + /* _CHECK_FUNCTION_EXACT_ARGS is not a viable micro-op for tier 2 */ + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + PyObject *owner; + owner = stack_pointer[-1]; + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); break; } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); + + DEOPT_IF(tstate->interp->eval_frame); break; } - case _CHECK_FUNCTION_EXACT_ARGS: { - PyObject *self_or_null; + case _CHECK_STACK_SPACE: { PyObject *callable; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)operand; - DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); + PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); + DEOPT_IF(tstate->py_recursion_remaining <= 1); break; } - case _CHECK_STACK_SPACE: { - PyObject *callable; - callable = stack_pointer[-2 - oparg]; - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); - DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); + case _CHECK_VALIDITY: { + + TIER_TWO_ONLY + DEOPT_IF(!current_executor->base.vm_data.valid); break; } - case _INIT_CALL_PY_EXACT_ARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - _PyInterpreterFrame *new_frame; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + case _COMPARE_OP: { + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = (PyObject *)new_frame; + stack_pointer[-2] = res; + stack_pointer += -1; + break; + } + + case _EXIT_TRACE: { + + TIER_TWO_ONLY + GOTO_TIER_ONE(); + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 */ + + case _GUARD_BOTH_FLOAT: { + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); break; } - case _PUSH_FRAME: { - _PyInterpreterFrame *new_frame; - new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; - STACK_SHRINK(1); - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - STORE_SP(); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); -#if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } -#endif + case _GUARD_BOTH_INT: { + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); break; } - case CALL_TYPE_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + case _GUARD_BOTH_UNICODE: { + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); break; } - case CALL_STR_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + /* _GUARD_BUILTINS_VERSION is not a viable micro-op for tier 2 */ + + case _GUARD_DORV_VALUES: { + PyObject *owner; + owner = stack_pointer[-1]; + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); break; } - case CALL_TUPLE_1: { - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + PyObject *owner; + owner = stack_pointer[-1]; + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); break; } - case EXIT_INIT_CHECK: { - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); - } - STACK_SHRINK(1); + /* _GUARD_GLOBALS_VERSION is not a viable micro-op for tier 2 */ + + case _GUARD_IS_FALSE_POP: { + PyObject *flag; + flag = stack_pointer[-1]; + + DEOPT_IF(Py_IsTrue(flag)); + assert(Py_IsFalse(flag)); + stack_pointer += -1; break; } - case CALL_BUILTIN_CLASS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + case _GUARD_IS_NONE_POP: { + PyObject *val; + val = stack_pointer[-1]; + DEOPT_IF(!Py_IsNone(val)); + stack_pointer += -1; break; } - case CALL_BUILTIN_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + case _GUARD_IS_NOT_NONE_POP: { + PyObject *val; + val = stack_pointer[-1]; + DEOPT_IF(Py_IsNone(val)); + Py_DECREF(val); + stack_pointer += -1; + break; + } - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + case _GUARD_IS_TRUE_POP: { + PyObject *flag; + flag = stack_pointer[-1]; + DEOPT_IF(Py_IsFalse(flag)); + assert(Py_IsTrue(flag)); + stack_pointer += -1; break; } - case CALL_BUILTIN_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* _GUARD_KEYS_VERSION is not a viable micro-op for tier 2 */ - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + case _GUARD_NOT_EXHAUSTED_LIST: { + PyObject *iter; + iter = stack_pointer[-1]; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + DEOPT_IF(seq == NULL); + DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq)); break; } - case CALL_BUILTIN_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + case _GUARD_NOT_EXHAUSTED_RANGE: { + PyObject *iter; + iter = stack_pointer[-1]; - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + DEOPT_IF(r->len <= 0); break; } - case CALL_LEN: { - PyObject **args; - PyObject *self_or_null; + case _GUARD_NOT_EXHAUSTED_TUPLE: { + PyObject *iter; + iter = stack_pointer[-1]; + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + DEOPT_IF(seq == NULL); + DEOPT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); + break; + } + + /* _GUARD_TYPE_VERSION is not a viable micro-op for tier 2 */ + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; + PyObject *func; + PyObject *self; callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); - } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; break; } - case CALL_ISINSTANCE: { + case _INIT_CALL_PY_EXACT_ARGS: { PyObject **args; PyObject *self_or_null; PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; + _PyInterpreterFrame *new_frame; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - int total_args = oparg; + + int argcount = oparg; if (self_or_null != NULL) { args--; - total_args++; + argcount++; } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + stack_pointer[-2 - oparg] = new_frame; + stack_pointer += -1 - oparg; + break; + } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + case _INSERT: { + PyObject *top; + top = stack_pointer[-1]; + + // Inserts TOS at position specified by oparg; + memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + stack_pointer[-1 - oparg] = top; break; } - case CALL_METHOD_DESCRIPTOR_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + case _IS_NONE: { + PyObject *value; + PyObject *b; + value = stack_pointer[-1]; + if (Py_IsNone(value)) { + b = Py_True; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + else { + b = Py_False; + Py_DECREF(value); } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + stack_pointer[-1] = b; break; } - case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + case _ITER_CHECK_LIST: { + PyObject *iter; + iter = stack_pointer[-1]; - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type); break; } - case CALL_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + case _ITER_CHECK_RANGE: { + PyObject *iter; + iter = stack_pointer[-1]; + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type); + break; + } + + case _ITER_CHECK_TUPLE: { + PyObject *iter; + iter = stack_pointer[-1]; + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type); + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + + case _ITER_NEXT_LIST: { + PyObject *iter; + PyObject *next; + iter = stack_pointer[-1]; + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + case _ITER_NEXT_RANGE: { + PyObject *iter; + PyObject *next; + iter = stack_pointer[-1]; + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + case _ITER_NEXT_TUPLE: { + PyObject *iter; + PyObject *next; + iter = stack_pointer[-1]; + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + stack_pointer[0] = next; + stack_pointer += 1; + break; + } + + case _JUMP_TO_TOP: { + + next_uop = current_executor->trace; + CHECK_EVAL_BREAKER(); break; } - case CALL_METHOD_DESCRIPTOR_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = stack_pointer - oparg; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + case _LOAD_ATTR: { + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } } - Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; - CHECK_EVAL_BREAKER(); + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); break; } - case MAKE_FUNCTION: { - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; + /* _LOAD_ATTR_CLASS is not a viable micro-op for tier 2 */ - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); + /* _LOAD_ATTR_INSTANCE_VALUE is not a viable micro-op for tier 2 */ - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } + /* _LOAD_ATTR_METHOD_LAZY_DICT is not a viable micro-op for tier 2 */ - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; - break; - } + /* _LOAD_ATTR_METHOD_NO_DICT is not a viable micro-op for tier 2 */ - case SET_FUNCTION_ATTRIBUTE: { - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); - } - STACK_SHRINK(1); - stack_pointer[-1] = func; - break; - } + /* _LOAD_ATTR_METHOD_WITH_VALUES is not a viable micro-op for tier 2 */ - case BUILD_SLICE: { - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error_tier_two; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; - break; - } + /* _LOAD_ATTR_MODULE is not a viable micro-op for tier 2 */ - case CONVERT_VALUE: { - PyObject *value; - PyObject *result; - value = stack_pointer[-1]; - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error_tier_two; - stack_pointer[-1] = result; - break; - } + /* _LOAD_ATTR_NONDESCRIPTOR_NO_DICT is not a viable micro-op for tier 2 */ - case FORMAT_SIMPLE: { - PyObject *value; + /* _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_SLOT is not a viable micro-op for tier 2 */ + + /* _LOAD_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + + case _LOAD_GLOBAL: { PyObject *res; - value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result - * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + PyObject *null = NULL; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) goto error; + } + Py_INCREF(res); } else { - res = value; + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } } - stack_pointer[-1] = res; + null = NULL; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); break; } - case FORMAT_WITH_SPEC: { - PyObject *fmt_spec; - PyObject *value; - PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } + /* _LOAD_GLOBAL_BUILTINS is not a viable micro-op for tier 2 */ - case COPY: { - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); - top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; - break; - } + /* _LOAD_GLOBAL_MODULE is not a viable micro-op for tier 2 */ - case _BINARY_OP: { - PyObject *rhs; - PyObject *lhs; - PyObject *res; - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error_tier_two; - STACK_SHRINK(1); - stack_pointer[-1] = res; - break; - } + /* _LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ - case SWAP: { - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; + case _POP_FRAME: { + PyObject *retval; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif break; } - case _GUARD_IS_TRUE_POP: { - PyObject *flag; - flag = stack_pointer[-1]; - DEOPT_IF(Py_IsFalse(flag), _GUARD_IS_TRUE_POP); - assert(Py_IsTrue(flag)); - STACK_SHRINK(1); - break; - } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ - case _GUARD_IS_FALSE_POP: { - PyObject *flag; - flag = stack_pointer[-1]; - DEOPT_IF(Py_IsTrue(flag), _GUARD_IS_FALSE_POP); - assert(Py_IsFalse(flag)); - STACK_SHRINK(1); - break; - } + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ - case _GUARD_IS_NONE_POP: { - PyObject *val; - val = stack_pointer[-1]; - DEOPT_IF(!Py_IsNone(val), _GUARD_IS_NONE_POP); - STACK_SHRINK(1); - break; - } + case _PUSH_FRAME: { + _PyInterpreterFrame *new_frame; + new_frame = stack_pointer[-1]; - case _GUARD_IS_NOT_NONE_POP: { - PyObject *val; - val = stack_pointer[-1]; - DEOPT_IF(Py_IsNone(val), _GUARD_IS_NOT_NONE_POP); - Py_DECREF(val); - STACK_SHRINK(1); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + stack_pointer += (((0) ? 1 : 0)); break; } - case _JUMP_TO_TOP: { - next_uop = current_executor->trace; - CHECK_EVAL_BREAKER(); - break; - } + /* _SAVE_RETURN_OFFSET is not a viable micro-op for tier 2 */ + + /* _SEND is not a viable micro-op for tier 2 */ case _SET_IP: { + TIER_TWO_ONLY // TODO: Put the code pointer in `operand` to avoid indirection via `frame` frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg; break; } - case _SAVE_RETURN_OFFSET: { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif + /* _SPECIALIZE_BINARY_OP is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_BINARY_SUBSCR is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_CALL is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_COMPARE_OP is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_FOR_ITER is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_LOAD_ATTR is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_LOAD_GLOBAL is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_SEND is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_STORE_ATTR is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_STORE_SUBSCR is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_TO_BOOL is not a viable micro-op for tier 2 */ + + /* _SPECIALIZE_UNPACK_SEQUENCE is not a viable micro-op for tier 2 */ + + case _STORE_ATTR: { + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; + stack_pointer += -2; break; } - case _EXIT_TRACE: { - TIER_TWO_ONLY - GOTO_TIER_ONE(); + /* _STORE_ATTR_INSTANCE_VALUE is not a viable micro-op for tier 2 */ + + /* _STORE_ATTR_SLOT is not a viable micro-op for tier 2 */ + + case _STORE_SUBSCR: { + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; + stack_pointer += -3; break; } - case _INSERT: { - PyObject *top; - top = stack_pointer[-1]; - // Inserts TOS at position specified by oparg; - memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); - stack_pointer[-1 - oparg] = top; + case _TO_BOOL: { + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; + stack_pointer[-1] = res; break; } - case _CHECK_VALIDITY: { - TIER_TWO_ONLY - DEOPT_IF(!current_executor->base.vm_data.valid, _CHECK_VALIDITY); + case _UNPACK_SEQUENCE: { + PyObject *seq; + seq = stack_pointer[-1]; + + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += -1 + oparg; break; } -#undef TIER_TWO +#undef TIER_ONE diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py new file mode 100644 index 00000000000000..3f919a483d174d --- /dev/null +++ b/Tools/cases_generator/tier2_generator.py @@ -0,0 +1,321 @@ +"""Generate the main interpreter switch. +Reads the instruction definitions from bytecodes.c. +Writes the cases to generated_cases.c.h, which is #included in ceval.c. +""" + +import argparse +import os.path +import sys + +from analyzer import Analysis, Instruction, Uop, Part, analyze_files, Skip, StackItem, analysis_error +from cwriter import CWriter +from typing import TextIO, Iterator +from lexer import Token +from stack import StackOffset + +HERE = os.path.dirname(__file__) +ROOT = os.path.join(HERE, "../..") +THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/") + +DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) +DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/executor_cases.c.h")) + +def write_header(filename: str, outfile: TextIO) -> None: + outfile.write(f"""// This file is generated by {THIS} +// from: +// {filename} +// Do not edit! + +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 +""") + +FOOTER = "#undef TIER_ONE\n" + +class SizeMismatch(Exception): + pass + + +class Stack: + + def __init__(self): + self.top_offset: StackOffset = StackOffset() + self.base_offset: StackOffset = StackOffset() + self.peek_offset: StackOffset = StackOffset() + self.variables: list[StackItem] = [] + self.defined: set[str] = set() + + def pop(self, var: StackItem) -> str: + self.top_offset.pop(var) + if not var.peek: + self.peek_offset.pop(var) + indirect = "&" if var.is_array() else "" + if self.variables: + popped = self.variables.pop() + if popped.size != var.size: + raise SizeMismatch(f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}") + if popped.name == var.name: + return "" + elif popped.name == "unused": + self.defined.add(var.name) + return f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + elif var.name == "unused": + return "" + else: + self.defined.add(var.name) + return f"{var.name} = {popped.name};\n" + self.base_offset.pop(var) + if var.name == "unused": + return "" + else: + self.defined.add(var.name) + assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];" + if var.condition: + return f"if ({var.condition}) {{ {assign} }}\n" + return f"{assign}\n" + + def push(self, var: StackItem) -> None: + self.variables.append(var) + if var.is_array() and var.name not in self.defined and var.name != "unused": + c_offset = self.top_offset.to_c() + self.top_offset.push(var) + self.defined.add(var.name) + return f"{var.name} = &stack_pointer[{c_offset}];\n" + else: + self.top_offset.push(var) + return "" + + def flush(self, out: CWriter) -> None: + for var in self.variables: + if not var.peek: + if var.name != "unused" and not var.is_array(): + if var.condition: + out.emit(f" if ({var.condition}) ") + out.emit(f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n") + self.base_offset.push(var) + if self.base_offset.to_c() != self.top_offset.to_c(): + print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) + assert False + number = self.base_offset.to_c() + if number != "0": + out.emit(f"stack_pointer += {number};\n") + self.variables = [] + self.base_offset.clear() + self.top_offset.clear() + self.peek_offset.clear() + + def as_comment(self): + return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" + + +def declare_variables(uop: Uop, out: CWriter)->None: + variables = {"unused"} + for var in reversed(uop.stack.inputs): + if var.name not in variables: + type = var.type if var.type else "PyObject *" + variables.add(var.name) + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + for var in uop.stack.outputs: + if var.name not in variables: + variables.add(var.name) + type = var.type if var.type else "PyObject *" + if var.condition: + out.emit(f"{type}{var.name} = NULL;\n") + else: + out.emit(f"{type}{var.name};\n") + +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + out.emit(tkn) + +def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack): + out.emit("DEOPT_IF") + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(");") + +def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): + out.emit("if ") + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "COMMA") + label = next(tkn_iter).text + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit(") ") + c_offset = stack.peek_offset.to_c() + try: + offset = -int(c_offset) + close = ";\n" + except ValueError: + offset = None + out.emit(f"{{ stack_pointer += {c_offset}; ") + close = "; }\n" + out.emit("goto ") + if offset: + out.emit(f"pop_{offset}_") + out.emit(label) + out.emit(close) + +def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + for var in uop.stack.inputs: + if var.name == "unused" or var.name == "null" or var.peek: + continue + if var.size != "1": + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit("}\n") + elif var.condition: + out.emit(f"Py_XDECREF({var.name});\n") + else: + out.emit(f"Py_DECREF({var.name});\n") + +def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + stack.flush(out) + out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + +def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + if not uop.properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();") + + + +REPLACEMENT_FUNCTIONS = { + "DEOPT_IF" : replace_deopt, + "ERROR_IF" : replace_error, + "DECREF_INPUTS": replace_decrefs, + "CHECK_EVAL_BREAKER": replace_check_eval_breaker, + "STORE_SP": replace_store_sp, +} + +#Move this to formatter +def emit_tokens(out: CWriter, uop: Uop, stack: Stack) -> None: + tkn_iter = iter(uop.body[1:-1]) + for tkn in tkn_iter: + if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: + out.set_position(tkn) + REPLACEMENT_FUNCTIONS[tkn.text](out, tkn_iter, uop, stack) + else: + out.emit(tkn) + +def write_uop(uop: Uop, out: CWriter, stack: Stack) -> int: + try: + # out.emit(stack.as_comment() + "\n") + out.start_line() + for var in reversed(uop.stack.inputs): + out.emit(stack.pop(var)) + if not uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + reader = "read_obj" + else: + type = f"uint{cache.size*16}_t " + reader = f"read_u{cache.size*16}" + out.emit(f"{type}{cache.name} = {reader}(&next_instr[1].operand);\n") + emit_tokens(out, uop, stack) + if uop.properties.stores_sp: + for i, var in enumerate(uop.stack.outputs): + out.emit(stack.push(var)) + # out.emit(stack.as_comment() + "\n") + except SizeMismatch as ex: + raise analysis_error(ex, uop.body[0]) + +def uses_this(uop: Uop) -> bool: + if uop.properties.needs_this: + return True + for cache in uop.caches: + if cache.name != "unused": + return True + return False + +#This belongs in the analysis phase +def viable(uop: Uop) -> bool: + if uses_this(uop): + return False + if "INSTRUMENTED" in uop.name: + return False + if "replaced" in uop.annotations: + return False + if uop.name in ( + "INTERPRETER_EXIT", + "JUMP_BACKWARD" + ): + return False + + if len([c for c in uop.caches if c.name != "unused"]) > 1: + return False + return True + +def generate_tier2(filenames: str, analysis: Analysis, outfile: TextIO, lines: bool) -> None: + write_header(filenames, outfile) + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, uop in sorted(analysis.uops.items()): + if not viable(uop): + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n"); + continue + out.emit(f"case {uop.name}: {{") + declare_variables(uop, out) + stack = Stack() + write_uop(uop, out, stack) + out.start_line() + if not uop.properties.always_exits: + stack.flush(out) + if uop.properties.ends_with_eval_breaker: + out.emit("CHECK_EVAL_BREAKER();\n") + out.emit("break;\n") + out.emit("}") + out.emit("\n") + outfile.write(FOOTER) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tier2(args.input, data, outfile, args.emit_line_directives) From bec93cfe2a6d1d8399e0a7d43c1ee0dbc5670c03 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 07:02:41 +0000 Subject: [PATCH 13/40] Indent within parens to match the parens --- Python/generated_cases.c.h | 11732 ++++++++++++++--------------- Tools/cases_generator/cwriter.py | 20 +- 2 files changed, 5878 insertions(+), 5874 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5710ac52382961..7c9c6aa48074e7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/tier1_generator.py // from: -// ['./Python/bytecodes.c'] +// ['Python/bytecodes.c'] // Do not edit! #ifdef TIER_TWO @@ -8,248 +8,248 @@ #endif #define TIER_ONE 1 - TARGET(BEFORE_ASYNC_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_ASYNC_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(BEFORE_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the + TARGET(BEFORE_ASYNC_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(BEFORE_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(BINARY_OP) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *rhs; - PyObject *lhs; - PyObject *res; - // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - // _BINARY_OP - { - - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_UNICODE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_UNICODE - { - - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_INPLACE_ADD_UNICODE - { - - assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(BINARY_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *rhs; + PyObject *lhs; + PyObject *res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_UNICODE + { + + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_INPLACE_ADD_UNICODE + { + + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. * * When possible, extend `left` in place rather than * allocating a new PyUnicodeObject. This attempts to avoid @@ -260,5612 +260,5612 @@ * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(BINARY_OP_MULTIPLY_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_MULTIPLY_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_SUBTRACT_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_SUBTRACT_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; - } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); - } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - stack_pointer[-3] = res; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *res; - // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SUBSCR - { - - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error; - // not found or error - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); - STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(BINARY_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_STR_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_TUPLE_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BUILD_CONST_KEY_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error; } - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - DISPATCH(); - } - - TARGET(BUILD_LIST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; - values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(BUILD_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; - values = &stack_pointer[-oparg*2]; - - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); - } - if (map == NULL) { stack_pointer += -oparg*2; goto error; } - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; - DISPATCH(); - } - - TARGET(BUILD_SET) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; - values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error; } - } - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(BUILD_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } - stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; - start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } - stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; - stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); - DISPATCH(); - } - - TARGET(BUILD_STRING) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; - pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); - } - if (str == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(BUILD_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; - values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(CACHE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); - } - - TARGET(CALL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - // _SPECIALIZE_CALL - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _CALL - { - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* This instruction does the following: + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_SUBTRACT_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_SUBTRACT_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res = NULL; + } + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); + } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; + PyObject *container; + PyObject *res; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SUBSCR + { + + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *cached = ht->_spec_cache.getitem; + DEOPT_IF(cached == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + uint32_t cached_version = ht->_spec_cache.getitem_version; + DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + Py_INCREF(getitem); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); + STACK_SHRINK(2); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(BINARY_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_STR_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_TUPLE_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *tuple; + PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BUILD_CONST_KEY_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, + "bad BUILD_CONST_KEY_MAP keys argument"); + GOTO_ERROR(error); // Pop the keys and values. + } + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); + } + Py_DECREF(keys); + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + DISPATCH(); + } + + TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + PyObject **values; + PyObject *list; + values = &stack_pointer[-oparg]; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); + } + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + DISPATCH(); + } + + TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + PyObject **values; + PyObject *set; + values = &stack_pointer[-oparg]; + set = PySet_New(NULL); + if (set == NULL) + GOTO_ERROR(error); + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set); + if (true) { stack_pointer += -oparg; goto error; } + } + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; + + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); + DISPATCH(); + } + + TARGET(BUILD_STRING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); + } + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + PyObject **values; + PyObject *tup; + values = &stack_pointer[-oparg]; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(CACHE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); + } + + TARGET(CALL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + // _SPECIALIZE_CALL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _CALL + { + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(tp); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; - } - - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *null; - PyObject *callable; - PyObject *func; - PyObject *self; - PyObject *self_or_null; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_FAST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was + tstate->py_recursion_remaining--; + goto start_frame; + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *func; + PyObject *self; + PyObject *self_or_null; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. */ - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_O) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } - callargs = stack_pointer[-1 - ((oparg & 1))]; - func = stack_pointer[-3 - ((oparg & 1))]; - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - GOTO_ERROR(error); - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - result = PyObject_Call(func, callargs, kwargs); - } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } - - stack_pointer[-3 - ((oparg & 1))] = result; - stack_pointer += -2 - ((oparg & 1)); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_INTRINSIC_1) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(CALL_INTRINSIC_2) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(CALL_ISINSTANCE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); - } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - DISPATCH(); - } - - TARGET(CALL_KW) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -3 - oparg; goto error; } - - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_LEN) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); - } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - DISPATCH(); - } - - TARGET(CALL_LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self; - PyObject *callable; - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_O) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *self_or_null; - PyObject *callable; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); - } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(CALL_STR_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_TUPLE_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_TYPE_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - DISPATCH(); - } - - TARGET(CHECK_EG_MATCH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); - } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; - DISPATCH(); - } - - TARGET(CHECK_EXC_MATCH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; - } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; - DISPATCH(); - } - - TARGET(CLEANUP_THROW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; - } - else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *right; - PyObject *left; - PyObject *res; - // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _COMPARE_OP - { - - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP_STR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(CONTAINS_OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONTAINS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(CONVERT_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - PyObject *value; - PyObject *result; - value = stack_pointer[-1]; - - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; - stack_pointer[-1] = result; - DISPATCH(); - } - - TARGET(COPY) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - - assert(oparg > 0); - top = Py_NewRef(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(COPY_FREE_VARS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); - } - DISPATCH(); - } - - TARGET(DELETE_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(DELETE_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - DISPATCH(); - } - - TARGET(DELETE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); - DISPATCH(); - } - - TARGET(DELETE_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - GOTO_ERROR(error); - } - DISPATCH(); - } - - TARGET(DELETE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - GOTO_ERROR(error); - } - err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - GOTO_ERROR(error); - } - DISPATCH(); - } - - TARGET(DELETE_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(DICT_MERGE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; - PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; - } - Py_DECREF(update); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(DICT_UPDATE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - PyObject *update; - PyObject *dict; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); - } - Py_DECREF(update); - if (true) goto pop_1_error; - } - Py_DECREF(update); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; - TIER_ONE_ONLY - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(END_FOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - PyObject *value; - // _POP_TOP - value = stack_pointer[-1]; - { - Py_DECREF(value); - } - // _POP_TOP - value = stack_pointer[-2]; - { - Py_DECREF(value); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(END_SEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - - TIER_ONE_ONLY - CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; - Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { - current_executor = (_PyUOpExecutorObject *)executor; - GOTO_TIER_TWO(); - } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; - goto resume_with_error; - } - goto enter_tier_one; - } - - TARGET(EXIT_INIT_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); - } - stack_pointer += -1; - DISPATCH(); - } - - TARGET(EXTENDED_ARG) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TARGET(FORMAT_SIMPLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; + PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; + + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + GOTO_ERROR(error); + } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + GOTO_ERROR(error); + } + Py_SETREF(callargs, tuple); + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) GOTO_ERROR(error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyObject_Call(func, callargs, kwargs); + } + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + assert(PEEK(2 + (oparg & 1)) == NULL); + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + PyObject *value1; + PyObject *value2; + PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CALL_ISINSTANCE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + GOTO_ERROR(error); + } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CALL_KW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwnames; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames + ); + Py_DECREF(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + Py_DECREF(kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_LEN) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + GOTO_ERROR(error); + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CALL_LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self; + PyObject *callable; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); + DEOPT_IF(!PyList_Check(self), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *self_or_null; + PyObject *callable; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(CALL_PY_WITH_DEFAULTS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(tstate->interp->eval_frame, CALL); + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; + DEOPT_IF(argcount > code->co_argcount, CALL); + DEOPT_IF(argcount < min_args, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); + new_frame->localsplus[i] = Py_NewRef(def); + } + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_TUPLE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; + } + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; + + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; + + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + DISPATCH(); + } + + TARGET(CHECK_EXC_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; + } + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; + DISPATCH(); + } + + TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; + PyObject *none; + PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + TIER_ONE_ONLY + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); + none = Py_None; + } + else { + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *right; + PyObject *left; + PyObject *res; + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONTAINS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; + + convertion_func_ptr conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = CONVERSION_FUNCTIONS[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; + DISPATCH(); + } + + TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; + + assert(oparg > 0); + top = Py_NewRef(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); + } + DISPATCH(); + } + + TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + PyObject *owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); + DISPATCH(); + } + + TARGET(DELETE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err; + err = PyDict_DelItem(GLOBALS(), name); + // Can't use ERROR_IF here. + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + GOTO_ERROR(error); + } + DISPATCH(); + } + + TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + GOTO_ERROR(error); + } + err = PyObject_DelItem(ns, name); + // Can't use ERROR_IF here. + if (err != 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + GOTO_ERROR(error); + } + DISPATCH(); + } + + TARGET(DELETE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_2_error; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DICT_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; + TIER_ONE_ONLY + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + Py_DECREF(awaitable); + Py_DECREF(exc); + } + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(END_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + PyObject *value; + // _POP_TOP + value = stack_pointer[-1]; + { + Py_DECREF(value); + } + // _POP_TOP + value = stack_pointer[-2]; + { + Py_DECREF(value); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(ENTER_EXECUTOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + + TIER_ONE_ONLY + CHECK_EVAL_BREAKER(); + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); + frame->instr_ptr = next_instr; + Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } + frame = executor->execute(executor, frame, stack_pointer); + if (frame == NULL) { + frame = tstate->current_frame; + goto resume_with_error; + } + goto enter_tier_one; + } + + TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; + + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + GOTO_ERROR(error); + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - } - else { - res = value; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(FORMAT_WITH_SPEC) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; - PyObject *value; - PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(FOR_ITER) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *iter; - PyObject *next; - // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - // Common case: no jump, leave it to the code generator - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - iter = stack_pointer[-1]; - - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(FOR_ITER_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - } - // _ITER_JUMP_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(FOR_ITER_RANGE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(FOR_ITER_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_AITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; - } - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(GET_ANEXT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + } + else { + res = value; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + PyObject *fmt_spec; + PyObject *value; + PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(FOR_ITER) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *iter; + PyObject *next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ForIter(iter, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + // Common case: no jump, leave it to the code generator + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + iter = stack_pointer[-1]; + + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + PyGenObject *gen = (PyGenObject *)iter; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyFrame_StackPush(gen_frame, Py_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + Py_DECREF(r); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } - } - stack_pointer[0] = awaitable; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_AWAITABLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being + Py_DECREF(obj); + if (true) goto pop_1_error; + } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + if (true) goto pop_1_error; + } + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_ANEXT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + GOTO_ERROR(error); + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + GOTO_ERROR(error); + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + GOTO_ERROR(error); + } + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + Py_DECREF(next_iter); + GOTO_ERROR(error); + } else { + Py_DECREF(next_iter); + } + } + stack_pointer[0] = awaitable; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_AWAITABLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + } + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } - } - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(GET_ITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(GET_LEN) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - PyObject *obj; - PyObject *len_o; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - stack_pointer[0] = len_o; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_YIELD_FROM_ITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } + } + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_LEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + PyObject *obj; + PyObject *len_o; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + stack_pointer[0] = len_o; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - GOTO_ERROR(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; - } - else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(iterable); - } - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(IMPORT_FROM) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; - from = stack_pointer[-1]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(IMPORT_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(CALL); - } - - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - } - - TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); - } - - TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - /* Need to create a fake StopIteration error here, + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + GOTO_ERROR(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + if (iter == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(iterable); + } + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(IMPORT_FROM) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + PyObject *from; + PyObject *res; + from = stack_pointer[-1]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + PyObject *fromlist; + PyObject *level; + PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + + int is_meth = PEEK(oparg + 1) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 2); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(CALL); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = PEEK(oparg + 2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 3); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PEEK(total_args + 1); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); + } + + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); - } - Py_DECREF(receiver); - Py_DECREF(value); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); - } - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr; - } - else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; - } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - if (next_opcode < 0) goto error; - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } - - TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } - - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyObject *value = POP(); - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; - } - else { - Py_DECREF(value); - offset = 0; - } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyObject *value = POP(); - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; - } - else { - Py_DECREF(value); - offset = oparg; - } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); - } - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = this_instr; - DISPATCH(); - } - } - DISPATCH(); - } - - TARGET(INSTRUMENTED_RETURN_CONST) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } - - TARGET(INTERPRETER_EXIT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; - } - - TARGET(IS_OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + Py_DECREF(value); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + oparg + 1; + } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } + + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + PyObject *value = POP(); + int flag = Py_IsNone(value); + int offset; + if (flag) { + offset = oparg; + } + else { + Py_DECREF(value); + offset = 0; + } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + PyObject *value = POP(); + int offset; + int nflag = Py_IsNone(value); + if (nflag) { + offset = 0; + } + else { + Py_DECREF(value); + offset = oparg; + } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { + GOTO_ERROR(error); + } + next_instr = this_instr; + } + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = this_instr; + DISPATCH(); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; + } + + TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #if ENABLE_SPECIALIZATION + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + /* We are using unsigned values, but we really want signed values, so * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); - if (optimized < 0) goto error; - if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - } - else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; - } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; - } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; - } - } - #endif /* ENABLE_SPECIALIZATION */ - DISPATCH(); - } - - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - - /* This bytecode is used in the `yield from` or `await` loop. + uint16_t ucounter = this_instr[1].cache + (1 << 15); + uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + OPT_STAT_INC(attempts); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + if (optimized < 0) goto error; + if (optimized) { + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + } + else { + int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + if (backoff < MINIMUM_TIER2_BACKOFF) { + backoff = MINIMUM_TIER2_BACKOFF; + } + else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { + backoff++; + } + assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); + this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; + } + } + #endif /* ENABLE_SPECIALIZATION */ + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + + /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ - JUMPBY(-oparg); - DISPATCH(); - } - - TARGET(JUMP_FORWARD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); - } - - TARGET(LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(LIST_EXTEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - } - Py_DECREF(iterable); - if (true) goto pop_1_error; - } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(LOAD_ASSERTION_ERROR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_ATTR - { - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. + JUMPBY(-oparg); + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } + Py_DECREF(iterable); + if (true) goto pop_1_error; + } + assert(Py_IsNone(none_val)); + Py_DECREF(iterable); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(LOAD_ASSERTION_ERROR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_ATTR + { + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; - } - } - else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - } - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; - } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_BUILD_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - if (true) goto error; - } - stack_pointer[0] = bc; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; - } - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST_AND_CLEAR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST_LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - DISPATCH(); - } - - TARGET(LOAD_FROM_DICT_OR_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; - - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); - } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - Py_INCREF(value); - } - Py_DECREF(class_dict); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - DISPATCH(); - } - - TARGET(LOAD_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *this_instr = next_instr - 5; - PyObject *res; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_GLOBAL - { - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } + } + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + } + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + Py_INCREF(f); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = Py_NewRef(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert(oparg & 1); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + if (true) goto error; + } + stack_pointer[0] = bc; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + PyObject *value; + + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; + } + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST_AND_CLEAR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + GOTO_ERROR(error); + } + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + Py_INCREF(value); + } + Py_DECREF(class_dict); + stack_pointer[-1] = value; + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } + } + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + DISPATCH(); + } + + TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; + PyObject *res; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_GLOBAL + { + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } - } - } - null = NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - // _GUARD_BUILTINS_VERSION - { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t index = read_u16(&this_instr[4].cache); - - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_LOCALS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - stack_pointer[0] = locals; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; - - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - stack_pointer[0] = v; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *class; - PyObject *global_super; - PyObject *self; - PyObject *attr; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_SUPER_ATTR - self = stack_pointer[-1]; - { - TIER_ONE_ONLY - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - if (err) goto pop_3_error; - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } - } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; - } - stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; - stack_pointer += -2 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - stack_pointer[-3] = attr; - stack_pointer += -2 + (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR_METHOD) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; - } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; - } - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(MAKE_CELL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - GOTO_ERROR(error); - } - SETLOCAL(oparg, cell); - DISPATCH(); - } - - TARGET(MAKE_FUNCTION) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; - - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; - DISPATCH(); - } - - TARGET(MAP_ADD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(MATCH_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = Py_None; // Failure! - } - stack_pointer[-3] = attrs; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(MATCH_KEYS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - stack_pointer[0] = values_or_none; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(MATCH_MAPPING) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(MATCH_SEQUENCE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(NOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); - } - - TARGET(POP_EXCEPT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; - cond = stack_pointer[-1]; - - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; - { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } - } - // _POP_JUMP_IF_TRUE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; - { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } - } - // _POP_JUMP_IF_FALSE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_TOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(PUSH_EXC_INFO) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; - } - else { - prev_exc = Py_None; - } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(PUSH_NULL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - PyObject *res; - res = NULL; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(RAISE_VARARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - PyObject **args; - args = &stack_pointer[-oparg]; - - TIER_ONE_ONLY - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); - break; - } - if (true) { stack_pointer += -oparg; goto error; } - stack_pointer += -oparg; - DISPATCH(); - } - - TARGET(RERAISE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - - TIER_ONE_ONLY - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); - } - } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - - TARGET(RESERVED) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); - } - - TARGET(RESUME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; - TIER_ONE_ONLY - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - this_instr->op.code = RESUME_CHECK; - } - DISPATCH(); - } - - TARGET(RESUME_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } - - TARGET(RETURN_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_CONST); - PyObject *value; - PyObject *retval; - // _LOAD_CONST - { - - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - } - // _POP_FRAME - retval = value; - { - - #if TIER_ONE - assert(frame != &entry_frame); - #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - DISPATCH(); - } - - TARGET(RETURN_GENERATOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); - } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(RETURN_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != &entry_frame); - #endif - stack_pointer += -1; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - DISPATCH(); - } - - TARGET(SEND) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *receiver; - PyObject *v; - PyObject *retval; - // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Send(receiver, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _SEND - v = stack_pointer[-1]; - { - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); - } - else { - GOTO_ERROR(error); - } - } - Py_DECREF(v); - } - stack_pointer[-1] = retval; - DISPATCH(); - } - - TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; - PyObject *receiver; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(SETUP_ANNOTATIONS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) goto error; - } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } - DISPATCH(); - } - - TARGET(SET_ADD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(SET_FUNCTION_ATTRIBUTE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(SET_UPDATE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; - PyObject *owner; - PyObject *v; - // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_ATTR - v = stack_pointer[-2]; - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _GUARD_DORV_VALUES - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); - } - // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); - } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _STORE_ATTR_SLOT - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); - } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_FAST_LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); - stack_pointer[-1] = value2; - DISPATCH(); - } - - TARGET(STORE_FAST_STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; - v = stack_pointer[-1]; - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; - } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; - } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); - } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - stack_pointer += -4; - DISPATCH(); - } - - TARGET(STORE_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *v; - // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SUBSCR - v = stack_pointer[-3]; - { - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; - } - stack_pointer += -3; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - stack_pointer += -3; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer += -3; - DISPATCH(); - } - - TARGET(SWAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; - DISPATCH(); - } - - TARGET(TO_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject *value; - PyObject *res; - // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _TO_BOOL - { - - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); - } - - TARGET(TO_BOOL_INT) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - Py_DECREF(value); - res = Py_True; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_LIST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_NONE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_STR) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_INVERT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NEGATIVE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NOT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNPACK_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; - seq = stack_pointer[-1]; - - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - stack_pointer += (oparg >> 8) + (oparg & 0xFF); - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *seq; - // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - } - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(WITH_EXCEPT_START) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - - /* At the top of the stack are 4 values: + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) goto error; + } + Py_INCREF(res); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } + } + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _GUARD_BUILTINS_VERSION + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t index = read_u16(&this_instr[4].cache); + + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + Py_INCREF(locals); + stack_pointer[0] = locals; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + PyObject *v; + + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } + } + stack_pointer[0] = v; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *class; + PyObject *global_super; + PyObject *self; + PyObject *attr; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_SUPER_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_SUPER_ATTR + self = stack_pointer[-1]; + { + TIER_ONE_ONLY + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } + } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; + } + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) goto pop_3_error; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *self_or_null; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; + } + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; + } + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + GOTO_ERROR(error); + } + SETLOCAL(oparg, cell); + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + Py_DECREF(codeobj); + if (func_obj == NULL) { + GOTO_ERROR(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = (PyObject *)func_obj; + stack_pointer[-1] = func; + DISPATCH(); + } + + TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + PyObject *value; + PyObject *key; + PyObject *dict; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! + } + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = Py_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; + stack_pointer[0] = values_or_none; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + PyObject *cond; + cond = stack_pointer[-1]; + + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } + } + // _POP_JUMP_IF_TRUE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } + } + // _POP_JUMP_IF_FALSE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + PyObject *cond; + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_TOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; + } + else { + prev_exc = Py_None; + } + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + PyObject *res; + res = NULL; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + PyObject **args; + args = &stack_pointer[-oparg]; + + TIER_ONE_ONLY + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = args[1]; + /* fall through */ + case 1: + exc = args[0]; + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, + "bad RAISE_VARARGS oparg"); + break; + } + if (true) { stack_pointer += -oparg; goto error; } + stack_pointer += -oparg; + DISPATCH(); + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + TIER_ONE_ONLY + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + GOTO_ERROR(error); + } + } + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + + TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); + } + + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + TIER_ONE_ONLY + assert(frame == tstate->current_frame); + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + } + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; + } + DISPATCH(); + } + + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + DISPATCH(); + } + + TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); + PyObject *value; + PyObject *retval; + // _LOAD_CONST + { + + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + } + // _POP_FRAME + retval = value; + { + + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + DISPATCH(); + } + + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + DISPATCH(); + } + + TARGET(SEND) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *receiver; + PyObject *v; + PyObject *retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _SEND + v = stack_pointer[-1]; + { + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + GOTO_ERROR(error); + } + } + Py_DECREF(v); + } + stack_pointer[-1] = retval; + DISPATCH(); + } + + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(SETUP_ANNOTATIONS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) goto error; + } + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); + } + DISPATCH(); + } + + TARGET(SET_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; + + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); + } + stack_pointer[-2] = func; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; + PyObject *owner; + PyObject *v; + // _SPECIALIZE_STORE_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_ATTR + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + } + // _GUARD_DORV_VALUES + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + } + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; + DISPATCH(); + } + + TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + PyObject *v; + v = stack_pointer[-1]; + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) goto pop_1_error; + } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; + } + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); + } + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + stack_pointer += -4; + DISPATCH(); + } + + TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; + PyObject *container; + PyObject *v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SUBSCR + v = stack_pointer[-3]; + { + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; + } + stack_pointer += -3; + DISPATCH(); + } + + TARGET(STORE_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; + + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + stack_pointer += -3; + DISPATCH(); + } + + TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; + DISPATCH(); + } + + TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + DISPATCH(); + } + + TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject *value; + PyObject *res; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(TO_BOOL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _TO_BOOL + { + + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + uint32_t version = read_u32(&this_instr[2].cache); + + // This one is a bit weird, because we expect *some* failures: + assert(version); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + STAT_INC(TO_BOOL, hit); + Py_DECREF(value); + res = Py_True; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); + } + + TARGET(TO_BOOL_INT) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortal(value)); + res = Py_False; + } + else { + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_LIST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_NONE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_STR) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value)); + res = Py_False; + } + else { + assert(Py_SIZE(value)); + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + + res = PyNumber_Invert(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + PyObject *seq; + seq = stack_pointer[-1]; + + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + + /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() @@ -5873,54 +5873,54 @@ We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. */ - PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); - } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(YIELD_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } + PyObject *exc, *tb; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } #undef TIER_ONE diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index ddb166e6e93e0b..90e8877044b27d 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -7,7 +7,8 @@ class CWriter: def __init__(self, out: TextIO, indent: int, line_directives: bool): self.out = out - self.initial_indent = self.indent = indent + self.base_column = indent * 4 + self.indents = [ i for i in range(indent+1) ] self.line = -1 self.column = 0 self.line_directives = line_directives @@ -19,7 +20,7 @@ def set_position(self, tkn: Token): if self.line_directives: self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') if self.column == 0: - self.out.write(" " * self.indent) + self.out.write(" " * self.indents[-1]) else: #Token colums are 1 based column = tkn.column - 1 @@ -31,16 +32,19 @@ def set_position(self, tkn: Token): def maybe_dedent(self, txt: str): parens = txt.count("(") - txt.count(")") if parens < 0: - self.indent += parens + self.indents.pop() elif "}" in txt or txt.endswith(":"): - self.indent -= 1 + self.indents.pop() def maybe_indent(self, txt: str): parens = txt.count("(") - txt.count(")") if parens > 0: - self.indent += parens + offset = self.column - self.base_column + 2 + if offset <= self.indents[-1] or offset > 40: + offset = self.indents[-1] + 4 + self.indents.append(offset) elif "{" in txt or txt.endswith(":"): - self.indent += 1 + self.indents.append(self.indents[-1] + 4) def emit_token(self, tkn: Token): self.maybe_dedent(tkn.text) @@ -50,8 +54,8 @@ def emit_token(self, tkn: Token): def emit_text(self, txt: str): if self.column == 0 and txt.strip(): - self.out.write(" " * self.indent) - self.column = self.initial_indent * 4 + self.out.write(" " * self.indents[-1]) + self.column = self.base_column self.out.write(txt) def emit_str(self, txt: str): From 8481bea2af6591dd57aea746cc29320afd08ced3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 07:10:23 +0000 Subject: [PATCH 14/40] Fix indenting --- Python/generated_cases.c.h | 11600 ++++++++++++++--------------- Tools/cases_generator/cwriter.py | 2 +- 2 files changed, 5801 insertions(+), 5801 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7c9c6aa48074e7..1799071e4725fe 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,6 +1,6 @@ // This file is generated by Tools/cases_generator/tier1_generator.py // from: -// ['Python/bytecodes.c'] +// ['./Python/bytecodes.c'] // Do not edit! #ifdef TIER_TWO @@ -8,248 +8,248 @@ #endif #define TIER_ONE 1 - TARGET(BEFORE_ASYNC_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_ASYNC_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, + TARGET(BEFORE_ASYNC_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object does not support the " "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object does not support the " "asynchronous context manager protocol " "(missed __aexit__ method)", Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(BEFORE_WITH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BEFORE_WITH); - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(BEFORE_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_WITH); + PyObject *mgr; + PyObject *exit; + PyObject *res; + mgr = stack_pointer[-1]; + /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, + PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + if (enter == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object does not support the " "context manager protocol", Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, + } + GOTO_ERROR(error); + } + exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + if (exit == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object does not support the " "context manager protocol " "(missed __exit__ method)", Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = _PyObject_CallNoArgsTstate(tstate, enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(BINARY_OP) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *rhs; - PyObject *lhs; - PyObject *res; - // _SPECIALIZE_BINARY_OP - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - // _BINARY_OP - { - - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_ADD_UNICODE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_ADD_UNICODE - { - - STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - // _GUARD_BOTH_UNICODE - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_INPLACE_ADD_UNICODE - { - - assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); - STAT_INC(BINARY_OP, hit); - /* Handle `left = left + right` or `left += right` for str. + } + Py_DECREF(enter); + GOTO_ERROR(error); + } + Py_DECREF(mgr); + res = _PyObject_CallNoArgsTstate(tstate, enter); + Py_DECREF(enter); + if (res == NULL) { + Py_DECREF(exit); + if (true) goto pop_1_error; + } + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(BINARY_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *rhs; + PyObject *lhs; + PyObject *res; + // _SPECIALIZE_BINARY_OP + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_ADD_UNICODE + { + + STAT_INC(BINARY_OP, hit); + res = PyUnicode_Concat(left, right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_INPLACE_ADD_UNICODE + { + + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(*target_local != left, BINARY_OP); + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. * * When possible, extend `left` in place rather than * allocating a new PyUnicodeObject. This attempts to avoid @@ -260,5612 +260,5612 @@ * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; - // The STORE_FAST is already done. - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(BINARY_OP_MULTIPLY_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_MULTIPLY_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_MULTIPLY_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_SUBTRACT_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_FLOAT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_FLOAT - { - - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_OP_SUBTRACT_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - // _GUARD_BOTH_INT - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - } - // _BINARY_OP_SUBTRACT_INT - { - - STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - // Can't use ERROR_IF() here, because we haven't - // DECREF'ed container yet, and we still own slice. - if (slice == NULL) { - res = NULL; - } - else { - res = PyObject_GetItem(container, slice); - Py_DECREF(slice); - } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; - stack_pointer[-3] = res; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *res; - // _SPECIALIZE_BINARY_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SUBSCR - { - - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); - } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) goto pop_2_error; - // not found or error - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_GETITEM) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); - PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); - assert(PyFunction_Check(cached)); - PyFunctionObject *getitem = (PyFunctionObject *)cached; - uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); - PyCodeObject *code = (PyCodeObject *)getitem->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); - STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(BINARY_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_STR_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BINARY_SUBSCR_TUPLE_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(BUILD_CONST_KEY_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, + assert(Py_REFCNT(left) >= 2); + _Py_DECREF_NO_DEALLOC(left); + PyUnicode_Append(target_local, right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + if (*target_local == NULL) goto pop_2_error; + // The STORE_FAST is already done. + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_MULTIPLY_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_SUBTRACT_FLOAT + { + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; + DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + } + // _BINARY_OP_SUBTRACT_INT + { + + STAT_INC(BINARY_OP, hit); + res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + // Can't use ERROR_IF() here, because we haven't + // DECREF'ed container yet, and we still own slice. + if (slice == NULL) { + res = NULL; + } + else { + res = PyObject_GetItem(container, slice); + Py_DECREF(slice); + } + Py_DECREF(container); + if (res == NULL) goto pop_3_error; + stack_pointer[-3] = res; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); + PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; + PyObject *container; + PyObject *res; + // _SPECIALIZE_BINARY_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SUBSCR + { + + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + + DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) goto pop_2_error; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + PyTypeObject *tp = Py_TYPE(container); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *cached = ht->_spec_cache.getitem; + DEOPT_IF(cached == NULL, BINARY_SUBSCR); + assert(PyFunction_Check(cached)); + PyFunctionObject *getitem = (PyFunctionObject *)cached; + uint32_t cached_version = ht->_spec_cache.getitem_version; + DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + PyCodeObject *code = (PyCodeObject *)getitem->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + Py_INCREF(getitem); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); + STACK_SHRINK(2); + new_frame->localsplus[0] = container; + new_frame->localsplus[1] = sub; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(BINARY_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyList_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_STR_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BINARY_SUBSCR_TUPLE_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *tuple; + PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + // Deopt unless 0 <= sub < PyTuple_Size(list) + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(BUILD_CONST_KEY_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + if (!PyTuple_CheckExact(keys) || + PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { + _PyErr_SetString(tstate, PyExc_SystemError, "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } - map = _PyDict_FromItems( + GOTO_ERROR(error); // Pop the keys and values. + } + map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error; } - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - DISPATCH(); - } - - TARGET(BUILD_LIST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; - values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(BUILD_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; - values = &stack_pointer[-oparg*2]; - - map = _PyDict_FromItems( + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); + } + Py_DECREF(keys); + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + DISPATCH(); + } + + TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + PyObject **values; + PyObject *list; + values = &stack_pointer[-oparg]; + list = _PyList_FromArraySteal(values, oparg); + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + + map = _PyDict_FromItems( values, 2, values+1, 2, oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); - } - if (map == NULL) { stack_pointer += -oparg*2; goto error; } - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; - DISPATCH(); - } - - TARGET(BUILD_SET) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; - values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error; } - } - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(BUILD_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } - stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; - start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } - stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; - stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); - DISPATCH(); - } - - TARGET(BUILD_STRING) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; - pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); - } - if (str == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(BUILD_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; - values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error; } - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - DISPATCH(); - } - - TARGET(CACHE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); - } - - TARGET(CALL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED(CALL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - // _SPECIALIZE_CALL - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _CALL - { - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); + } + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + DISPATCH(); + } + + TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + PyObject **values; + PyObject *set; + values = &stack_pointer[-oparg]; + set = PySet_New(NULL); + if (set == NULL) + GOTO_ERROR(error); + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = values[i]; + if (err == 0) + err = PySet_Add(set, item); + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set); + if (true) { stack_pointer += -oparg; goto error; } + } + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; + + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); + DISPATCH(); + } + + TARGET(BUILD_STRING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); + } + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(BUILD_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + PyObject **values; + PyObject *tup; + values = &stack_pointer[-oparg]; + tup = _PyTuple_FromArraySteal(values, oparg); + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + DISPATCH(); + } + + TARGET(CACHE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); + } + + TARGET(CALL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + // _SPECIALIZE_CALL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _CALL + { + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* This instruction does the following: + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; - PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; - PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); - STAT_INC(CALL, hit); - PyObject *self = _PyType_NewManagedObject(tp); - if (self == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(tp); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); - /* Push self onto stack of shim */ - Py_INCREF(self); - shim->localsplus[0] = self; - Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; - for (int i = 0; i < oparg; i++) { - init_frame->localsplus[i+1] = args[i]; - } - frame->return_offset = (uint16_t)(next_instr - this_instr); - STACK_SHRINK(oparg+2); - _PyFrame_SetStackPointer(frame, stack_pointer); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; - frame = tstate->current_frame = init_frame; - CALL_STAT_INC(inlined_py_calls); - /* Account for pushing the extra frame. + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); + if (self == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(tp); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Copy self followed by args to __init__ frame */ + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + STACK_SHRINK(oparg+2); + _PyFrame_SetStackPointer(frame, stack_pointer); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; + frame = tstate->current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); + /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ - tstate->py_recursion_remaining--; - goto start_frame; - } - - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *null; - PyObject *callable; - PyObject *func; - PyObject *self; - PyObject *self_or_null; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - - STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = self; - callable = func; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_FAST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((_PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - /* Not deopting because this doesn't mean our optimization was + tstate->py_recursion_remaining--; + goto start_frame; + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *func; + PyObject *self; + PyObject *self_or_null; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + + STAT_INC(CALL, hit); + self = Py_NewRef(((PyMethodObject *)callable)->im_self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + func = Py_NewRef(((PyMethodObject *)callable)->im_func); + stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization + Py_DECREF(callable); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = self; + callable = func; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyType_Check(callable), CALL); + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. */ - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_BUILTIN_O) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } - callargs = stack_pointer[-1 - ((oparg & 1))]; - func = stack_pointer[-3 - ((oparg & 1))]; - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); - } - PyObject *tuple = PySequence_Tuple(callargs); - if (tuple == NULL) { - GOTO_ERROR(error); - } - Py_SETREF(callargs, tuple); - } - assert(PyTuple_CheckExact(callargs)); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && - !PyFunction_Check(func) && !PyMethod_Check(func) - ) { - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); - result = PyObject_Call(func, callargs, kwargs); - if (result == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - if (err < 0) { - Py_CLEAR(result); - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - assert(PyTuple_CheckExact(callargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); - // Need to manually shrink the stack since we exit with DISPATCH_INLINED. - STACK_SHRINK(oparg + 3); - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - result = PyObject_Call(func, callargs, kwargs); - } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } - - stack_pointer[-3 - ((oparg & 1))] = result; - stack_pointer += -2 - ((oparg & 1)); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_INTRINSIC_1) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(CALL_INTRINSIC_2) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(CALL_ISINSTANCE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); - } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - DISPATCH(); - } - - TARGET(CALL_KW) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); - _Py_CODEUNIT *this_instr = next_instr - 1; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames - ); - Py_DECREF(kwnames); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 3); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - GOTO_ERROR(error); - } - assert(next_instr - this_instr == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_BUILTIN_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; + PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; + + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (!PyTuple_CheckExact(callargs)) { + if (check_args_iterable(tstate, func, callargs) < 0) { + GOTO_ERROR(error); + } + PyObject *tuple = PySequence_Tuple(callargs); + if (tuple == NULL) { + GOTO_ERROR(error); + } + Py_SETREF(callargs, tuple); + } + assert(PyTuple_CheckExact(callargs)); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + if (err) GOTO_ERROR(error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + result = PyObject_Call(func, callargs, kwargs); + } + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + assert(PEEK(2 + (oparg & 1)) == NULL); + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + PyObject *value1; + PyObject *value2; + PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + Py_DECREF(value2); + Py_DECREF(value1); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CALL_ISINSTANCE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + /* isinstance(o, o2) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 2, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + STAT_INC(CALL, hit); + PyObject *cls = args[1]; + PyObject *inst = args[0]; + int retval = PyObject_IsInstance(inst, cls); + if (retval < 0) { + GOTO_ERROR(error); + } + res = PyBool_FromLong(retval); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CALL_KW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_KW); + PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; + PyObject *kwnames; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames + ); + Py_DECREF(kwnames); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 3); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + GOTO_ERROR(error); + } + assert(next_instr - this_instr == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); - } - } - } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { stack_pointer += -3 - oparg; goto error; } - - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_LEN) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); - } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - DISPATCH(); - } - - TARGET(CALL_LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self; - PyObject *callable; - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - assert(oparg == 1); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); - STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } - Py_DECREF(self); - Py_DECREF(callable); - STACK_SHRINK(3); - // Skip POP_TOP - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_METHOD_DESCRIPTOR_O) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *self_or_null; - PyObject *callable; - PyObject **args; - _PyInterpreterFrame *new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - DEOPT_IF(tstate->interp->eval_frame, CALL); - } - // _CHECK_FUNCTION_EXACT_ARGS - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - { - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); - } - // _CHECK_STACK_SPACE - { - - PyFunctionObject *func = (PyFunctionObject *)callable; - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); - } - // _INIT_CALL_PY_EXACT_ARGS - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - { - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; - new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - } - // _SAVE_RETURN_OFFSET - { - - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. - assert(tstate->interp->eval_frame == NULL); - stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame->previous = frame; - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = new_frame; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(CALL_PY_WITH_DEFAULTS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); - } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(CALL_STR_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PyObject_Str(arg); - Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_TUPLE_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - res = PySequence_Tuple(arg); - Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_TYPE_1) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; - PyObject *res; - args = &stack_pointer[-oparg]; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); - STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(obj); - Py_DECREF(&PyType_Type); // I.e., callable - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - DISPATCH(); - } - - TARGET(CHECK_EG_MATCH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) goto pop_2_error; - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) goto pop_2_error; - - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); - } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; - DISPATCH(); - } - - TARGET(CHECK_EXC_MATCH) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; - } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-1] = b; - DISPATCH(); - } - - TARGET(CLEANUP_THROW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - TIER_ONE_ONLY - assert(throwflag); - assert(exc_value && PyExceptionInstance_Check(exc_value)); - if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; - } - else { - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *right; - PyObject *left; - PyObject *res; - // _SPECIALIZE_COMPARE_OP - right = stack_pointer[-1]; - left = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _COMPARE_OP - { - - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP_FLOAT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + Py_DECREF(kwnames); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_LEN) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* len(o) */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.len, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + Py_ssize_t len_i = PyObject_Length(arg); + if (len_i < 0) { + GOTO_ERROR(error); + } + res = PyLong_FromSsize_t(len_i); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + Py_DECREF(arg); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CALL_LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self; + PyObject *callable; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + assert(self != NULL); + DEOPT_IF(!PyList_Check(self), CALL); + STAT_INC(CALL, hit); + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already + } + Py_DECREF(self); + Py_DECREF(callable); + STACK_SHRINK(3); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + _PyCFunctionFast cfunc = + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + DEOPT_IF(total_args != 1, CALL); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + PyObject *arg = args[1]; + PyObject *self = args[0]; + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + GOTO_ERROR(error); + } + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *self_or_null; + PyObject *callable; + PyObject **args; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + { + + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + } + // _INIT_CALL_PY_EXACT_ARGS + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + } + // _SAVE_RETURN_OFFSET + { + + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(CALL_PY_WITH_DEFAULTS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(tstate->interp->eval_frame, CALL); + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; + DEOPT_IF(argcount > code->co_argcount, CALL); + DEOPT_IF(argcount < min_args, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + STAT_INC(CALL, hit); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); + new_frame->localsplus[i] = Py_NewRef(def); + } + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_TUPLE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + STAT_INC(CALL, hit); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); + Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject **args; + PyObject *null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 1); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + STAT_INC(CALL, hit); + res = Py_NewRef(Py_TYPE(obj)); + Py_DECREF(obj); + Py_DECREF(&PyType_Type); // I.e., callable + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + DISPATCH(); + } + + TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) goto pop_2_error; + } + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) goto pop_2_error; + + assert((match == NULL) == (rest == NULL)); + if (match == NULL) goto pop_2_error; + + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + DISPATCH(); + } + + TARGET(CHECK_EXC_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert(PyExceptionInstance_Check(left)); + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + Py_DECREF(right); + if (true) goto pop_1_error; + } + int res = PyErr_GivenExceptionMatches(left, right); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-1] = b; + DISPATCH(); + } + + TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; + PyObject *none; + PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + TIER_ONE_ONLY + assert(throwflag); + assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { + value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + Py_DECREF(sub_iter); + Py_DECREF(last_sent_val); + Py_DECREF(exc_value); + none = Py_None; + } + else { + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED(COMPARE_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *right; + PyObject *left; + PyObject *res; + // _SPECIALIZE_COMPARE_OP + right = stack_pointer[-1]; + left = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(COMPARE_OP_STR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(CONTAINS_OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONTAINS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(CONVERT_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - PyObject *value; - PyObject *result; - value = stack_pointer[-1]; - - convertion_func_ptr conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = CONVERSION_FUNCTIONS[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; - stack_pointer[-1] = result; - DISPATCH(); - } - - TARGET(COPY) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; - bottom = stack_pointer[-1 - (oparg-1)]; - - assert(oparg > 0); - top = Py_NewRef(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(COPY_FREE_VARS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); - } - DISPATCH(); - } - - TARGET(DELETE_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(DELETE_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - // Can't use ERROR_IF here. - // Fortunately we don't need its superpower. - if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - DISPATCH(); - } - - TARGET(DELETE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; - SETLOCAL(oparg, NULL); - DISPATCH(); - } - - TARGET(DELETE_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err; - err = PyDict_DelItem(GLOBALS(), name); - // Can't use ERROR_IF here. - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left, right); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONTAINS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = PySequence_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + PyObject *value; + PyObject *result; + value = stack_pointer[-1]; + + convertion_func_ptr conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = CONVERSION_FUNCTIONS[oparg]; + result = conv_fn(value); + Py_DECREF(value); + if (result == NULL) goto pop_1_error; + stack_pointer[-1] = result; + DISPATCH(); + } + + TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + PyObject *bottom; + PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; + + assert(oparg > 0); + top = Py_NewRef(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + /* Copy closure variables to free variables */ + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyFunction_Check(frame->f_funcobj)); + PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = Py_NewRef(o); + } + DISPATCH(); + } + + TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + PyObject *owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_DelAttr(owner, name); + Py_DECREF(owner); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + // Can't use ERROR_IF here. + // Fortunately we don't need its superpower. + if (oldobj == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + PyCell_SET(cell, NULL); + Py_DECREF(oldobj); + DISPATCH(); + } + + TARGET(DELETE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + PyObject *v = GETLOCAL(oparg); + if (v == NULL) goto unbound_local_error; + SETLOCAL(oparg, NULL); + DISPATCH(); + } + + TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err; + err = PyDict_DelItem(GLOBALS(), name); + // Can't use ERROR_IF here. + if (err != 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - GOTO_ERROR(error); - } - DISPATCH(); - } - - TARGET(DELETE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, + } + GOTO_ERROR(error); + } + DISPATCH(); + } + + TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); - } - err = PyObject_DelItem(ns, name); - // Can't use ERROR_IF here. - if (err != 0) { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + GOTO_ERROR(error); + } + err = PyObject_DelItem(ns, name); + // Can't use ERROR_IF here. + if (err != 0) { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - DISPATCH(); - } - - TARGET(DELETE_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - PyObject *sub; - PyObject *container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_2_error; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(DICT_MERGE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; - PyObject *callable; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); - if (true) goto pop_1_error; - } - Py_DECREF(update); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(DICT_UPDATE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - PyObject *update; - PyObject *dict; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, + GOTO_ERROR(error); + } + DISPATCH(); + } + + TARGET(DELETE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + /* del container[sub] */ + int err = PyObject_DelItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_2_error; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + PyObject *update; + PyObject *dict; + PyObject *callable; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + + if (_PyDict_MergeEx(dict, update, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable, update); + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(DICT_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); - } - Py_DECREF(update); - if (true) goto pop_1_error; - } - Py_DECREF(update); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; - TIER_ONE_ONLY - assert(exc && PyExceptionInstance_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); - } - else { - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(END_FOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - PyObject *value; - // _POP_TOP - value = stack_pointer[-1]; - { - Py_DECREF(value); - } - // _POP_TOP - value = stack_pointer[-2]; - { - Py_DECREF(value); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(END_SEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(ENTER_EXECUTOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - - TIER_ONE_ONLY - CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; - int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); - JUMPBY(1-original_oparg); - frame->instr_ptr = next_instr; - Py_INCREF(executor); - if (executor->execute == _PyUopExecute) { - current_executor = (_PyUOpExecutorObject *)executor; - GOTO_TIER_TWO(); - } - frame = executor->execute(executor, frame, stack_pointer); - if (frame == NULL) { - frame = tstate->current_frame; - goto resume_with_error; - } - goto enter_tier_one; - } - - TARGET(EXIT_INIT_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; - should_be_none = stack_pointer[-1]; - - assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { - PyErr_Format(PyExc_TypeError, + } + Py_DECREF(update); + if (true) goto pop_1_error; + } + Py_DECREF(update); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; + TIER_ONE_ONLY + assert(exc && PyExceptionInstance_Check(exc)); + if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + Py_DECREF(awaitable); + Py_DECREF(exc); + } + else { + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(END_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + PyObject *value; + // _POP_TOP + value = stack_pointer[-1]; + { + Py_DECREF(value); + } + // _POP_TOP + value = stack_pointer[-2]; + { + Py_DECREF(value); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(ENTER_EXECUTOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + + TIER_ONE_ONLY + CHECK_EVAL_BREAKER(); + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; + int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); + JUMPBY(1-original_oparg); + frame->instr_ptr = next_instr; + Py_INCREF(executor); + if (executor->execute == _PyUopExecute) { + current_executor = (_PyUOpExecutorObject *)executor; + GOTO_TIER_TWO(); + } + frame = executor->execute(executor, frame, stack_pointer); + if (frame == NULL) { + frame = tstate->current_frame; + goto resume_with_error; + } + goto enter_tier_one; + } + + TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; + + assert(STACK_LEVEL() == 2); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); - } - stack_pointer += -1; - DISPATCH(); - } - - TARGET(EXTENDED_ARG) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TARGET(FORMAT_SIMPLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - /* If value is a unicode object, then we know the result + GOTO_ERROR(error); + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - } - else { - res = value; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(FORMAT_WITH_SPEC) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; - PyObject *value; - PyObject *res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(FOR_ITER) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *iter; - PyObject *next; - // _SPECIALIZE_FOR_ITER - iter = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - // Common case: no jump, leave it to the code generator - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(FOR_ITER_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - iter = stack_pointer[-1]; - - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); - PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr[oparg].op.code == END_FOR || + if (!PyUnicode_CheckExact(value)) { + res = PyObject_Format(value, NULL); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + } + else { + res = value; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + PyObject *fmt_spec; + PyObject *value; + PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + res = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_DECREF(fmt_spec); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(FOR_ITER) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *iter; + PyObject *next; + // _SPECIALIZE_FOR_ITER + iter = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ForIter(iter, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + // Common case: no jump, leave it to the code generator + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + iter = stack_pointer[-1]; + + DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + PyGenObject *gen = (PyGenObject *)iter; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + _PyFrame_StackPush(gen_frame, Py_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(FOR_ITER_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - } - // _ITER_JUMP_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_LIST - { - - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(FOR_ITER_RANGE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - iter = stack_pointer[-1]; - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - } - // _ITER_JUMP_RANGE - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - STACK_SHRINK(1); - Py_DECREF(r); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - assert(Py_TYPE(r) == &PyRangeIter_Type); - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(FOR_ITER_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - iter = stack_pointer[-1]; - { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - } - // _ITER_JUMP_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - } - stack_pointer[0] = next; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_AITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + } + // _ITER_JUMP_LIST + { + + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyList_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_LIST + { + + _PyListIterObject *it = (_PyListIterObject *)iter; + assert(Py_TYPE(iter) == &PyListIter_Type); + PyListObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyList_GET_SIZE(seq)); + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + } + // _ITER_JUMP_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + STACK_SHRINK(1); + Py_DECREF(r); + // Jump over END_FOR instruction. + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + assert(Py_TYPE(r) == &PyRangeIter_Type); + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) goto error; + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(FOR_ITER_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + PyObject *iter; + PyObject *next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; + { + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + } + // _ITER_JUMP_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { + if (seq != NULL) { + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyTupleObject *seq = it->it_seq; + assert(seq); + assert(it->it_index < PyTuple_GET_SIZE(seq)); + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + } + stack_pointer[0] = next; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); - if (true) goto pop_1_error; - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, + Py_DECREF(obj); + if (true) goto pop_1_error; + } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) goto pop_1_error; + + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) goto pop_1_error; - } - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(GET_ANEXT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - GOTO_ERROR(error); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - GOTO_ERROR(error); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, + Py_DECREF(iter); + if (true) goto pop_1_error; + } + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_ANEXT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + GOTO_ERROR(error); + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + GOTO_ERROR(error); + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( + GOTO_ERROR(error); + } + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - GOTO_ERROR(error); - } else { - Py_DECREF(next_iter); - } - } - stack_pointer[0] = awaitable; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_AWAITABLE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being + Py_DECREF(next_iter); + GOTO_ERROR(error); + } else { + Py_DECREF(next_iter); + } + } + stack_pointer[0] = awaitable; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_AWAITABLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + } + Py_DECREF(iterable); + + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } - } - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(GET_ITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - - /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(GET_LEN) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - PyObject *obj; - PyObject *len_o; - obj = stack_pointer[-1]; - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error; - stack_pointer[0] = len_o; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(GET_YIELD_FROM_ITER) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyObject *iterable; - PyObject *iter; - iterable = stack_pointer[-1]; - - /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a + /* The code below jumps to `error` if `iter` is NULL. */ + } + } + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + + /* before: [obj]; after [getiter(obj)] */ + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (iter == NULL) goto pop_1_error; + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(GET_LEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + PyObject *obj; + PyObject *len_o; + obj = stack_pointer[-1]; + // PUSH(len(TOS)) + Py_ssize_t len_i = PyObject_Length(obj); + if (len_i < 0) goto error; + len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) goto error; + stack_pointer[0] = len_o; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + + /* before: [obj]; after [getiter(obj)] */ + if (PyCoro_CheckExact(iterable)) { + /* `iterable` is a coroutine */ + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + /* and it is used in a 'yield from' expression of a regular generator. */ - _PyErr_SetString(tstate, PyExc_TypeError, + _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable)) { - iter = iterable; - } - else { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(iterable); - } - stack_pointer[-1] = iter; - DISPATCH(); - } - - TARGET(IMPORT_FROM) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; - from = stack_pointer[-1]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(IMPORT_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; - stack_pointer[-2] = res; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - - int is_meth = PEEK(oparg + 1) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(CALL); - } - - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - } - - TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); - PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - GO_TO_INSTRUCTION(CALL_KW); - } - - TARGET(INSTRUMENTED_END_FOR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - /* Need to create a fake StopIteration error here, + GOTO_ERROR(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable)) { + iter = iterable; + } + else { + /* `iterable` is not a generator. */ + iter = PyObject_GetIter(iterable); + if (iter == NULL) { + GOTO_ERROR(error); + } + Py_DECREF(iterable); + } + stack_pointer[-1] = iter; + DISPATCH(); + } + + TARGET(IMPORT_FROM) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + PyObject *from; + PyObject *res; + from = stack_pointer[-1]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_from(tstate, from, name); + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + PyObject *fromlist; + PyObject *level; + PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + TIER_ONE_ONLY + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + res = import_name(tstate, frame, name, fromlist, level); + Py_DECREF(level); + Py_DECREF(fromlist); + if (res == NULL) goto pop_2_error; + stack_pointer[-2] = res; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + + int is_meth = PEEK(oparg + 1) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 2); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(CALL); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = PEEK(oparg + 2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(oparg + 3); + PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING + : PEEK(total_args + 1); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + if (err) goto error; + GO_TO_INSTRUCTION(CALL_KW); + } + + TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); - } - Py_DECREF(receiver); - Py_DECREF(value); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - TIER_ONE_ONLY - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); - } - PyErr_SetRaisedException(NULL); - } - Py_DECREF(receiver); - stack_pointer[-2] = value; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - - _Py_CODEUNIT *target; - PyObject *iter = TOP(); - PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - target = next_instr; - } - else { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); - } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + Py_DECREF(value); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + TIER_ONE_ONLY + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, this_instr)) { + GOTO_ERROR(error); + } + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + stack_pointer[-2] = value; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + GOTO_ERROR(error); + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - STACK_SHRINK(1); - Py_DECREF(iter); - /* Skip END_FOR */ - target = next_instr + oparg + 1; - } - INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - if (next_opcode < 0) goto error; - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - - CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } - - TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - DISPATCH(); - } - - TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyObject *value = POP(); - int flag = Py_IsNone(value); - int offset; - if (flag) { - offset = oparg; - } - else { - Py_DECREF(value); - offset = 0; - } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyObject *value = POP(); - int offset; - int nflag = Py_IsNone(value); - if (nflag) { - offset = 0; - } - else { - Py_DECREF(value); - offset = oparg; - } - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - int offset = flag * oparg; - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - DISPATCH(); - } - - TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - if (code_version != global_version) { - if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); - } - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) goto error; - if (frame->instr_ptr != this_instr) { - /* Instrumentation has jumped */ - next_instr = this_instr; - DISPATCH(); - } - } - DISPATCH(); - } - - TARGET(INSTRUMENTED_RETURN_CONST) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); - if (err) GOTO_ERROR(error); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } - - TARGET(INTERPRETER_EXIT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; - retval = stack_pointer[-1]; - assert(frame == &entry_frame); - assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; - } - - TARGET(IS_OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(JUMP_BACKWARD) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - - CHECK_EVAL_BREAKER(); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - #if ENABLE_SPECIALIZATION - this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - /* We are using unsigned values, but we really want signed values, so + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + oparg + 1; + } + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + if (next_opcode < 0) goto error; + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + + CHECK_EVAL_BREAKER(); + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } + + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + PyObject *value = POP(); + int flag = Py_IsNone(value); + int offset; + if (flag) { + offset = oparg; + } + else { + Py_DECREF(value); + offset = 0; + } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + PyObject *value = POP(); + int offset; + int nflag = Py_IsNone(value); + if (nflag) { + offset = 0; + } + else { + Py_DECREF(value); + offset = oparg; + } + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + PyObject *cond = POP(); + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + int offset = flag * oparg; + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + if (code_version != global_version) { + if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { + GOTO_ERROR(error); + } + next_instr = this_instr; + } + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + /* Instrumentation has jumped */ + next_instr = this_instr; + DISPATCH(); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); + + PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); + if (err) GOTO_ERROR(error); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } + + TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + PyObject *retval; + retval = stack_pointer[-1]; + assert(frame == &entry_frame); + assert(_PyFrame_IsIncomplete(frame)); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return retval; + } + + TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + + int res = Py_Is(left, right) ^ oparg; + Py_DECREF(left); + Py_DECREF(right); + b = res ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + + CHECK_EVAL_BREAKER(); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + #if ENABLE_SPECIALIZATION + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + /* We are using unsigned values, but we really want signed values, so * do the 2s complement comparison manually */ - uint16_t ucounter = this_instr[1].cache + (1 << 15); - uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: - if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { - OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); - if (optimized < 0) goto error; - if (optimized) { - // Rewind and enter the executor: - assert(this_instr->op.code == ENTER_EXECUTOR); - next_instr = this_instr; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - } - else { - int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); - if (backoff < MINIMUM_TIER2_BACKOFF) { - backoff = MINIMUM_TIER2_BACKOFF; - } - else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { - backoff++; - } - assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); - this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; - } - } - #endif /* ENABLE_SPECIALIZATION */ - DISPATCH(); - } - - TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - - /* This bytecode is used in the `yield from` or `await` loop. + uint16_t ucounter = this_instr[1].cache + (1 << 15); + uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); + // Double-check that the opcode isn't instrumented or something: + if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { + OPT_STAT_INC(attempts); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); + if (optimized < 0) goto error; + if (optimized) { + // Rewind and enter the executor: + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + } + else { + int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + if (backoff < MINIMUM_TIER2_BACKOFF) { + backoff = MINIMUM_TIER2_BACKOFF; + } + else if (backoff < 15 - OPTIMIZER_BITS_IN_COUNTER) { + backoff++; + } + assert(backoff <= 15 - OPTIMIZER_BITS_IN_COUNTER); + this_instr[1].cache = ((1 << 16) - ((1 << OPTIMIZER_BITS_IN_COUNTER) << backoff)) | backoff; + } + } + #endif /* ENABLE_SPECIALIZATION */ + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + + /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ - JUMPBY(-oparg); - DISPATCH(); - } - - TARGET(JUMP_FORWARD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - JUMPBY(oparg); - DISPATCH(); - } - - TARGET(LIST_APPEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(LIST_EXTEND) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + JUMPBY(-oparg); + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - } - Py_DECREF(iterable); - if (true) goto pop_1_error; - } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(LOAD_ASSERTION_ERROR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); - PyObject *value; - value = Py_NewRef(PyExc_AssertionError); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 10; - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - // _SPECIALIZE_LOAD_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_ATTR - { - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. + } + Py_DECREF(iterable); + if (true) goto pop_1_error; + } + assert(Py_IsNone(none_val)); + Py_DECREF(iterable); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(LOAD_ASSERTION_ERROR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); + PyObject *value; + value = Py_NewRef(PyExc_AssertionError); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_ATTR + { + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; - } - } - else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - } - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); - } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); - self = owner; - } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - self = owner; - } - stack_pointer[-1] = attr; - if (1) stack_pointer[0] = self; - stack_pointer += (((1) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _CHECK_ATTR_MODULE - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; - assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < dict->ma_keys->dk_nentries); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); - } - stack_pointer[-1] = attr; - stack_pointer += (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - - TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_WITH_HINT - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; - } - DEOPT_IF(attr == NULL, LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); - } - /* Skip 5 cache entries */ - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_BUILD_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { - _PyErr_SetString(tstate, PyExc_NameError, + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } + } + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + } + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + assert(type_version != 0); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + null = NULL; + Py_DECREF(owner); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + Py_INCREF(f); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + new_frame->localsplus[1] = Py_NewRef(name); + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + + Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; + assert(dictoffset > 0); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert(oparg & 1); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = Py_NewRef(descr); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *self = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; + } + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _CHECK_ATTR_MODULE + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict != NULL); + DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < dict->ma_keys->dk_nentries); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + assert(Py_TYPE(owner)->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(owner); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + Py_DECREF(owner); + attr = Py_NewRef(descr); + } + stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + PyTypeObject *cls = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + assert(func_version != 0); + DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + // Manipulate stack directly because we exit with DISPATCH_INLINED(). + STACK_SHRINK(1); + new_frame->localsplus[0] = owner; + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + + TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + + char *addr = (char *)owner + index; + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_WITH_HINT + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, LOAD_ATTR); + attr = ep->me_value; + } + DEOPT_IF(attr == NULL, LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(attr); + null = NULL; + Py_DECREF(owner); + } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + PyObject *bc; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + if (bc == NULL) { + _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error; - } - stack_pointer[0] = bc; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error; - } - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST_AND_CLEAR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; - value = GETLOCAL(oparg); - // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; - Py_INCREF(value); - stack_pointer[0] = value; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_FAST_LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - DISPATCH(); - } - - TARGET(LOAD_FROM_DICT_OR_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; - - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); - } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); - } - Py_INCREF(value); - } - Py_DECREF(class_dict); - stack_pointer[-1] = value; - DISPATCH(); - } - - TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - DISPATCH(); - } - - TARGET(LOAD_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); - _Py_CODEUNIT *this_instr = next_instr - 5; - PyObject *res; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_GLOBAL - { - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + if (true) goto error; + } + stack_pointer[0] = bc; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + PyObject *value; + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + PyObject *value; + + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + if (true) goto error; + } + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + PyObject *value; + value = GETLOCAL(oparg); + assert(value != NULL); + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST_AND_CLEAR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + PyObject *value; + value = GETLOCAL(oparg); + // do not use SETLOCAL here, it decrefs the old value + GETLOCAL(oparg) = NULL; + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + PyObject *value; + value = GETLOCAL(oparg); + if (value == NULL) goto unbound_local_error; + Py_INCREF(value); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = GETLOCAL(oparg1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value1); + Py_INCREF(value2); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + GOTO_ERROR(error); + } + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + GOTO_ERROR(error); + } + Py_INCREF(value); + } + Py_DECREF(class_dict); + stack_pointer[-1] = value; + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } + } + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + DISPATCH(); + } + + TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; + PyObject *res; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_GLOBAL + { + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) goto error; - } - } - } - null = NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - // _GUARD_BUILTINS_VERSION - { - uint16_t version = read_u16(&this_instr[3].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t index = read_u16(&this_instr[4].cache); - - PyDictObject *bdict = (PyDictObject *)BUILTINS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); - assert(DK_IS_UNICODE(dict->ma_keys)); - } - /* Skip 1 cache entry */ - // _LOAD_GLOBAL_MODULE - { - uint16_t index = read_u16(&this_instr[4].cache); - - PyDictObject *dict = (PyDictObject *)GLOBALS(); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); - STAT_INC(LOAD_GLOBAL, hit); - null = NULL; - } - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_LOCALS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) goto error; + } + Py_INCREF(res); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } + } + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _GUARD_BUILTINS_VERSION + { + uint16_t version = read_u16(&this_instr[3].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t index = read_u16(&this_instr[4].cache); + + PyDictObject *bdict = (PyDictObject *)BUILTINS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; + PyObject *null = NULL; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + assert(DK_IS_UNICODE(dict->ma_keys)); + } + /* Skip 1 cache entry */ + // _LOAD_GLOBAL_MODULE + { + uint16_t index = read_u16(&this_instr[4].cache); + + PyDictObject *dict = (PyDictObject *)GLOBALS(); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); + res = entries[index].me_value; + DEOPT_IF(res == NULL, LOAD_GLOBAL); + Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, hit); + null = NULL; + } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - stack_pointer[0] = locals; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; - - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, + if (true) goto error; + } + Py_INCREF(locals); + stack_pointer[0] = locals; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + PyObject *v; + + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - stack_pointer[0] = v; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *class; - PyObject *global_super; - PyObject *self; - PyObject *attr; - PyObject *null = NULL; - // _SPECIALIZE_LOAD_SUPER_ATTR - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _LOAD_SUPER_ATTR - self = stack_pointer[-1]; - { - TIER_ONE_ONLY - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - if (err) goto pop_3_error; - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - if (err < 0) { - Py_CLEAR(super); - } - } - } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; - } - stack_pointer[-3] = attr; - if (oparg & 1) stack_pointer[-2] = null; - stack_pointer += -2 + ((oparg & 1)); - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) goto pop_3_error; - stack_pointer[-3] = attr; - stack_pointer += -2 + (((0) ? 1 : 0)); - DISPATCH(); - } - - TARGET(LOAD_SUPER_ATTR_METHOD) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, + if (true) goto error; + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + GOTO_ERROR(error); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + GOTO_ERROR(error); + } + } + } + stack_pointer[0] = v; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *class; + PyObject *global_super; + PyObject *self; + PyObject *attr; + PyObject *null = NULL; + // _SPECIALIZE_LOAD_SUPER_ATTR + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_SUPER_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_SUPER_ATTR + self = stack_pointer[-1]; + { + TIER_ONE_ONLY + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } + } + } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; + } + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + ((oparg & 1)); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(!(oparg & 1)); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) goto pop_3_error; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *self_or_null; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) goto pop_3_error; - } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; - } - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(MAKE_CELL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - GOTO_ERROR(error); - } - SETLOCAL(oparg, cell); - DISPATCH(); - } - - TARGET(MAKE_FUNCTION) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; - - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); - if (func_obj == NULL) { - GOTO_ERROR(error); - } - _PyFunction_SetVersion( + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) goto pop_3_error; + } + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; + } + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + // "initial" is probably NULL but not if it's an arg (or set + // via PyFrame_LocalsToFast() before MAKE_CELL has run). + PyObject *initial = GETLOCAL(oparg); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + GOTO_ERROR(error); + } + SETLOCAL(oparg, cell); + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + PyObject *codeobj; + PyObject *func; + codeobj = stack_pointer[-1]; + + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + Py_DECREF(codeobj); + if (func_obj == NULL) { + GOTO_ERROR(error); + } + _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; - stack_pointer[-1] = func; - DISPATCH(); - } - - TARGET(MAP_ADD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; - assert(PyDict_CheckExact(dict)); - /* dict[key] = value */ - // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(MATCH_CLASS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) goto pop_3_error; - // Error! - attrs = Py_None; // Failure! - } - stack_pointer[-3] = attrs; - stack_pointer += -2; - DISPATCH(); - } - - TARGET(MATCH_KEYS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; - stack_pointer[0] = values_or_none; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(MATCH_MAPPING) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(MATCH_SEQUENCE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; - PyObject *res; - subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(NOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - DISPATCH(); - } - - TARGET(POP_EXCEPT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; - cond = stack_pointer[-1]; - - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; - { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } - } - // _POP_JUMP_IF_TRUE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; - // _IS_NONE - value = stack_pointer[-1]; - { - if (Py_IsNone(value)) { - b = Py_True; - } - else { - b = Py_False; - Py_DECREF(value); - } - } - // _POP_JUMP_IF_FALSE - cond = b; - { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - } - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyObject *cond; - cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); - #if ENABLE_SPECIALIZATION - this_instr[1].cache = (this_instr[1].cache << 1) | flag; - #endif - JUMPBY(oparg * flag); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(POP_TOP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - PyObject *value; - value = stack_pointer[-1]; - Py_DECREF(value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(PUSH_EXC_INFO) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; - new_exc = stack_pointer[-1]; - - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; - } - else { - prev_exc = Py_None; - } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(PUSH_NULL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - PyObject *res; - res = NULL; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(RAISE_VARARGS) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - PyObject **args; - args = &stack_pointer[-oparg]; - - TIER_ONE_ONLY - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = args[1]; - /* fall through */ - case 1: - exc = args[0]; - /* fall through */ - case 0: - if (do_raise(tstate, exc, cause)) { - assert(oparg == 0); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - break; - default: - _PyErr_SetString(tstate, PyExc_SystemError, + func = (PyObject *)func_obj; + stack_pointer[-1] = func; + DISPATCH(); + } + + TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + PyObject *value; + PyObject *key; + PyObject *dict; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict = stack_pointer[-3 - (oparg - 1)]; + assert(PyDict_CheckExact(dict)); + /* dict[key] = value */ + // Do not DECREF INPUTS because the function steals the references + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! + } + else { + if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! + attrs = Py_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + PyObject *keys; + PyObject *subject; + PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + // On successful match, PUSH(values). Otherwise, PUSH(None). + values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + if (values_or_none == NULL) goto error; + stack_pointer[0] = values_or_none; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + PyObject *subject; + PyObject *res; + subject = stack_pointer[-1]; + int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? Py_True : Py_False; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + PyObject *exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + Py_XSETREF(exc_info->exc_value, exc_value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + PyObject *cond; + cond = stack_pointer[-1]; + + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } + } + // _POP_JUMP_IF_TRUE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + PyObject *value; + PyObject *b; + PyObject *cond; + // _IS_NONE + value = stack_pointer[-1]; + { + if (Py_IsNone(value)) { + b = Py_True; + } + else { + b = Py_False; + Py_DECREF(value); + } + } + // _POP_JUMP_IF_FALSE + cond = b; + { + assert(PyBool_Check(cond)); + int flag = Py_IsFalse(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + } + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + PyObject *cond; + cond = stack_pointer[-1]; + assert(PyBool_Check(cond)); + int flag = Py_IsTrue(cond); + #if ENABLE_SPECIALIZATION + this_instr[1].cache = (this_instr[1].cache << 1) | flag; + #endif + JUMPBY(oparg * flag); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(POP_TOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + PyObject *value; + value = stack_pointer[-1]; + Py_DECREF(value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + PyObject *new_exc; + PyObject *prev_exc; + new_exc = stack_pointer[-1]; + + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = exc_info->exc_value; + } + else { + prev_exc = Py_None; + } + assert(PyExceptionInstance_Check(new_exc)); + exc_info->exc_value = Py_NewRef(new_exc); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + PyObject *res; + res = NULL; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + PyObject **args; + args = &stack_pointer[-oparg]; + + TIER_ONE_ONLY + PyObject *cause = NULL, *exc = NULL; + switch (oparg) { + case 2: + cause = args[1]; + /* fall through */ + case 1: + exc = args[0]; + /* fall through */ + case 0: + if (do_raise(tstate, exc, cause)) { + assert(oparg == 0); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + break; + default: + _PyErr_SetString(tstate, PyExc_SystemError, "bad RAISE_VARARGS oparg"); - break; - } - if (true) { stack_pointer += -oparg; goto error; } - stack_pointer += -oparg; - DISPATCH(); - } - - TARGET(RERAISE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - - TIER_ONE_ONLY - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - PyObject *lasti = values[0]; - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - assert(PyLong_Check(lasti)); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); - } - } - assert(exc && PyExceptionInstance_Check(exc)); - Py_INCREF(exc); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - goto exception_unwind; - } - - TARGET(RESERVED) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); - } - - TARGET(RESUME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); - _Py_CODEUNIT *this_instr = next_instr - 1; - TIER_ONE_ONLY - assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); - } - this_instr->op.code = RESUME_CHECK; - } - DISPATCH(); - } - - TARGET(RESUME_CHECK) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); - uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); - DISPATCH(); - } - - TARGET(RETURN_CONST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_CONST); - PyObject *value; - PyObject *retval; - // _LOAD_CONST - { - - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); - } - // _POP_FRAME - retval = value; - { - - #if TIER_ONE - assert(frame != &entry_frame); - #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - } - DISPATCH(); - } - - TARGET(RETURN_GENERATOR) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - if (gen == NULL) { - GOTO_ERROR(error); - } - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->instr_ptr = next_instr; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - _PyFrame_StackPush(frame, (PyObject *)gen); - LOAD_IP(frame->return_offset); - goto resume_frame; - } - - TARGET(RETURN_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - #if TIER_ONE - assert(frame != &entry_frame); - #endif - stack_pointer += -1; - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(EMPTY()); - _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_SP(); - LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; - } - #endif - DISPATCH(); - } - - TARGET(SEND) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED(SEND); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *receiver; - PyObject *v; - PyObject *retval; - // _SPECIALIZE_SEND - receiver = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_Send(receiver, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _SEND - v = stack_pointer[-1]; - { - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); - } - else { - GOTO_ERROR(error); - } - } - Py_DECREF(v); - } - stack_pointer[-1] = retval; - DISPATCH(); - } - - TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; - PyObject *receiver; - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); - } - - TARGET(SETUP_ANNOTATIONS) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, + break; + } + if (true) { stack_pointer += -oparg; goto error; } + stack_pointer += -oparg; + DISPATCH(); + } + + TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + + TIER_ONE_ONLY + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + PyObject *lasti = values[0]; + if (PyLong_Check(lasti)) { + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + assert(!_PyErr_Occurred(tstate)); + } + else { + assert(PyLong_Check(lasti)); + _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); + GOTO_ERROR(error); + } + } + assert(exc && PyExceptionInstance_Check(exc)); + Py_INCREF(exc); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + goto exception_unwind; + } + + TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + + assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); + } + + TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; + TIER_ONE_ONLY + assert(frame == tstate->current_frame); + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + } + else { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } + this_instr->op.code = RESUME_CHECK; + } + DISPATCH(); + } + + TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); + uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + DEOPT_IF(eval_breaker != version, RESUME); + DISPATCH(); + } + + TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); + PyObject *value; + PyObject *retval; + // _LOAD_CONST + { + + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + } + // _POP_FRAME + retval = value; + { + + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + } + DISPATCH(); + } + + TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + + assert(PyFunction_Check(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + if (gen == NULL) { + GOTO_ERROR(error); + } + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); + goto resume_frame; + } + + TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + LOAD_SP(); + LOAD_IP(frame->return_offset); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + DISPATCH(); + } + + TARGET(SEND) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *receiver; + PyObject *v; + PyObject *retval; + // _SPECIALIZE_SEND + receiver = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _SEND + v = stack_pointer[-1]; + { + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); + } + else { + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + GOTO_ERROR(error); + } + } + Py_DECREF(v); + } + stack_pointer[-1] = retval; + DISPATCH(); + } + + TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + DEOPT_IF(tstate->interp->eval_frame, SEND); + PyGenObject *gen = (PyGenObject *)receiver; + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(next_instr - this_instr + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + DISPATCH_INLINED(gen_frame); + } + + TARGET(SETUP_ANNOTATIONS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error; - } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error; - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + if (true) goto error; + } + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) goto error; + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - Py_DECREF(ann_dict); - if (err) goto error; - } - else { - Py_DECREF(ann_dict); - } - DISPATCH(); - } - - TARGET(SET_ADD) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(SET_FUNCTION_ATTRIBUTE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; - - assert(PyFunction_Check(func)); - PyFunctionObject *func_obj = (PyFunctionObject *)func; - switch(oparg) { - case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; - case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; - case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; - case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; - default: - Py_UNREACHABLE(); - } - stack_pointer[-2] = func; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(SET_UPDATE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_ATTR) { - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); - _Py_CODEUNIT *this_instr = next_instr - 5; - PyObject *owner; - PyObject *v; - // _SPECIALIZE_STORE_ATTR - owner = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_ATTR - v = stack_pointer[-2]; - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _GUARD_DORV_VALUES - { - - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); - } - // _STORE_ATTR_INSTANCE_VALUE - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyObject *old_value = values->values[index]; - values->values[index] = value; - if (old_value == NULL) { - _PyDictValues_AddToInsertionOrder(values, index); - } - else { - Py_DECREF(old_value); - } - Py_DECREF(owner); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - owner = stack_pointer[-1]; - { - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - } - // _STORE_ATTR_SLOT - value = stack_pointer[-2]; - { - uint16_t index = read_u16(&this_instr[4].cache); - - char *addr = (char *)owner + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); - } - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint16_t hint = read_u16(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(owner); - assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); - assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - PyObject *old_value; - uint64_t new_version; - if (DK_IS_UNICODE(dict->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - else { - PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; - } - Py_DECREF(old_value); - STAT_INC(STORE_ATTR, hit); - /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { - _PyObject_GC_TRACK(dict); - } - /* PEP 509 */ - dict->ma_version_tag = new_version; - Py_DECREF(owner); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_DEREF) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; - v = stack_pointer[-1]; - PyObject *cell = GETLOCAL(oparg); - PyObject *oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - PyObject *value; - value = stack_pointer[-1]; - SETLOCAL(oparg, value); - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_FAST_LOAD_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); - stack_pointer[-1] = value2; - DISPATCH(); - } - - TARGET(STORE_FAST_STORE_FAST) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - SETLOCAL(oparg1, value1); - SETLOCAL(oparg2, value2); - stack_pointer += -2; - DISPATCH(); - } - - TARGET(STORE_GLOBAL) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; - v = stack_pointer[-1]; - - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_NAME) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - PyObject *v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, + Py_DECREF(ann_dict); + if (err) goto error; + } + else { + Py_DECREF(ann_dict); + } + DISPATCH(); + } + + TARGET(SET_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = PySet_Add(set, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; + + assert(PyFunction_Check(func)); + PyFunctionObject *func_obj = (PyFunctionObject *)func; + switch(oparg) { + case MAKE_FUNCTION_CLOSURE: + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; + case MAKE_FUNCTION_ANNOTATIONS: + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; + case MAKE_FUNCTION_KWDEFAULTS: + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; + case MAKE_FUNCTION_DEFAULTS: + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; + default: + Py_UNREACHABLE(); + } + stack_pointer[-2] = func; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + int err = _PySet_Update(set, iterable); + Py_DECREF(iterable); + if (err < 0) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; + PyObject *owner; + PyObject *v; + // _SPECIALIZE_STORE_ATTR + owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_ATTR + v = stack_pointer[-2]; + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + } + // _GUARD_DORV_VALUES + { + + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + } + // _STORE_ATTR_INSTANCE_VALUE + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + STAT_INC(STORE_ATTR, hit); + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyObject *old_value = values->values[index]; + values->values[index] = value; + if (old_value == NULL) { + _PyDictValues_AddToInsertionOrder(values, index); + } + else { + Py_DECREF(old_value); + } + Py_DECREF(owner); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + } + // _STORE_ATTR_SLOT + value = stack_pointer[-2]; + { + uint16_t index = read_u16(&this_instr[4].cache); + + char *addr = (char *)owner + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + *(PyObject **)addr = value; + Py_XDECREF(old_value); + Py_DECREF(owner); + } + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + DEOPT_IF(dict == NULL, STORE_ATTR); + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + PyObject *old_value; + uint64_t new_version; + if (DK_IS_UNICODE(dict->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + else { + PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; + DEOPT_IF(ep->me_key != name, STORE_ATTR); + old_value = ep->me_value; + DEOPT_IF(old_value == NULL, STORE_ATTR); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); + ep->me_value = value; + } + Py_DECREF(old_value); + STAT_INC(STORE_ATTR, hit); + /* Ensure dict is GC tracked if it needs to be */ + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + _PyObject_GC_TRACK(dict); + } + /* PEP 509 */ + dict->ma_version_tag = new_version; + Py_DECREF(owner); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + PyObject *v; + v = stack_pointer[-1]; + PyObject *cell = GETLOCAL(oparg); + PyObject *oldobj = PyCell_GET(cell); + PyCell_SET(cell, v); + Py_XDECREF(oldobj); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + PyObject *value; + value = stack_pointer[-1]; + SETLOCAL(oparg, value); + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + value2 = GETLOCAL(oparg2); + Py_INCREF(value2); + stack_pointer[-1] = value2; + DISPATCH(); + } + + TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + SETLOCAL(oparg1, value1); + SETLOCAL(oparg2, value2); + stack_pointer += -2; + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + PyObject *v; + v = stack_pointer[-1]; + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err = PyDict_SetItem(GLOBALS(), name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); - if (true) goto pop_1_error; - } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) goto pop_1_error; - stack_pointer += -1; - DISPATCH(); - } - - TARGET(STORE_SLICE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; - } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); - } - Py_DECREF(v); - Py_DECREF(container); - if (err) goto pop_4_error; - stack_pointer += -4; - DISPATCH(); - } - - TARGET(STORE_SUBSCR) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *sub; - PyObject *container; - PyObject *v; - // _SPECIALIZE_STORE_SUBSCR - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SUBSCR - v = stack_pointer[-3]; - { - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; - } - stack_pointer += -3; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_DICT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - value = stack_pointer[-3]; - - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); - if (err) goto pop_3_error; - stack_pointer += -3; - DISPATCH(); - } - - TARGET(STORE_SUBSCR_LIST_INT) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer += -3; - DISPATCH(); - } - - TARGET(SWAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - - assert(oparg >= 2); - stack_pointer[-2 - (oparg-2)] = top; - stack_pointer[-1] = bottom; - DISPATCH(); - } - - TARGET(TO_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); - _Py_CODEUNIT *this_instr = next_instr - 4; - PyObject *value; - PyObject *res; - // _SPECIALIZE_TO_BOOL - value = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - } - // _TO_BOOL - { - - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_BOOL) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - DISPATCH(); - } - - TARGET(TO_BOOL_INT) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - Py_DECREF(value); - res = Py_True; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_LIST) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_NONE) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - res = Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(TO_BOOL_STR) { - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); - STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; - } - else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; - } - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_INVERT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NEGATIVE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNARY_NOT) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; - value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; - stack_pointer[-1] = res; - DISPATCH(); - } - - TARGET(UNPACK_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; - seq = stack_pointer[-1]; - - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - stack_pointer += (oparg >> 8) + (oparg & 0xFF); - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); - _Py_CODEUNIT *this_instr = next_instr - 2; - PyObject *seq; - // _SPECIALIZE_UNPACK_SEQUENCE - seq = stack_pointer[-1]; - { - uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; - } - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_LIST) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); - for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); - } - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); - assert(oparg == 2); - STAT_INC(UNPACK_SEQUENCE, hit); - values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - Py_DECREF(seq); - stack_pointer += -1 + oparg; - DISPATCH(); - } - - TARGET(WITH_EXCEPT_START) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - - /* At the top of the stack are 4 values: + Py_DECREF(v); + if (true) goto pop_1_error; + } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) goto pop_1_error; + stack_pointer += -1; + DISPATCH(); + } + + TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; + } + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); + } + Py_DECREF(v); + Py_DECREF(container); + if (err) goto pop_4_error; + stack_pointer += -4; + DISPATCH(); + } + + TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *sub; + PyObject *container; + PyObject *v; + // _SPECIALIZE_STORE_SUBSCR + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SUBSCR + v = stack_pointer[-3]; + { + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; + } + stack_pointer += -3; + DISPATCH(); + } + + TARGET(STORE_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; + + DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + Py_DECREF(dict); + if (err) goto pop_3_error; + stack_pointer += -3; + DISPATCH(); + } + + TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); + DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + // Ensure nonnegative, zero-or-one-digit ints. + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; + DISPATCH(); + } + + TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + + assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + DISPATCH(); + } + + TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; + PyObject *value; + PyObject *res; + // _SPECIALIZE_TO_BOOL + value = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(TO_BOOL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _TO_BOOL + { + + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + uint32_t version = read_u32(&this_instr[2].cache); + + // This one is a bit weird, because we expect *some* failures: + assert(version); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + STAT_INC(TO_BOOL, hit); + Py_DECREF(value); + res = Py_True; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + value = stack_pointer[-1]; + DEOPT_IF(!PyBool_Check(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + DISPATCH(); + } + + TARGET(TO_BOOL_INT) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value)) { + assert(_Py_IsImmortal(value)); + res = Py_False; + } + else { + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_LIST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_SIZE(value) ? Py_True : Py_False; + Py_DECREF(value); + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_NONE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + // This one is a bit weird, because we expect *some* failures: + DEOPT_IF(!Py_IsNone(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + res = Py_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_STR) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + STAT_INC(TO_BOOL, hit); + if (value == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value)); + res = Py_False; + } + else { + assert(Py_SIZE(value)); + Py_DECREF(value); + res = Py_True; + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + + res = PyNumber_Invert(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + res = PyNumber_Negative(value); + Py_DECREF(value); + if (res == NULL) goto pop_1_error; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + PyObject *value; + PyObject *res; + value = stack_pointer[-1]; + assert(PyBool_Check(value)); + res = Py_IsFalse(value) ? Py_True : Py_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + PyObject *seq; + seq = stack_pointer[-1]; + + int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); + PyObject **top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + stack_pointer += (oparg >> 8) + (oparg & 0xFF); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; + PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE + seq = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + + DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq); + for (int i = oparg; --i >= 0; ) { + *values++ = Py_NewRef(items[i]); + } + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + assert(oparg == 2); + STAT_INC(UNPACK_SEQUENCE, hit); + values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + Py_DECREF(seq); + stack_pointer += -1 + oparg; + DISPATCH(); + } + + TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + + /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception - lasti: THIRD = lasti of exception in exc_info() @@ -5873,54 +5873,54 @@ We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. */ - PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); - } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, + PyObject *exc, *tb; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; - stack_pointer[0] = res; - stack_pointer += 1; - DISPATCH(); - } - - TARGET(YIELD_VALUE) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; - retval = stack_pointer[-1]; - // NOTE: It's important that YIELD_VALUE never raises an exception! - // The compiler treats any exception raised here as a failed close() - // or throw() call. - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyFrame_GetGenerator(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } + if (res == NULL) goto error; + stack_pointer[0] = res; + stack_pointer += 1; + DISPATCH(); + } + + TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + PyObject *retval; + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame != &entry_frame); + frame->instr_ptr = next_instr; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + goto resume_frame; + } #undef TIER_ONE diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 90e8877044b27d..fab6d5fed227b6 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -8,7 +8,7 @@ class CWriter: def __init__(self, out: TextIO, indent: int, line_directives: bool): self.out = out self.base_column = indent * 4 - self.indents = [ i for i in range(indent+1) ] + self.indents = [ i*4 for i in range(indent+1) ] self.line = -1 self.column = 0 self.line_directives = line_directives From c91e61aa157e355fa1dce875ef0cdddaaf4a9c38 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 07:34:30 +0000 Subject: [PATCH 15/40] Fix indenting --- Python/generated_cases.c.h | 136 +++++++++++++++---------------- Tools/cases_generator/cwriter.py | 2 +- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1799071e4725fe..4a75a1c3d12169 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,9 +21,9 @@ if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "asynchronous context manager protocol", + Py_TYPE(mgr)->tp_name); } GOTO_ERROR(error); } @@ -31,10 +31,10 @@ if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "asynchronous context manager protocol " + "(missed __aexit__ method)", + Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); GOTO_ERROR(error); @@ -67,9 +67,9 @@ if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "context manager protocol", + Py_TYPE(mgr)->tp_name); } GOTO_ERROR(error); } @@ -77,10 +77,10 @@ if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); + "'%.200s' object does not support the " + "context manager protocol " + "(missed __exit__ method)", + Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); GOTO_ERROR(error); @@ -609,12 +609,12 @@ if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); + "bad BUILD_CONST_KEY_MAP keys argument"); GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } @@ -648,9 +648,9 @@ values = &stack_pointer[-oparg*2]; map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } @@ -817,9 +817,9 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; @@ -1400,9 +1400,9 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; @@ -2066,7 +2066,7 @@ DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); + _PyLong_DigitCount((PyLongObject *)right) <= 1); Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg @@ -2229,7 +2229,7 @@ if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } @@ -2245,15 +2245,15 @@ int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); + "no locals when deleting %R", name); GOTO_ERROR(error); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } DISPATCH(); @@ -2308,8 +2308,8 @@ if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error; @@ -2412,8 +2412,8 @@ assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } stack_pointer += -1; @@ -2509,7 +2509,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2542,7 +2542,7 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); @@ -2702,9 +2702,9 @@ } if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); Py_DECREF(obj); if (true) goto pop_1_error; } @@ -2715,9 +2715,9 @@ if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); Py_DECREF(iter); if (true) goto pop_1_error; } @@ -2752,18 +2752,18 @@ } else { _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); GOTO_ERROR(error); } awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); GOTO_ERROR(error); } else { @@ -2797,7 +2797,7 @@ Py_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); + "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } @@ -2854,8 +2854,8 @@ /* and it is used in a 'yield from' expression of a regular generator. */ _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); GOTO_ERROR(error); } iter = iterable; @@ -3021,7 +3021,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ @@ -3384,8 +3384,8 @@ { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error; @@ -3962,7 +3962,7 @@ if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); + "__build_class__ not found"); if (true) goto error; } stack_pointer[0] = bc; @@ -4151,8 +4151,8 @@ && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising @@ -4272,7 +4272,7 @@ locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); + "no locals found"); if (true) goto error; } Py_INCREF(locals); @@ -4290,7 +4290,7 @@ PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); + "no locals found"); if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -4442,7 +4442,7 @@ PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); if (attr == NULL) { @@ -4491,7 +4491,7 @@ GOTO_ERROR(error); } _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; DISPATCH(); @@ -4778,7 +4778,7 @@ break; default: _PyErr_SetString(tstate, PyExc_SystemError, - "bad RAISE_VARARGS oparg"); + "bad RAISE_VARARGS oparg"); break; } if (true) { stack_pointer += -oparg; goto error; } @@ -5068,7 +5068,7 @@ PyObject *ann_dict; if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); + "no locals found when setting up annotations"); if (true) goto error; } /* check if __annotations__ in locals()... */ @@ -5077,7 +5077,7 @@ ann_dict = PyDict_New(); if (ann_dict == NULL) goto error; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); + ann_dict); Py_DECREF(ann_dict); if (err) goto error; } @@ -5405,7 +5405,7 @@ int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); + "no locals found when storing %R", name); Py_DECREF(v); if (true) goto pop_1_error; } @@ -5887,7 +5887,7 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; stack_pointer[0] = res; stack_pointer += 1; diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index fab6d5fed227b6..e5feee2f6da7ee 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -39,7 +39,7 @@ def maybe_dedent(self, txt: str): def maybe_indent(self, txt: str): parens = txt.count("(") - txt.count(")") if parens > 0: - offset = self.column - self.base_column + 2 + offset = self.column if offset <= self.indents[-1] or offset > 40: offset = self.indents[-1] + 4 self.indents.append(offset) From f76160c80d1a395f3a4d601cfae619ff09926bf0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Nov 2023 08:16:55 +0000 Subject: [PATCH 16/40] Remove surplus whitespace --- Python/generated_cases.c.h | 130 ++--------------------- Tools/cases_generator/cwriter.py | 8 +- Tools/cases_generator/tier1_generator.py | 8 +- 3 files changed, 15 insertions(+), 131 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4a75a1c3d12169..d16f83bb0d9580 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,7 @@ #endif #define TIER_ONE 1 + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -16,7 +17,6 @@ PyObject *exit; PyObject *res; mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -112,7 +112,6 @@ lhs = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -128,7 +127,6 @@ } // _BINARY_OP { - assert(_PyEval_BinaryOps[oparg]); res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); @@ -157,7 +155,6 @@ } // _BINARY_OP_ADD_FLOAT { - STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -186,7 +183,6 @@ } // _BINARY_OP_ADD_INT { - STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -210,13 +206,11 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { - STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -244,7 +238,6 @@ } // _BINARY_OP_INPLACE_ADD_UNICODE { - assert(next_instr->op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); @@ -290,7 +283,6 @@ } // _BINARY_OP_MULTIPLY_FLOAT { - STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -319,7 +311,6 @@ } // _BINARY_OP_MULTIPLY_INT { - STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -343,13 +334,11 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { - STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -378,7 +367,6 @@ } // _BINARY_OP_SUBTRACT_INT { - STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -401,7 +389,6 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -446,7 +433,6 @@ } // _BINARY_SUBSCR { - res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); @@ -467,7 +453,6 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); @@ -605,7 +590,6 @@ PyObject *map; keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -646,7 +630,6 @@ PyObject **values; PyObject *map; values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( values, 2, values+1, 2, @@ -697,7 +680,6 @@ if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; - slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -743,7 +725,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); - assert(0 && "Executing a cache."); Py_UNREACHABLE(); } @@ -777,8 +758,7 @@ } // _CALL { - - // oparg counts all of the args, but *not* self: + // oparg counts all of the args, but *not* self: int total_args = oparg; if (self_or_null != NULL) { args--; @@ -862,7 +842,6 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -938,7 +917,6 @@ } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { - STAT_INC(CALL, hit); self = Py_NewRef(((PyMethodObject *)callable)->im_self); stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS @@ -951,7 +929,6 @@ callable = func; { uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -960,7 +937,6 @@ } // _CHECK_STACK_SPACE { - PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); @@ -984,7 +960,6 @@ } // _SAVE_RETURN_OFFSET { - #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif @@ -1028,7 +1003,6 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1064,7 +1038,6 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ int total_args = oparg; if (self_or_null != NULL) { @@ -1194,7 +1167,6 @@ if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } callargs = stack_pointer[-1 - ((oparg & 1))]; func = stack_pointer[-3 - ((oparg & 1))]; - // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -1314,7 +1286,6 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ int total_args = oparg; if (self_or_null != NULL) { @@ -1357,8 +1328,7 @@ args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; - - // oparg counts all of the args, but *not* self: + // oparg counts all of the args, but *not* self: int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1482,7 +1452,6 @@ args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.list_append, CALL); @@ -1513,7 +1482,6 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; if (self_or_null != NULL) { args--; @@ -1706,7 +1674,6 @@ } // _CHECK_STACK_SPACE { - PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); @@ -1730,7 +1697,6 @@ } // _SAVE_RETURN_OFFSET { - #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif @@ -1774,7 +1740,6 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(tstate->interp->eval_frame, CALL); int argcount = oparg; if (self_or_null != NULL) { @@ -1820,7 +1785,6 @@ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); @@ -1992,7 +1956,6 @@ left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -2006,7 +1969,6 @@ } // _COMPARE_OP { - assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); @@ -2133,7 +2095,6 @@ PyObject *value; PyObject *result; value = stack_pointer[-1]; - convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -2151,7 +2112,6 @@ PyObject *bottom; PyObject *top; bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); top = Py_NewRef(bottom); stack_pointer[0] = top; @@ -2194,7 +2154,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -2286,7 +2245,6 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); @@ -2370,7 +2328,6 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; @@ -2381,7 +2338,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - TIER_ONE_ONLY CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); @@ -2408,7 +2364,6 @@ INSTRUCTION_STATS(EXIT_INIT_CHECK); PyObject *should_be_none; should_be_none = stack_pointer[-1]; - assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -2424,7 +2379,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -2496,7 +2450,6 @@ } // _FOR_ITER { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next == NULL) { @@ -2530,7 +2483,6 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -2563,7 +2515,6 @@ } // _ITER_JUMP_LIST { - _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); @@ -2582,7 +2533,6 @@ } // _ITER_NEXT_LIST { - _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -2606,13 +2556,11 @@ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); @@ -2626,7 +2574,6 @@ } // _ITER_NEXT_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -2656,7 +2603,6 @@ } // _ITER_JUMP_TUPLE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); @@ -2675,7 +2621,6 @@ } // _ITER_NEXT_TUPLE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -2813,7 +2758,6 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -2846,7 +2790,6 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; - /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2915,7 +2858,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -2933,7 +2875,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } @@ -3003,7 +2944,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3035,7 +2975,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); if (next_opcode < 0) goto error; @@ -3052,7 +2991,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); @@ -3080,7 +3018,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - PyObject *cond = POP(); assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); @@ -3183,7 +3120,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -3233,7 +3169,6 @@ INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; - assert(frame != &entry_frame); frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); @@ -3282,7 +3217,6 @@ PyObject *b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -3296,7 +3230,6 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); - CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); @@ -3337,7 +3270,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3376,7 +3308,6 @@ PyObject *list; iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -3420,7 +3351,6 @@ owner = stack_pointer[-1]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -3435,7 +3365,6 @@ } // _LOAD_ATTR { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ @@ -3486,7 +3415,6 @@ owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); @@ -3517,7 +3445,6 @@ uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -3561,7 +3488,6 @@ } // _CHECK_MANAGED_OBJECT_HAS_VALUES { - assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); @@ -3604,7 +3530,6 @@ } // _CHECK_ATTR_METHOD_LAZY_DICT { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); @@ -3649,7 +3574,6 @@ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3683,7 +3607,6 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); @@ -3771,7 +3694,6 @@ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3802,7 +3724,6 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); @@ -3817,7 +3738,6 @@ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3881,7 +3801,6 @@ // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -3916,7 +3835,6 @@ } // _CHECK_ATTR_WITH_HINT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); @@ -3987,7 +3905,6 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; - PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -4045,7 +3962,6 @@ INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; - uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; value1 = GETLOCAL(oparg1); @@ -4065,7 +3981,6 @@ PyObject *class_dict; PyObject *value; class_dict = stack_pointer[-1]; - PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -4130,7 +4045,6 @@ // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -4145,7 +4059,6 @@ } // _LOAD_GLOBAL { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -4214,7 +4127,6 @@ // _LOAD_GLOBAL_BUILTINS { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -4249,7 +4161,6 @@ // _LOAD_GLOBAL_MODULE { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -4286,7 +4197,6 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); PyObject *v; - PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -4334,7 +4244,6 @@ global_super = stack_pointer[-3]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION int load_method = oparg & 1; @@ -4483,7 +4392,6 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); @@ -4526,7 +4434,6 @@ names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -4596,6 +4503,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); + DISPATCH(); } @@ -4617,7 +4525,6 @@ INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION @@ -4726,7 +4633,6 @@ PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -4759,7 +4665,6 @@ INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; args = &stack_pointer[-oparg]; - TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { @@ -4794,7 +4699,6 @@ PyObject **values; exc = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { @@ -4820,7 +4724,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); - assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } @@ -4876,14 +4779,12 @@ PyObject *retval; // _LOAD_CONST { - value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); } // _POP_FRAME retval = value; { - #if TIER_ONE assert(frame != &entry_frame); #endif @@ -4911,7 +4812,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4978,7 +4878,6 @@ receiver = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -5063,7 +4962,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -5110,7 +5008,6 @@ PyObject *attr; func = stack_pointer[-1]; attr = stack_pointer[-2]; - assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -5204,14 +5101,12 @@ owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_VALUES { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); @@ -5257,7 +5152,6 @@ value = stack_pointer[-2]; { uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -5385,7 +5279,6 @@ INSTRUCTION_STATS(STORE_GLOBAL); PyObject *v; v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -5461,7 +5354,6 @@ container = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); - TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -5498,7 +5390,6 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -5545,7 +5436,6 @@ PyObject *bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; @@ -5577,7 +5467,6 @@ } // _TO_BOOL { - int err = PyObject_IsTrue(value); Py_DECREF(value); if (err < 0) goto pop_1_error; @@ -5596,8 +5485,7 @@ PyObject *res; value = stack_pointer[-1]; uint32_t version = read_u32(&this_instr[2].cache); - - // This one is a bit weird, because we expect *some* failures: + // This one is a bit weird, because we expect *some* failures: assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -5665,7 +5553,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - // This one is a bit weird, because we expect *some* failures: + // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_False; @@ -5703,7 +5591,6 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -5744,7 +5631,6 @@ INSTRUCTION_STATS(UNPACK_EX); PyObject *seq; seq = stack_pointer[-1]; - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -5780,7 +5666,6 @@ } // _UNPACK_SEQUENCE { - PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); @@ -5799,7 +5684,6 @@ PyObject **values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5864,7 +5748,6 @@ val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_func = stack_pointer[-4]; - /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -5922,5 +5805,4 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - #undef TIER_ONE diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index e5feee2f6da7ee..82b5a37e89fe8a 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -13,10 +13,11 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): self.column = 0 self.line_directives = line_directives - def set_position(self, tkn: Token): + def set_position(self, tkn: Token, emit_lines=True): if self.line < tkn.line: self.column = 0 - self.out.write("\n") + if emit_lines: + self.out.write("\n") if self.line_directives: self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') if self.column == 0: @@ -66,9 +67,8 @@ def emit_str(self, txt: str): self.line += 1 elif txt.endswith("{") or txt.startswith("}"): if self.column: - self.out.write("\n") self.column = 0 - self.line += 1 + self.line = 0 else: self.column += len(txt) self.maybe_indent(txt) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 3259bcce95b7c7..3e065a31dd355f 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -217,6 +217,7 @@ def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop #Move this to formatter def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None: tkn_iter = iter(uop.body[1:-1]) + out.set_position(uop.body[1], False) for tkn in tkn_iter: if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: out.set_position(tkn) @@ -237,7 +238,7 @@ def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruct for var in reversed(uop.stack.inputs): out.emit(stack.pop(var)) if braces: - out.emit("{") + out.emit("{\n") if not uop.properties.stores_sp: for i, var in enumerate(uop.stack.outputs): out.emit(stack.push(var)) @@ -257,7 +258,7 @@ def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruct out.emit(stack.push(var)) if braces: out.start_line() - out.emit("}") + out.emit("}\n") # out.emit(stack.as_comment() + "\n") return offset except SizeMismatch as ex: @@ -280,7 +281,8 @@ def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO, lines: b out.emit("\n") for name, inst in sorted(analysis.instructions.items()): needs_this = uses_this(inst) - out.emit(f"TARGET({name}) {{") + out.emit("\n") + out.emit(f"TARGET({name}) {{\n") if needs_this and not inst.is_target: out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") else: From ae2baead6ea7574b6d3a578a746375286b3f05c5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 05:58:36 +0000 Subject: [PATCH 17/40] Remove extra blank line --- Python/bytecodes.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c1723aae2a50da..71d7bbf6f4c1b9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1165,7 +1165,6 @@ dummy_func( } } - inst(STORE_NAME, (v -- )) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); From 92c94401e8ae79390ac88cc912e896b3bca04982 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 09:04:13 +0000 Subject: [PATCH 18/40] Attempt to placate MyPy --- Tools/cases_generator/analyzer.py | 67 ++++++++++++------------ Tools/cases_generator/cwriter.py | 16 +++--- Tools/cases_generator/lexer.py | 2 +- Tools/cases_generator/parser.py | 3 +- Tools/cases_generator/stack.py | 14 ++--- Tools/cases_generator/tier1_generator.py | 32 +++++------ Tools/cases_generator/tier2_generator.py | 22 ++++---- 7 files changed, 80 insertions(+), 76 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b7cf9ab51ead5a..bf385ad23d576a 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -2,6 +2,7 @@ from dataclasses import dataclass import lexer import parser +from typing import Optional @dataclass class Properties: @@ -15,7 +16,7 @@ class Properties: always_exits: bool stores_sp: bool - def dump(self, indent): + def dump(self, indent:str) -> None: print(indent, "escapes:", self.escapes, ", infallible:", self.infallible, @@ -48,11 +49,11 @@ class Skip: size: int @property - def name(self): + def name(self)->str: return f"unused/{self.size}" @property - def properties(self): + def properties(self)->Properties: return SKIP_PROPERTIES @dataclass @@ -63,13 +64,13 @@ class StackItem: size: str peek: bool = False - def __str__(self): + def __str__(self) -> str: cond = " if (" + self.condition + ")" if self.condition else "" size = "[" + self.size + "]" if self.size != "1" else "" type = "" if self.type is None else self.type + " " return f"{type}{self.name}{size}{cond} {self.peek}" - def is_array(self): + def is_array(self) -> bool: return self.type == "PyObject **" @dataclass @@ -77,7 +78,7 @@ class StackEffect: inputs: list[StackItem] outputs: list[StackItem] - def __str__(self): + def __str__(self) -> str: return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" @dataclass @@ -85,7 +86,7 @@ class CacheEntry: name: str size: int - def __str__(self): + def __str__(self) -> str: return f"{self.name}/{self.size}" @dataclass @@ -99,13 +100,13 @@ class Uop: properties: Properties _size: int = -1 - def dump(self, indent): + def dump(self, indent: str) -> None: print(indent, self.name, ", ".join(self.annotations) if self.annotations else "") print(indent, self.stack, ", ".join([str(c) for c in self.caches])) self.properties.dump(" " + indent) @property - def size(self): + def size(self) -> int: if self._size < 0: self._size = sum(c.size for c in self.caches) return self._size @@ -118,7 +119,7 @@ class Instruction: uops: list[Part] _properties : Properties | None is_target: bool = False - family: "Family" = None + family: Optional["Family"] = None @property def properties(self) -> Properties: @@ -139,12 +140,12 @@ def _compute_properties(self) -> Properties: any(part.properties.stores_sp for part in self.uops), ) - def dump(self, indent): + def dump(self, indent: str) -> None: print(indent, self.name, "=", ", ".join([op.name for op in self.uops])) self.properties.dump(" " + indent) @property - def size(self): + def size(self) -> int: return 1 + sum(uop.size for uop in self.uops) @dataclass @@ -152,7 +153,7 @@ class PseudoInstruction: name: str targets: list[Instruction] - def dump(self, indent): + def dump(self, indent: str) -> None: print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) @@ -162,7 +163,7 @@ class Family: size: str members: list[Instruction] - def dump(self, indent): + def dump(self, indent: str) -> None: print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) @@ -181,7 +182,7 @@ def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: message, "", tkn.line, tkn.column, "" ) -def override_error(name, context, prev_context, token) -> SyntaxError: +def override_error(name: str, context: parser.Context, prev_context: parser.Context, token: lexer.Token) -> SyntaxError: return analysis_error( f"Duplicate definition of '{name}' @ {context} " f"previous definition @ {prev_context}", @@ -232,10 +233,10 @@ def is_infallible(op: parser.InstDef) -> bool: "DISPATCH_GOTO", ]) -def eval_breaker_at_end(op: parser.InstDef): +def eval_breaker_at_end(op: parser.InstDef) -> bool: return op.tokens[-5].text == "CHECK_EVAL_BREAKER" -def always_exits(op: parser.InstDef): +def always_exits(op: parser.InstDef) -> bool: depth = 0 for tkn in op.tokens: if tkn.kind == "LBRACE": @@ -263,7 +264,7 @@ def compute_properties(op: parser.InstDef) -> Properties: variable_used(op, "STORE_SP"), ) -def make_uop(name, op: parser.InstDef) -> Uop: +def make_uop(name: str, op: parser.InstDef) -> Uop: return Uop( name, op.context, @@ -281,23 +282,23 @@ def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: raise override_error(op.name, op.context, uops[op.name].context,op.tokens[0]) uops[op.name] = make_uop(op.name, op) -def add_instruction(name: str, parts: list[Part], instructions: dict[str, Instruction]): +def add_instruction(name: str, parts: list[Part], instructions: dict[str, Instruction]) -> None: instructions[name] = Instruction(name, parts, None) -def desugar_inst(inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop]): +def desugar_inst(inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop]) -> None: assert inst.kind == "inst" name = inst.name uop = make_uop("_" + inst.name, inst) uops[inst.name] = uop add_instruction(name, [uop], instructions) -def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop]): - parts = [] +def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop]) -> None: + parts: list[Uop | Skip] = [] for part in macro.uops: match part: case parser.OpName(): if part.name not in uops: - analysis_error(f"No Uop named {part.name}", macro.context, macro.tokens[0]) + analysis_error(f"No Uop named {part.name}", macro.tokens[0]) parts.append(uops[part.name]) case parser.CacheEffect(): parts.append(Skip(part.size)) @@ -306,7 +307,7 @@ def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: d assert(parts) add_instruction(macro.name, parts, instructions) -def add_family(family: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]): +def add_family(family: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]) -> None: family = Family( family.name, family.size, @@ -318,7 +319,7 @@ def add_family(family: parser.Family, instructions: dict[str, Instruction], fami instructions[family.name].is_target = True families[family.name] = family -def add_pseudo(pseudo: parser.Pseudo, instructions: dict[str, Instruction], pseudos: dict[str, PseudoInstruction]): +def add_pseudo(pseudo: parser.Pseudo, instructions: dict[str, Instruction], pseudos: dict[str, PseudoInstruction]) -> None: pseudos[pseudo.name] = PseudoInstruction( pseudo.name, [ instructions[target] for target in pseudo.targets ] @@ -376,17 +377,17 @@ def analyze_files(filenames: list[str]) -> Analysis: def dump_analysis(analysis: Analysis) -> None: print("Uops:") - for i in analysis.uops.values(): - i.dump(" ") + for u in analysis.uops.values(): + u.dump(" ") print("Instructions:") for i in analysis.instructions.values(): i.dump(" ") print("Families:") - for i in analysis.families.values(): - i.dump(" ") + for f in analysis.families.values(): + f.dump(" ") print("Pseudos:") - for i in analysis.pseudos.values(): - i.dump(" ") + for p in analysis.pseudos.values(): + p.dump(" ") if __name__ == "__main__": @@ -394,8 +395,8 @@ def dump_analysis(analysis: Analysis) -> None: if len(sys.argv) < 2: print("No input") else: - filename = sys.argv[1] - dump_analysis(analyze_file(filename)) + filenames = sys.argv[1:] + dump_analysis(analyze_files(filenames)) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 82b5a37e89fe8a..6d0e8f17c8bd90 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -13,7 +13,7 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): self.column = 0 self.line_directives = line_directives - def set_position(self, tkn: Token, emit_lines=True): + def set_position(self, tkn: Token, emit_lines: bool = True) -> None: if self.line < tkn.line: self.column = 0 if emit_lines: @@ -30,14 +30,14 @@ def set_position(self, tkn: Token, emit_lines=True): self.line = tkn.end_line self.column = tkn.end_column - 1 - def maybe_dedent(self, txt: str): + def maybe_dedent(self, txt: str) -> None: parens = txt.count("(") - txt.count(")") if parens < 0: self.indents.pop() elif "}" in txt or txt.endswith(":"): self.indents.pop() - def maybe_indent(self, txt: str): + def maybe_indent(self, txt: str) -> None: parens = txt.count("(") - txt.count(")") if parens > 0: offset = self.column @@ -47,19 +47,19 @@ def maybe_indent(self, txt: str): elif "{" in txt or txt.endswith(":"): self.indents.append(self.indents[-1] + 4) - def emit_token(self, tkn: Token): + def emit_token(self, tkn: Token) -> None: self.maybe_dedent(tkn.text) self.set_position(tkn) self.emit_text(tkn.text) self.maybe_indent(tkn.text) - def emit_text(self, txt: str): + def emit_text(self, txt: str) -> None: if self.column == 0 and txt.strip(): self.out.write(" " * self.indents[-1]) self.column = self.base_column self.out.write(txt) - def emit_str(self, txt: str): + def emit_str(self, txt: str) -> None: self.maybe_dedent(txt) self.emit_text(txt) if txt.endswith("\n"): @@ -73,7 +73,7 @@ def emit_str(self, txt: str): self.column += len(txt) self.maybe_indent(txt) - def emit(self, txt: str | Token): + def emit(self, txt: str | Token) -> None: if isinstance(txt, Token): self.emit_token(txt) elif isinstance(txt, str): @@ -81,7 +81,7 @@ def emit(self, txt: str | Token): else: assert False - def start_line(self): + def start_line(self) -> None: if self.column: self.out.write("\n") self.column = 0 diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 068ac909656ed8..c3c2954a42083f 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -273,7 +273,7 @@ def __repr__(self) -> str: return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: +def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 1b07d756c11993..be2dd23a2be037 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -7,7 +7,7 @@ AstNode = InstDef | Macro | Pseudo | Family -def parse_files(filenames: list[str]) -> None: +def parse_files(filenames: list[str]) -> list[AstNode]: result: list[AstNode] = [] for filename in filenames: with open(filename) as file: @@ -35,6 +35,7 @@ def parse_files(filenames: list[str]) -> None: psr.setpos(start) thing_first_token = psr.peek() while node := psr.definition(): + assert node is not None result.append(node) if not psr.eof(): psr.backup() diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 1bbf629627af33..778f6c14ce28ee 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from formatting import maybe_parenthesize -def var_size(var): +def var_size(var: StackItem) -> str: if var.condition: # Special case simplification if var.condition == "oparg & 1" and var.size == "1": @@ -17,17 +17,17 @@ def var_size(var): class StackOffset: "The stack offset of the virtual base of the stack from the physical stack pointer" - def __init__(self): + def __init__(self) -> None: self.popped: list[str] = [] self.pushed: list[str] = [] - def pop(self, item: StackItem): + def pop(self, item: StackItem) -> None: self.popped.append(var_size(item)) - def push(self, item: StackItem): + def push(self, item: StackItem) -> None: self.pushed.append(var_size(item)) - def simplify(self): + def simplify(self) -> None: if not self.popped or not self.pushed: return popped = sorted(self.popped) @@ -48,7 +48,7 @@ def simplify(self): self.popped.extend(popped) self.pushed.extend(pushed) - def to_c(self): + def to_c(self) -> str: self.simplify() int_offset = 0 symbol_offset = "" @@ -72,6 +72,6 @@ def to_c(self): res = "-" + res[3:] return res - def clear(self): + def clear(self) -> None: self.popped = [] self.pushed = [] diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 3e065a31dd355f..50c82377338e2a 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -13,6 +13,7 @@ from lexer import Token from stack import StackOffset + HERE = os.path.dirname(__file__) ROOT = os.path.join(HERE, "../..") THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/") @@ -40,7 +41,7 @@ class SizeMismatch(Exception): class Stack: - def __init__(self): + def __init__(self) -> None: self.top_offset: StackOffset = StackOffset() self.base_offset: StackOffset = StackOffset() self.peek_offset: StackOffset = StackOffset() @@ -77,7 +78,7 @@ def pop(self, var: StackItem) -> str: return f"if ({var.condition}) {{ {assign} }}\n" return f"{assign}\n" - def push(self, var: StackItem) -> None: + def push(self, var: StackItem) -> str: self.variables.append(var) if var.is_array() and var.name not in self.defined and var.name != "unused": c_offset = self.top_offset.to_c() @@ -107,7 +108,7 @@ def flush(self, out: CWriter) -> None: self.top_offset.clear() self.peek_offset.clear() - def as_comment(self): + def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" @@ -132,7 +133,7 @@ def declare_variables(inst: Instruction, out: CWriter)->None: else: out.emit(f"{type}{var.name};\n") -def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens = 0 for tkn in tkn_iter: if tkn.kind == end and parens == 0: @@ -143,16 +144,17 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): parens -= 1 out.emit(tkn) -def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack, inst: Instruction): +def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack, inst: Instruction) -> None: out.emit("DEOPT_IF") out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon out.emit(", ") + assert inst.family is not None out.emit(inst.family.name) out.emit(");") -def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): +def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: out.emit("if ") out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "COMMA") @@ -174,7 +176,7 @@ def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stac out.emit(label) out.emit(close) -def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): +def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -190,14 +192,14 @@ def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: St else: out.emit(f"Py_DECREF({var.name});\n") -def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): +def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) stack.flush(out) out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") -def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction): +def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -226,12 +228,12 @@ def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None out.emit(tkn) def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces:bool) -> int: + # out.emit(stack.as_comment() + "\n") + if isinstance(uop, Skip): + entries = "entries" if uop.size > 1 else "entry" + out.emit(f"/* Skip {uop.size} cache {entries} */\n") + return offset + uop.size try: - # out.emit(stack.as_comment() + "\n") - if isinstance(uop, Skip): - entries = "entries" if uop.size > 1 else "entry" - out.emit(f"/* Skip {uop.size} cache {entries} */\n") - return offset + uop.size out.start_line() if braces: out.emit(f"// {uop.name}\n") @@ -262,7 +264,7 @@ def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruct # out.emit(stack.as_comment() + "\n") return offset except SizeMismatch as ex: - raise analysis_error(ex, uop.body[0]) + raise analysis_error(ex.args[0], uop.body[0]) def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 3f919a483d174d..893995b9615665 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -40,7 +40,7 @@ class SizeMismatch(Exception): class Stack: - def __init__(self): + def __init__(self) -> None: self.top_offset: StackOffset = StackOffset() self.base_offset: StackOffset = StackOffset() self.peek_offset: StackOffset = StackOffset() @@ -77,7 +77,7 @@ def pop(self, var: StackItem) -> str: return f"if ({var.condition}) {{ {assign} }}\n" return f"{assign}\n" - def push(self, var: StackItem) -> None: + def push(self, var: StackItem) -> str: self.variables.append(var) if var.is_array() and var.name not in self.defined and var.name != "unused": c_offset = self.top_offset.to_c() @@ -107,7 +107,7 @@ def flush(self, out: CWriter) -> None: self.top_offset.clear() self.peek_offset.clear() - def as_comment(self): + def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" @@ -130,7 +130,7 @@ def declare_variables(uop: Uop, out: CWriter)->None: else: out.emit(f"{type}{var.name};\n") -def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): +def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens = 0 for tkn in tkn_iter: if tkn.kind == end and parens == 0: @@ -141,14 +141,14 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str): parens -= 1 out.emit(tkn) -def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack): +def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack) -> None: out.emit("DEOPT_IF") out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon out.emit(");") -def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): +def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: out.emit("if ") out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "COMMA") @@ -170,7 +170,7 @@ def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stac out.emit(label) out.emit(close) -def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): +def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -186,14 +186,14 @@ def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: St else: out.emit(f"Py_DECREF({var.name});\n") -def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): +def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) stack.flush(out) out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") -def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack): +def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -220,7 +220,7 @@ def emit_tokens(out: CWriter, uop: Uop, stack: Stack) -> None: else: out.emit(tkn) -def write_uop(uop: Uop, out: CWriter, stack: Stack) -> int: +def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: try: # out.emit(stack.as_comment() + "\n") out.start_line() @@ -244,7 +244,7 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> int: out.emit(stack.push(var)) # out.emit(stack.as_comment() + "\n") except SizeMismatch as ex: - raise analysis_error(ex, uop.body[0]) + raise analysis_error(ex.args[0], uop.body[0]) def uses_this(uop: Uop) -> bool: if uop.properties.needs_this: From f9e41c63f93d46a73c3ff7e7514d876776520a41 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 09:10:58 +0000 Subject: [PATCH 19/40] Annotate false positives --- Tools/cases_generator/analyzer.py | 10 +++++----- Tools/cases_generator/parser.py | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index bf385ad23d576a..dc1f99769a629c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -92,7 +92,7 @@ def __str__(self) -> str: @dataclass class Uop: name: str - context: parser.Context + context: parser.Context # type: ignore annotations: list[str] stack: StackEffect caches: list[CacheEntry] @@ -195,7 +195,7 @@ def convert_stack_item(item: parser.StackEffect) -> StackItem: def analyze_stack(op: parser.InstDef) -> StackEffect: - inputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] + inputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] # type: ignore outputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.outputs] for (input, output) in zip(inputs, outputs): if input.name == output.name: @@ -204,7 +204,7 @@ def analyze_stack(op: parser.InstDef) -> StackEffect: def analyze_caches(op: parser.InstDef) -> list[CacheEntry]: - caches: list[parser.CacheEffect] = [i for i in op.inputs if isinstance(i, parser.CacheEffect)] + caches: list[parser.CacheEffect] = [i for i in op.inputs if isinstance(i, parser.CacheEffect)] # type: ignore return [ CacheEntry(i.name, int(i.size)) for i in caches ] @@ -234,7 +234,7 @@ def is_infallible(op: parser.InstDef) -> bool: ]) def eval_breaker_at_end(op: parser.InstDef) -> bool: - return op.tokens[-5].text == "CHECK_EVAL_BREAKER" + return op.tokens[-5].text == "CHECK_EVAL_BREAKER" # type: ignore def always_exits(op: parser.InstDef) -> bool: depth = 0 @@ -347,7 +347,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: case _: assert False for node in forest: - if isinstance(node, parser.Macro): + if isinstance(node, parser.Macro): # type: ignore add_macro(node, instructions, uops) for node in forest: match node: diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index be2dd23a2be037..7a48e666db306d 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -7,6 +7,8 @@ AstNode = InstDef | Macro | Pseudo | Family +Context = Context + def parse_files(filenames: list[str]) -> list[AstNode]: result: list[AstNode] = [] for filename in filenames: @@ -35,8 +37,8 @@ def parse_files(filenames: list[str]) -> list[AstNode]: psr.setpos(start) thing_first_token = psr.peek() while node := psr.definition(): - assert node is not None - result.append(node) + assert node is not None + result.append(node) # type: ignore if not psr.eof(): psr.backup() raise psr.make_syntax_error(f"Extra stuff at the end of {filename}", psr.next(True)) From ab2dbb39a876c0bcaf5152fb48ec7280b503e47d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 11:21:54 +0000 Subject: [PATCH 20/40] Further attempts to make MyPy happy --- Tools/cases_generator/analyzer.py | 10 +++++----- Tools/cases_generator/mypy.ini | 2 +- Tools/cases_generator/parser.py | 10 ++++++---- Tools/cases_generator/parsing.py | 3 ++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index dc1f99769a629c..2041ed1ac7e1a2 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -92,7 +92,7 @@ def __str__(self) -> str: @dataclass class Uop: name: str - context: parser.Context # type: ignore + context: parser.Context | None annotations: list[str] stack: StackEffect caches: list[CacheEntry] @@ -182,7 +182,7 @@ def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: message, "", tkn.line, tkn.column, "" ) -def override_error(name: str, context: parser.Context, prev_context: parser.Context, token: lexer.Token) -> SyntaxError: +def override_error(name: str, context: parser.Context | None, prev_context: parser.Context | None, token: lexer.Token) -> SyntaxError: return analysis_error( f"Duplicate definition of '{name}' @ {context} " f"previous definition @ {prev_context}", @@ -195,7 +195,7 @@ def convert_stack_item(item: parser.StackEffect) -> StackItem: def analyze_stack(op: parser.InstDef) -> StackEffect: - inputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] # type: ignore + inputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] outputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.outputs] for (input, output) in zip(inputs, outputs): if input.name == output.name: @@ -204,7 +204,7 @@ def analyze_stack(op: parser.InstDef) -> StackEffect: def analyze_caches(op: parser.InstDef) -> list[CacheEntry]: - caches: list[parser.CacheEffect] = [i for i in op.inputs if isinstance(i, parser.CacheEffect)] # type: ignore + caches: list[parser.CacheEffect] = [i for i in op.inputs if isinstance(i, parser.CacheEffect)] return [ CacheEntry(i.name, int(i.size)) for i in caches ] @@ -347,7 +347,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: case _: assert False for node in forest: - if isinstance(node, parser.Macro): # type: ignore + if isinstance(node, parser.Macro): add_macro(node, instructions, uops) for node in forest: match node: diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index e7175e263350b2..04b657016041fc 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -9,5 +9,5 @@ python_version = 3.10 # ...And be strict: strict = True strict_concatenate = True -enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined +enable_error_code = redundant-expr,truthy-bool warn_unreachable = True diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 7a48e666db306d..5643619be12f2e 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -1,11 +1,10 @@ -from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName +from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName, AstNode from formatting import prettify_filename BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" -AstNode = InstDef | Macro | Pseudo | Family Context = Context @@ -37,9 +36,12 @@ def parse_files(filenames: list[str]) -> list[AstNode]: psr.setpos(start) thing_first_token = psr.peek() while node := psr.definition(): - assert node is not None - result.append(node) # type: ignore + assert node is not None + result.append(node) #type: ignore[arg-type] if not psr.eof(): psr.backup() raise psr.make_syntax_error(f"Extra stuff at the end of {filename}", psr.next(True)) return result + +#Keep MyPy happy +__all__ = [ "parse_files", "Context", "Macro", "InstDef", "Pseudo", "StackEffect", "CacheEffect" ] \ No newline at end of file diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index d36bd52b022ea9..3d4bd038b280e6 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -140,10 +140,11 @@ class Pseudo(Node): name: str targets: list[str] # opcodes this can be replaced by +AstNode = InstDef | Macro | Pseudo | Family class Parser(PLexer): @contextual - def definition(self) -> InstDef | Macro | Pseudo | Family | None: + def definition(self) -> AstNode | None: if macro := self.macro_def(): return macro if family := self.family_def(): From 21a018b5edaa8ee129d79237594e9c7f5f445a28 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 11:24:34 +0000 Subject: [PATCH 21/40] Remove white space --- Tools/cases_generator/analyzer.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 2041ed1ac7e1a2..a8ec87138924e8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -397,16 +397,3 @@ def dump_analysis(analysis: Analysis) -> None: else: filenames = sys.argv[1:] dump_analysis(analyze_files(filenames)) - - - - - - - - - - - - - From 8078302e25e7d68339d74f498cb51923b236b016 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 19:53:38 +0000 Subject: [PATCH 22/40] Fix newline and indentation handling --- Python/generated_cases.c.h | 4 -- Tools/cases_generator/cwriter.py | 60 +++++++++++------------- Tools/cases_generator/tier1_generator.py | 5 +- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d16f83bb0d9580..a6e3dad41deeb2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1876,10 +1876,8 @@ Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; - assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error; - if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -2656,7 +2654,6 @@ iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2732,7 +2729,6 @@ _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 6d0e8f17c8bd90..7ab8c96263a91a 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -9,26 +9,24 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): self.out = out self.base_column = indent * 4 self.indents = [ i*4 for i in range(indent+1) ] - self.line = -1 - self.column = 0 self.line_directives = line_directives + self.last_token = None + self.newline = True def set_position(self, tkn: Token, emit_lines: bool = True) -> None: - if self.line < tkn.line: - self.column = 0 - if emit_lines: + if self.last_token is not None: + if self.last_token.line < tkn.line: self.out.write("\n") - if self.line_directives: - self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') - if self.column == 0: + if self.line_directives: + self.out.write(f'#line {tkn.line} "{tkn.filename}"\n') + self.out.write(" " * self.indents[-1]) + else: + gap = tkn.column - self.last_token.end_column + self.out.write(" " * gap) + elif self.newline: self.out.write(" " * self.indents[-1]) - else: - #Token colums are 1 based - column = tkn.column - 1 - if self.column < column: - self.out.write(" " * (column-self.column)) - self.line = tkn.end_line - self.column = tkn.end_column - 1 + self.last_token = tkn + self.newline = False def maybe_dedent(self, txt: str) -> None: parens = txt.count("(") - txt.count(")") @@ -39,39 +37,34 @@ def maybe_dedent(self, txt: str) -> None: def maybe_indent(self, txt: str) -> None: parens = txt.count("(") - txt.count(")") - if parens > 0: - offset = self.column + if parens > 0 and self.last_token: + offset = self.last_token.end_column - 1 if offset <= self.indents[-1] or offset > 40: offset = self.indents[-1] + 4 self.indents.append(offset) elif "{" in txt or txt.endswith(":"): self.indents.append(self.indents[-1] + 4) + def emit_text(self, txt: str) -> None: + self.out.write(txt) + def emit_token(self, tkn: Token) -> None: self.maybe_dedent(tkn.text) self.set_position(tkn) self.emit_text(tkn.text) self.maybe_indent(tkn.text) - def emit_text(self, txt: str) -> None: - if self.column == 0 and txt.strip(): - self.out.write(" " * self.indents[-1]) - self.column = self.base_column - self.out.write(txt) - def emit_str(self, txt: str) -> None: self.maybe_dedent(txt) + if self.newline: + if txt != "\n": + self.out.write(" " * self.indents[-1]) + self.newline = False self.emit_text(txt) if txt.endswith("\n"): - self.column = 0 - self.line += 1 - elif txt.endswith("{") or txt.startswith("}"): - if self.column: - self.column = 0 - self.line = 0 - else: - self.column += len(txt) + self.newline = True self.maybe_indent(txt) + self.last_token = None def emit(self, txt: str | Token) -> None: if isinstance(txt, Token): @@ -82,6 +75,7 @@ def emit(self, txt: str | Token) -> None: assert False def start_line(self) -> None: - if self.column: + if not self.newline: self.out.write("\n") - self.column = 0 + self.newline = True + self.last_token = None diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 50c82377338e2a..0090131c9a0bd5 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -152,7 +152,7 @@ def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Sta out.emit(", ") assert inst.family is not None out.emit(inst.family.name) - out.emit(");") + out.emit(");\n") def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: out.emit("if ") @@ -204,7 +204,7 @@ def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop next(tkn_iter) next(tkn_iter) if not uop.properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();") + out.emit("CHECK_EVAL_BREAKER();\n") @@ -311,6 +311,7 @@ def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO, lines: b if inst.uops[-1].properties.ends_with_eval_breaker: out.emit("CHECK_EVAL_BREAKER();\n") out.emit("DISPATCH();\n") + out.start_line() out.emit("}") out.emit("\n") outfile.write(FOOTER) From d5fcbed2a34628cb6b9d833115ab1a5c6418f340 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 20:20:11 +0000 Subject: [PATCH 23/40] Fix newline and indentation handling --- Python/generated_cases.c.h | 1 - Tools/cases_generator/cwriter.py | 2 +- Tools/cases_generator/tier1_generator.py | 7 +++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a6e3dad41deeb2..de6db4af2a1f66 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4499,7 +4499,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); - DISPATCH(); } diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 7ab8c96263a91a..d5048854b1c858 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -13,7 +13,7 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): self.last_token = None self.newline = True - def set_position(self, tkn: Token, emit_lines: bool = True) -> None: + def set_position(self, tkn: Token) -> None: if self.last_token is not None: if self.last_token.line < tkn.line: self.out.write("\n") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 0090131c9a0bd5..56e70385730bd1 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -218,8 +218,11 @@ def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop #Move this to formatter def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None: - tkn_iter = iter(uop.body[1:-1]) - out.set_position(uop.body[1], False) + tkns = uop.body[1:-1] + if not tkns: + return + tkn_iter = iter(tkns) + out.set_position(tkns[0]) for tkn in tkn_iter: if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: out.set_position(tkn) From 6c1ed31af1f270cb3cdb67d9e49cf285823ace22 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 20:23:06 +0000 Subject: [PATCH 24/40] A couple more type annotations --- Tools/cases_generator/cwriter.py | 2 ++ Tools/cases_generator/parser.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index d5048854b1c858..21e80461469421 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -5,6 +5,8 @@ class CWriter: 'A writer that understands tokens and how to format C code' + last_token : Token | None + def __init__(self, out: TextIO, indent: int, line_directives: bool): self.out = out self.base_column = indent * 4 diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 5643619be12f2e..b0e579509336a3 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -44,4 +44,4 @@ def parse_files(filenames: list[str]) -> list[AstNode]: return result #Keep MyPy happy -__all__ = [ "parse_files", "Context", "Macro", "InstDef", "Pseudo", "StackEffect", "CacheEffect" ] \ No newline at end of file +__all__ = [ "parse_files", "Context", "Macro", "InstDef", "Pseudo", "StackEffect", "CacheEffect", "StackItem" ] From fc0f6edaffa34aad13f76d20bb9e884357d8b646 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 20:41:25 +0000 Subject: [PATCH 25/40] Fix alignment of comments --- Python/generated_cases.c.h | 14 +++++++------- Tools/cases_generator/cwriter.py | 13 +++++++++---- Tools/cases_generator/tier1_generator.py | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index de6db4af2a1f66..75abae977fb5f6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -542,7 +542,7 @@ DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); - // Specialize for reading an ASCII character from any string: + // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -3126,7 +3126,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: + // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); @@ -3150,7 +3150,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: + // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); @@ -3235,13 +3235,13 @@ * do the 2s complement comparison manually */ uint16_t ucounter = this_instr[1].cache + (1 << 15); uint16_t threshold = tstate->interp->optimizer_backedge_threshold + (1 << 15); - // Double-check that the opcode isn't instrumented or something: + // Double-check that the opcode isn't instrumented or something: if (ucounter > threshold && this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { - // Rewind and enter the executor: + // Rewind and enter the executor: assert(this_instr->op.code == ENTER_EXECUTOR); next_instr = this_instr; this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); @@ -4786,7 +4786,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: + // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); @@ -4844,7 +4844,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); - // GH-99729: We need to unlink the frame *before* clearing it: + // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 21e80461469421..baa254e0615c36 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -2,6 +2,7 @@ from lexer import Token from typing import TextIO + class CWriter: 'A writer that understands tokens and how to format C code' @@ -34,7 +35,7 @@ def maybe_dedent(self, txt: str) -> None: parens = txt.count("(") - txt.count(")") if parens < 0: self.indents.pop() - elif "}" in txt or txt.endswith(":"): + elif "}" in txt or is_label(txt): self.indents.pop() def maybe_indent(self, txt: str) -> None: @@ -44,7 +45,7 @@ def maybe_indent(self, txt: str) -> None: if offset <= self.indents[-1] or offset > 40: offset = self.indents[-1] + 4 self.indents.append(offset) - elif "{" in txt or txt.endswith(":"): + elif "{" in txt or is_label(txt): self.indents.append(self.indents[-1] + 4) def emit_text(self, txt: str) -> None: @@ -58,8 +59,8 @@ def emit_token(self, tkn: Token) -> None: def emit_str(self, txt: str) -> None: self.maybe_dedent(txt) - if self.newline: - if txt != "\n": + if self.newline and txt: + if txt[0] != "\n": self.out.write(" " * self.indents[-1]) self.newline = False self.emit_text(txt) @@ -81,3 +82,7 @@ def start_line(self) -> None: self.out.write("\n") self.newline = True self.last_token = None + + +def is_label(txt: str) -> bool: + return not txt.startswith("//") and txt.endswith(":") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 56e70385730bd1..60dfa1b8f5198d 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -222,7 +222,7 @@ def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None if not tkns: return tkn_iter = iter(tkns) - out.set_position(tkns[0]) + out.start_line() for tkn in tkn_iter: if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: out.set_position(tkn) From 39ebd6cd108091e6557848201eb7044d38576c43 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:03:21 +0000 Subject: [PATCH 26/40] Avoid emitting blank lines --- Python/generated_cases.c.h | 13 ------------- Tools/cases_generator/cwriter.py | 4 ++++ Tools/cases_generator/tier1_generator.py | 21 +++++++++++---------- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 75abae977fb5f6..bbea83d48da814 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -823,7 +823,6 @@ Py_DECREF(args[i]); } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1019,7 +1018,6 @@ } Py_DECREF(tp); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1065,7 +1063,6 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1105,7 +1102,6 @@ } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1147,7 +1143,6 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1233,7 +1228,6 @@ Py_XDECREF(kwargs); assert(PEEK(2 + (oparg & 1)) == NULL); if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } - stack_pointer[-3 - ((oparg & 1))] = result; stack_pointer += -2 - ((oparg & 1)); CHECK_EVAL_BREAKER(); @@ -1397,7 +1391,6 @@ Py_DECREF(args[i]); } if (res == NULL) { stack_pointer += -3 - oparg; goto error; } - stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; CHECK_EVAL_BREAKER(); @@ -1506,7 +1499,6 @@ } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1549,7 +1541,6 @@ } Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1594,7 +1585,6 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1640,7 +1630,6 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1794,7 +1783,6 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -1822,7 +1810,6 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { stack_pointer += -2 - oparg; goto error; } - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index baa254e0615c36..c5816bf24334bf 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -31,6 +31,10 @@ def set_position(self, tkn: Token) -> None: self.last_token = tkn self.newline = False + def emit_at(self, txt: str, where: Token) -> None: + self.set_position(where) + self.out.write(txt) + def maybe_dedent(self, txt: str) -> None: parens = txt.count("(") - txt.count(")") if parens < 0: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 60dfa1b8f5198d..af90f854a43eb1 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -144,8 +144,8 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens -= 1 out.emit(tkn) -def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack, inst: Instruction) -> None: - out.emit("DEOPT_IF") +def replace_deopt(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, unused: Stack, inst: Instruction) -> None: + out.emit_at("DEOPT_IF", tkn) out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon @@ -154,8 +154,8 @@ def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Sta out.emit(inst.family.name) out.emit(");\n") -def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: - out.emit("if ") +def replace_error(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: + out.emit_at("if ", tkn) out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "COMMA") label = next(tkn_iter).text @@ -176,10 +176,11 @@ def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stac out.emit(label) out.emit(close) -def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: +def replace_decrefs(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) + out.emit_at("", tkn) for var in uop.stack.inputs: if var.name == "unused" or var.name == "null" or var.peek: continue @@ -192,19 +193,20 @@ def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: St else: out.emit(f"Py_DECREF({var.name});\n") -def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: +def replace_store_sp(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) + out.emit_at("", tkn) stack.flush(out) out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") -def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: +def replace_check_eval_breaker(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) if not uop.properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();\n") + out.emit_at("CHECK_EVAL_BREAKER();", tkn) @@ -225,8 +227,7 @@ def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None out.start_line() for tkn in tkn_iter: if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: - out.set_position(tkn) - REPLACEMENT_FUNCTIONS[tkn.text](out, tkn_iter, uop, stack, inst) + REPLACEMENT_FUNCTIONS[tkn.text](out, tkn, tkn_iter, uop, stack, inst) else: out.emit(tkn) From 7b210253d0777524c88b588587a763763ae7c2a5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:13:29 +0000 Subject: [PATCH 27/40] Further changes to keep MyPy happy --- Tools/cases_generator/analyzer.py | 10 +++++----- Tools/cases_generator/mypy.ini | 2 +- Tools/cases_generator/parser.py | 6 +++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index a8ec87138924e8..5f4e1ca1a51e6b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -234,7 +234,7 @@ def is_infallible(op: parser.InstDef) -> bool: ]) def eval_breaker_at_end(op: parser.InstDef) -> bool: - return op.tokens[-5].text == "CHECK_EVAL_BREAKER" # type: ignore + return op.tokens[-5].text == "CHECK_EVAL_BREAKER" def always_exits(op: parser.InstDef) -> bool: depth = 0 @@ -307,11 +307,11 @@ def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: d assert(parts) add_instruction(macro.name, parts, instructions) -def add_family(family: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]) -> None: +def add_family(pfamily: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]) -> None: family = Family( - family.name, - family.size, - [ instructions[member] for member in family.members ] + pfamily.name, + pfamily.size, + [ instructions[member_name] for member_name in pfamily.members ] ) for member in family.members: member.family = family diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index 04b657016041fc..e7175e263350b2 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -9,5 +9,5 @@ python_version = 3.10 # ...And be strict: strict = True strict_concatenate = True -enable_error_code = redundant-expr,truthy-bool +enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index b0e579509336a3..e6253d3d13bec1 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -44,4 +44,8 @@ def parse_files(filenames: list[str]) -> list[AstNode]: return result #Keep MyPy happy -__all__ = [ "parse_files", "Context", "Macro", "InstDef", "Pseudo", "StackEffect", "CacheEffect", "StackItem" ] +__all__ = [ + "parse_files", "Context", "Macro", + "InstDef", "Family", "Pseudo", "AstNode", + "StackEffect", "CacheEffect", "StackItem" +] From 796dcf6bcd57b898d31924d011599fd7eee78eaf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:15:55 +0000 Subject: [PATCH 28/40] Fix type annotation --- Tools/cases_generator/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index e6253d3d13bec1..01762c394db143 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -1,4 +1,4 @@ -from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName, AstNode +from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName, AstNode, StackItem from formatting import prettify_filename @@ -47,5 +47,5 @@ def parse_files(filenames: list[str]) -> list[AstNode]: __all__ = [ "parse_files", "Context", "Macro", "InstDef", "Family", "Pseudo", "AstNode", - "StackEffect", "CacheEffect", "StackItem" + "StackEffect", "CacheEffect", "StackItem", ] From 22729374251e39a858081d9f89af88e0a817f87a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:18:55 +0000 Subject: [PATCH 29/40] Correct previous 'fix' --- Tools/cases_generator/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 01762c394db143..4542603daaea7b 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -1,4 +1,4 @@ -from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName, AstNode, StackItem +from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName, AstNode from formatting import prettify_filename @@ -47,5 +47,5 @@ def parse_files(filenames: list[str]) -> list[AstNode]: __all__ = [ "parse_files", "Context", "Macro", "InstDef", "Family", "Pseudo", "AstNode", - "StackEffect", "CacheEffect", "StackItem", + "StackEffect", "CacheEffect", ] From 65b0c4896f6e39568c68ff55fc25b6a0250e0492 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:20:47 +0000 Subject: [PATCH 30/40] Fix type annotation --- Tools/cases_generator/analyzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 5f4e1ca1a51e6b..c9d5c0c1e87a15 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -195,8 +195,8 @@ def convert_stack_item(item: parser.StackEffect) -> StackItem: def analyze_stack(op: parser.InstDef) -> StackEffect: - inputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] - outputs: list[parser.StackItem] = [convert_stack_item(i) for i in op.outputs] + inputs: list[StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] + outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] for (input, output) in zip(inputs, outputs): if input.name == output.name: input.peek = output.peek = True From 0ce4dd204d45a0f13649cc704c69465fed0b7ee0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:39:48 +0000 Subject: [PATCH 31/40] Fix up multi-line comments --- Python/generated_cases.c.h | 48 ++++++++++++++++---------------- Tools/cases_generator/cwriter.py | 20 +++++++++++++ 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bbea83d48da814..75aebfdd0780b2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -243,16 +243,16 @@ DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. - * - * When possible, extend `left` in place rather than - * allocating a new PyUnicodeObject. This attempts to avoid - * quadratic behavior when one neglects to use str.join(). - * - * If `left` has only two references remaining (one from - * the stack, one in the locals), DECREFing `left` leaves - * only the locals reference, so PyUnicode_Append knows - * that the string is safe to mutate. - */ + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); PyUnicode_Append(target_local, right); @@ -1059,10 +1059,10 @@ Py_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3354,19 +3354,19 @@ attr = NULL; if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ assert(attr != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - NULL | meth | arg1 | ... | argN - */ + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; @@ -4052,7 +4052,7 @@ if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ + * an exception if the key doesn't exist */ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } @@ -5737,7 +5737,7 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index c5816bf24334bf..dda78a186c512d 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -55,7 +55,27 @@ def maybe_indent(self, txt: str) -> None: def emit_text(self, txt: str) -> None: self.out.write(txt) + def emit_multiline_comment(self, tkn: Token) -> None: + self.set_position(tkn) + lines = tkn.text.splitlines(True) + first = True + for line in lines: + text = line.lstrip() + if first: + spaces = 0 + else: + spaces = self.indents[-1] + if text.startswith("*"): + spaces += 1 + else: + spaces += 3 + first = False + self.out.write(" " * spaces) + self.out.write(text) + def emit_token(self, tkn: Token) -> None: + if tkn.kind == "COMMENT" and "\n" in tkn.text: + return self.emit_multiline_comment(tkn) self.maybe_dedent(tkn.text) self.set_position(tkn) self.emit_text(tkn.text) From 54a727f8da87cc9a978158697942fc35c5c2832b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:48:16 +0000 Subject: [PATCH 32/40] Tier 2 generator can wait for another PR --- Tools/cases_generator/tier2_generator.py | 321 ----------------------- 1 file changed, 321 deletions(-) delete mode 100644 Tools/cases_generator/tier2_generator.py diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py deleted file mode 100644 index 893995b9615665..00000000000000 --- a/Tools/cases_generator/tier2_generator.py +++ /dev/null @@ -1,321 +0,0 @@ -"""Generate the main interpreter switch. -Reads the instruction definitions from bytecodes.c. -Writes the cases to generated_cases.c.h, which is #included in ceval.c. -""" - -import argparse -import os.path -import sys - -from analyzer import Analysis, Instruction, Uop, Part, analyze_files, Skip, StackItem, analysis_error -from cwriter import CWriter -from typing import TextIO, Iterator -from lexer import Token -from stack import StackOffset - -HERE = os.path.dirname(__file__) -ROOT = os.path.join(HERE, "../..") -THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/") - -DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) -DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/executor_cases.c.h")) - -def write_header(filename: str, outfile: TextIO) -> None: - outfile.write(f"""// This file is generated by {THIS} -// from: -// {filename} -// Do not edit! - -#ifdef TIER_ONE - #error "This file is for Tier 2 only" -#endif -#define TIER_TWO 2 -""") - -FOOTER = "#undef TIER_ONE\n" - -class SizeMismatch(Exception): - pass - - -class Stack: - - def __init__(self) -> None: - self.top_offset: StackOffset = StackOffset() - self.base_offset: StackOffset = StackOffset() - self.peek_offset: StackOffset = StackOffset() - self.variables: list[StackItem] = [] - self.defined: set[str] = set() - - def pop(self, var: StackItem) -> str: - self.top_offset.pop(var) - if not var.peek: - self.peek_offset.pop(var) - indirect = "&" if var.is_array() else "" - if self.variables: - popped = self.variables.pop() - if popped.size != var.size: - raise SizeMismatch(f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " - f"Expected {var.size} got {popped.size}") - if popped.name == var.name: - return "" - elif popped.name == "unused": - self.defined.add(var.name) - return f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" - elif var.name == "unused": - return "" - else: - self.defined.add(var.name) - return f"{var.name} = {popped.name};\n" - self.base_offset.pop(var) - if var.name == "unused": - return "" - else: - self.defined.add(var.name) - assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];" - if var.condition: - return f"if ({var.condition}) {{ {assign} }}\n" - return f"{assign}\n" - - def push(self, var: StackItem) -> str: - self.variables.append(var) - if var.is_array() and var.name not in self.defined and var.name != "unused": - c_offset = self.top_offset.to_c() - self.top_offset.push(var) - self.defined.add(var.name) - return f"{var.name} = &stack_pointer[{c_offset}];\n" - else: - self.top_offset.push(var) - return "" - - def flush(self, out: CWriter) -> None: - for var in self.variables: - if not var.peek: - if var.name != "unused" and not var.is_array(): - if var.condition: - out.emit(f" if ({var.condition}) ") - out.emit(f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n") - self.base_offset.push(var) - if self.base_offset.to_c() != self.top_offset.to_c(): - print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) - assert False - number = self.base_offset.to_c() - if number != "0": - out.emit(f"stack_pointer += {number};\n") - self.variables = [] - self.base_offset.clear() - self.top_offset.clear() - self.peek_offset.clear() - - def as_comment(self) -> str: - return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" - - -def declare_variables(uop: Uop, out: CWriter)->None: - variables = {"unused"} - for var in reversed(uop.stack.inputs): - if var.name not in variables: - type = var.type if var.type else "PyObject *" - variables.add(var.name) - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") - for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type = var.type if var.type else "PyObject *" - if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") - else: - out.emit(f"{type}{var.name};\n") - -def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: - parens = 0 - for tkn in tkn_iter: - if tkn.kind == end and parens == 0: - return - if tkn.kind == "LPAREN": - parens += 1 - if tkn.kind == "RPAREN": - parens -= 1 - out.emit(tkn) - -def replace_deopt(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, unused: Stack) -> None: - out.emit("DEOPT_IF") - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - out.emit(");") - -def replace_error(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: - out.emit("if ") - out.emit(next(tkn_iter)) - emit_to(out, tkn_iter, "COMMA") - label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset.to_c() - try: - offset = -int(c_offset) - close = ";\n" - except ValueError: - offset = None - out.emit(f"{{ stack_pointer += {c_offset}; ") - close = "; }\n" - out.emit("goto ") - if offset: - out.emit(f"pop_{offset}_") - out.emit(label) - out.emit(close) - -def replace_decrefs(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - for var in uop.stack.inputs: - if var.name == "unused" or var.name == "null" or var.peek: - continue - if var.size != "1": - out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"Py_DECREF({var.name}[_i]);\n") - out.emit("}\n") - elif var.condition: - out.emit(f"Py_XDECREF({var.name});\n") - else: - out.emit(f"Py_DECREF({var.name});\n") - -def replace_store_sp(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - stack.flush(out) - out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") - -def replace_check_eval_breaker(out: CWriter, tkn_iter: Iterator[Token], uop: Uop, stack: Stack) -> None: - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - if not uop.properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();") - - - -REPLACEMENT_FUNCTIONS = { - "DEOPT_IF" : replace_deopt, - "ERROR_IF" : replace_error, - "DECREF_INPUTS": replace_decrefs, - "CHECK_EVAL_BREAKER": replace_check_eval_breaker, - "STORE_SP": replace_store_sp, -} - -#Move this to formatter -def emit_tokens(out: CWriter, uop: Uop, stack: Stack) -> None: - tkn_iter = iter(uop.body[1:-1]) - for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS: - out.set_position(tkn) - REPLACEMENT_FUNCTIONS[tkn.text](out, tkn_iter, uop, stack) - else: - out.emit(tkn) - -def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: - try: - # out.emit(stack.as_comment() + "\n") - out.start_line() - for var in reversed(uop.stack.inputs): - out.emit(stack.pop(var)) - if not uop.properties.stores_sp: - for i, var in enumerate(uop.stack.outputs): - out.emit(stack.push(var)) - for cache in uop.caches: - if cache.name != "unused": - if cache.size == 4: - type = "PyObject *" - reader = "read_obj" - else: - type = f"uint{cache.size*16}_t " - reader = f"read_u{cache.size*16}" - out.emit(f"{type}{cache.name} = {reader}(&next_instr[1].operand);\n") - emit_tokens(out, uop, stack) - if uop.properties.stores_sp: - for i, var in enumerate(uop.stack.outputs): - out.emit(stack.push(var)) - # out.emit(stack.as_comment() + "\n") - except SizeMismatch as ex: - raise analysis_error(ex.args[0], uop.body[0]) - -def uses_this(uop: Uop) -> bool: - if uop.properties.needs_this: - return True - for cache in uop.caches: - if cache.name != "unused": - return True - return False - -#This belongs in the analysis phase -def viable(uop: Uop) -> bool: - if uses_this(uop): - return False - if "INSTRUMENTED" in uop.name: - return False - if "replaced" in uop.annotations: - return False - if uop.name in ( - "INTERPRETER_EXIT", - "JUMP_BACKWARD" - ): - return False - - if len([c for c in uop.caches if c.name != "unused"]) > 1: - return False - return True - -def generate_tier2(filenames: str, analysis: Analysis, outfile: TextIO, lines: bool) -> None: - write_header(filenames, outfile) - out = CWriter(outfile, 2, lines) - out.emit("\n") - for name, uop in sorted(analysis.uops.items()): - if not viable(uop): - out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n"); - continue - out.emit(f"case {uop.name}: {{") - declare_variables(uop, out) - stack = Stack() - write_uop(uop, out, stack) - out.start_line() - if not uop.properties.always_exits: - stack.flush(out) - if uop.properties.ends_with_eval_breaker: - out.emit("CHECK_EVAL_BREAKER();\n") - out.emit("break;\n") - out.emit("}") - out.emit("\n") - outfile.write(FOOTER) - - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter switch.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) - -arg_parser.add_argument( - "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" -) - -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) - -if __name__ == "__main__": - args = arg_parser.parse_args() - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - data = analyze_files(args.input) - with open(args.output, "w") as outfile: - generate_tier2(args.input, data, outfile, args.emit_line_directives) From 9a81bbef4dd52d8ad9b4b6823d787c0ae52b9de2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 21:58:56 +0000 Subject: [PATCH 33/40] Don't use old case generator for tier 1 --- Tools/cases_generator/generate_cases.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 1f94c1fedb2ac7..4b7f028970bd0c 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -883,7 +883,6 @@ def main() -> None: return # These raise OSError if output can't be written - a.write_instructions(args.output, args.emit_line_directives) a.assign_opcode_ids() a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) From 28561cd8a6387c3bf5be50df460aaf8b94de2940 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Nov 2023 22:10:11 +0000 Subject: [PATCH 34/40] Avoid emitting unreachable code --- Python/generated_cases.c.h | 2 -- Tools/cases_generator/analyzer.py | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 10e8eaed322244..24243ecfb5b8df 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4669,8 +4669,6 @@ break; } if (true) { stack_pointer += -oparg; goto error; } - stack_pointer += -oparg; - DISPATCH(); } TARGET(RERAISE) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c9d5c0c1e87a15..7f2008419e6562 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -238,17 +238,27 @@ def eval_breaker_at_end(op: parser.InstDef) -> bool: def always_exits(op: parser.InstDef) -> bool: depth = 0 - for tkn in op.tokens: + tkn_iter = iter(op.tokens) + for tkn in tkn_iter: if tkn.kind == "LBRACE": depth += 1 elif tkn.kind == "RBRACE": depth -= 1 + elif depth > 1: + continue elif tkn.kind == "GOTO" or tkn.kind == "RETURN": - if depth <= 1: + return True + elif tkn.kind == "KEYWORD": + if tkn.text in EXITS: return True - elif tkn.kind == "KEYWORD" or tkn.kind == "IDENTIFIER": - if depth <= 1 and tkn.text in EXITS: + elif tkn.kind == "IDENTIFIER": + if tkn.text in EXITS: return True + if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": + next(tkn_iter) # '(' + t = next(tkn_iter) + if t.text == "true": + return True return False def compute_properties(op: parser.InstDef) -> Properties: From 30f3015b0a819ec2f2d997dba71cedd4445d1b65 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Nov 2023 00:17:43 +0000 Subject: [PATCH 35/40] Format new files with black --- Tools/cases_generator/analyzer.py | 171 +++++++++++++++-------- Tools/cases_generator/cwriter.py | 7 +- Tools/cases_generator/parser.py | 35 ++++- Tools/cases_generator/stack.py | 2 + Tools/cases_generator/tier1_generator.py | 124 ++++++++++++---- 5 files changed, 246 insertions(+), 93 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 7f2008419e6562..f5451698a2cb7e 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,9 +1,9 @@ - from dataclasses import dataclass import lexer import parser from typing import Optional + @dataclass class Properties: escapes: bool @@ -16,18 +16,28 @@ class Properties: always_exits: bool stores_sp: bool - def dump(self, indent:str) -> None: - print(indent, - "escapes:", self.escapes, - ", infallible:", self.infallible, - ", deopts:", self.deopts, - ", oparg:", self.oparg, - ", jumps:", self.jumps, - ", eval_breaker:", self.ends_with_eval_breaker, - ", needs_this:", self.needs_this, - ", always_exits:", self.always_exits, - ", stores_sp:", self.stores_sp, - sep="" + def dump(self, indent: str) -> None: + print( + indent, + "escapes:", + self.escapes, + ", infallible:", + self.infallible, + ", deopts:", + self.deopts, + ", oparg:", + self.oparg, + ", jumps:", + self.jumps, + ", eval_breaker:", + self.ends_with_eval_breaker, + ", needs_this:", + self.needs_this, + ", always_exits:", + self.always_exits, + ", stores_sp:", + self.stores_sp, + sep="", ) @@ -43,18 +53,20 @@ def dump(self, indent:str) -> None: False, ) + @dataclass class Skip: - "Unused cache entry" - size: int + "Unused cache entry" + size: int - @property - def name(self)->str: - return f"unused/{self.size}" + @property + def name(self) -> str: + return f"unused/{self.size}" + + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES - @property - def properties(self)->Properties: - return SKIP_PROPERTIES @dataclass class StackItem: @@ -73,6 +85,7 @@ def __str__(self) -> str: def is_array(self) -> bool: return self.type == "PyObject **" + @dataclass class StackEffect: inputs: list[StackItem] @@ -81,6 +94,7 @@ class StackEffect: def __str__(self) -> str: return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})" + @dataclass class CacheEntry: name: str @@ -89,6 +103,7 @@ class CacheEntry: def __str__(self) -> str: return f"{self.name}/{self.size}" + @dataclass class Uop: name: str @@ -101,7 +116,9 @@ class Uop: _size: int = -1 def dump(self, indent: str) -> None: - print(indent, self.name, ", ".join(self.annotations) if self.annotations else "") + print( + indent, self.name, ", ".join(self.annotations) if self.annotations else "" + ) print(indent, self.stack, ", ".join([str(c) for c in self.caches])) self.properties.dump(" " + indent) @@ -111,13 +128,15 @@ def size(self) -> int: self._size = sum(c.size for c in self.caches) return self._size + Part = Uop | Skip + @dataclass class Instruction: name: str uops: list[Part] - _properties : Properties | None + _properties: Properties | None is_target: bool = False family: Optional["Family"] = None @@ -148,6 +167,7 @@ def dump(self, indent: str) -> None: def size(self) -> int: return 1 + sum(uop.size for uop in self.uops) + @dataclass class PseudoInstruction: name: str @@ -176,36 +196,44 @@ class Analysis: def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: - #To do -- support file and line output + # To do -- support file and line output # Construct a SyntaxError instance from message and token - return lexer.make_syntax_error( - message, "", tkn.line, tkn.column, "" - ) + return lexer.make_syntax_error(message, "", tkn.line, tkn.column, "") + -def override_error(name: str, context: parser.Context | None, prev_context: parser.Context | None, token: lexer.Token) -> SyntaxError: +def override_error( + name: str, + context: parser.Context | None, + prev_context: parser.Context | None, + token: lexer.Token, +) -> SyntaxError: return analysis_error( f"Duplicate definition of '{name}' @ {context} " f"previous definition @ {prev_context}", - token + token, ) + def convert_stack_item(item: parser.StackEffect) -> StackItem: - return StackItem(item.name, item.type, - item.cond, (item.size if item.size else "1")) + return StackItem(item.name, item.type, item.cond, (item.size if item.size else "1")) def analyze_stack(op: parser.InstDef) -> StackEffect: - inputs: list[StackItem] = [convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)] + inputs: list[StackItem] = [ + convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect) + ] outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs] - for (input, output) in zip(inputs, outputs): + for input, output in zip(inputs, outputs): if input.name == output.name: input.peek = output.peek = True return StackEffect(inputs, outputs) def analyze_caches(op: parser.InstDef) -> list[CacheEntry]: - caches: list[parser.CacheEffect] = [i for i in op.inputs if isinstance(i, parser.CacheEffect)] - return [ CacheEntry(i.name, int(i.size)) for i in caches ] + caches: list[parser.CacheEffect] = [ + i for i in op.inputs if isinstance(i, parser.CacheEffect) + ] + return [CacheEntry(i.name, int(i.size)) for i in caches] def variable_used(node: parser.InstDef, name: str) -> bool: @@ -214,6 +242,7 @@ def variable_used(node: parser.InstDef, name: str) -> bool: token.kind == "IDENTIFIER" and token.text == name for token in node.tokens ) + def is_infallible(op: parser.InstDef) -> bool: return not ( variable_used(op, "ERROR_IF") @@ -223,19 +252,24 @@ def is_infallible(op: parser.InstDef) -> bool: or variable_used(op, "resume_with_error") ) + from flags import makes_escaping_api_call -EXITS = set([ - "DISPATCH", - "GO_TO_INSTRUCTION", - "Py_UNREACHABLE", - "DISPATCH_INLINED", - "DISPATCH_GOTO", -]) +EXITS = set( + [ + "DISPATCH", + "GO_TO_INSTRUCTION", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", + ] +) + def eval_breaker_at_end(op: parser.InstDef) -> bool: return op.tokens[-5].text == "CHECK_EVAL_BREAKER" + def always_exits(op: parser.InstDef) -> bool: depth = 0 tkn_iter = iter(op.tokens) @@ -255,12 +289,13 @@ def always_exits(op: parser.InstDef) -> bool: if tkn.text in EXITS: return True if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF": - next(tkn_iter) # '(' + next(tkn_iter) # '(' t = next(tkn_iter) if t.text == "true": return True return False + def compute_properties(op: parser.InstDef) -> Properties: return Properties( makes_escaping_api_call(op), @@ -274,6 +309,7 @@ def compute_properties(op: parser.InstDef) -> Properties: variable_used(op, "STORE_SP"), ) + def make_uop(name: str, op: parser.InstDef) -> Uop: return Uop( name, @@ -285,24 +321,36 @@ def make_uop(name: str, op: parser.InstDef) -> Uop: compute_properties(op), ) + def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None: assert op.kind == "op" if op.name in uops: if "override" not in op.annotations: - raise override_error(op.name, op.context, uops[op.name].context,op.tokens[0]) + raise override_error( + op.name, op.context, uops[op.name].context, op.tokens[0] + ) uops[op.name] = make_uop(op.name, op) -def add_instruction(name: str, parts: list[Part], instructions: dict[str, Instruction]) -> None: + +def add_instruction( + name: str, parts: list[Part], instructions: dict[str, Instruction] +) -> None: instructions[name] = Instruction(name, parts, None) -def desugar_inst(inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop]) -> None: + +def desugar_inst( + inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: assert inst.kind == "inst" name = inst.name uop = make_uop("_" + inst.name, inst) uops[inst.name] = uop add_instruction(name, [uop], instructions) -def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop]) -> None: + +def add_macro( + macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] +) -> None: parts: list[Uop | Skip] = [] for part in macro.uops: match part: @@ -314,27 +362,37 @@ def add_macro(macro: parser.Macro, instructions: dict[str, Instruction], uops: d parts.append(Skip(part.size)) case _: assert False - assert(parts) + assert parts add_instruction(macro.name, parts, instructions) -def add_family(pfamily: parser.Family, instructions: dict[str, Instruction], families: dict[str, Family]) -> None: + +def add_family( + pfamily: parser.Family, + instructions: dict[str, Instruction], + families: dict[str, Family], +) -> None: family = Family( pfamily.name, pfamily.size, - [ instructions[member_name] for member_name in pfamily.members ] + [instructions[member_name] for member_name in pfamily.members], ) for member in family.members: member.family = family - #The head of the family is an implicit jump target for DEOPTs + # The head of the family is an implicit jump target for DEOPTs instructions[family.name].is_target = True families[family.name] = family -def add_pseudo(pseudo: parser.Pseudo, instructions: dict[str, Instruction], pseudos: dict[str, PseudoInstruction]) -> None: + +def add_pseudo( + pseudo: parser.Pseudo, + instructions: dict[str, Instruction], + pseudos: dict[str, PseudoInstruction], +) -> None: pseudos[pseudo.name] = PseudoInstruction( - pseudo.name, - [ instructions[target] for target in pseudo.targets ] + pseudo.name, [instructions[target] for target in pseudo.targets] ) + def analyze_forest(forest: list[parser.AstNode]) -> Analysis: instructions: dict[str, Instruction] = {} uops: dict[str, Uop] = {} @@ -378,13 +436,15 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: continue if target.text in instructions: instructions[target.text].is_target = True - #Hack + # Hack instructions["BINARY_OP_INPLACE_ADD_UNICODE"].family = families["BINARY_OP"] return Analysis(instructions, uops, families, pseudos) + def analyze_files(filenames: list[str]) -> Analysis: return analyze_forest(parser.parse_files(filenames)) + def dump_analysis(analysis: Analysis) -> None: print("Uops:") for u in analysis.uops.values(): @@ -402,6 +462,7 @@ def dump_analysis(analysis: Analysis) -> None: if __name__ == "__main__": import sys + if len(sys.argv) < 2: print("No input") else: diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index dda78a186c512d..0b7edd03fd9e47 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -1,17 +1,16 @@ - from lexer import Token from typing import TextIO class CWriter: - 'A writer that understands tokens and how to format C code' + "A writer that understands tokens and how to format C code" - last_token : Token | None + last_token: Token | None def __init__(self, out: TextIO, indent: int, line_directives: bool): self.out = out self.base_column = indent * 4 - self.indents = [ i*4 for i in range(indent+1) ] + self.indents = [i * 4 for i in range(indent + 1)] self.line_directives = line_directives self.last_token = None self.newline = True diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 4542603daaea7b..6c573261bfe3bc 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -1,4 +1,15 @@ -from parsing import InstDef, Macro, Pseudo, Family, Parser, Context, CacheEffect, StackEffect, OpName, AstNode +from parsing import ( + InstDef, + Macro, + Pseudo, + Family, + Parser, + Context, + CacheEffect, + StackEffect, + OpName, + AstNode, +) from formatting import prettify_filename @@ -8,6 +19,7 @@ Context = Context + def parse_files(filenames: list[str]) -> list[AstNode]: result: list[AstNode] = [] for filename in filenames: @@ -37,15 +49,24 @@ def parse_files(filenames: list[str]) -> list[AstNode]: thing_first_token = psr.peek() while node := psr.definition(): assert node is not None - result.append(node) #type: ignore[arg-type] + result.append(node) # type: ignore[arg-type] if not psr.eof(): psr.backup() - raise psr.make_syntax_error(f"Extra stuff at the end of {filename}", psr.next(True)) + raise psr.make_syntax_error( + f"Extra stuff at the end of {filename}", psr.next(True) + ) return result -#Keep MyPy happy + +# Keep MyPy happy __all__ = [ - "parse_files", "Context", "Macro", - "InstDef", "Family", "Pseudo", "AstNode", - "StackEffect", "CacheEffect", + "parse_files", + "Context", + "Macro", + "InstDef", + "Family", + "Pseudo", + "AstNode", + "StackEffect", + "CacheEffect", ] diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 778f6c14ce28ee..2b92d501a6757b 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from formatting import maybe_parenthesize + def var_size(var: StackItem) -> str: if var.condition: # Special case simplification @@ -13,6 +14,7 @@ def var_size(var: StackItem) -> str: else: return var.size + @dataclass class StackOffset: "The stack offset of the virtual base of the stack from the physical stack pointer" diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index af90f854a43eb1..2ff52857d0380a 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -7,7 +7,16 @@ import os.path import sys -from analyzer import Analysis, Instruction, Uop, Part, analyze_files, Skip, StackItem, analysis_error +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + StackItem, + analysis_error, +) from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token @@ -21,8 +30,10 @@ DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) + def write_header(filename: str, outfile: TextIO) -> None: - outfile.write(f"""// This file is generated by {THIS} + outfile.write( + f"""// This file is generated by {THIS} // from: // {filename} // Do not edit! @@ -31,16 +42,18 @@ def write_header(filename: str, outfile: TextIO) -> None: #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 -""") +""" + ) + FOOTER = "#undef TIER_ONE\n" + class SizeMismatch(Exception): pass class Stack: - def __init__(self) -> None: self.top_offset: StackOffset = StackOffset() self.base_offset: StackOffset = StackOffset() @@ -56,13 +69,17 @@ def pop(self, var: StackItem) -> str: if self.variables: popped = self.variables.pop() if popped.size != var.size: - raise SizeMismatch(f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " - f"Expected {var.size} got {popped.size}") + raise SizeMismatch( + f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " + f"Expected {var.size} got {popped.size}" + ) if popped.name == var.name: return "" elif popped.name == "unused": self.defined.add(var.name) - return f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + return ( + f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" + ) elif var.name == "unused": return "" else: @@ -95,7 +112,9 @@ def flush(self, out: CWriter) -> None: if var.name != "unused" and not var.is_array(): if var.condition: out.emit(f" if ({var.condition}) ") - out.emit(f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n") + out.emit( + f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n" + ) self.base_offset.push(var) if self.base_offset.to_c() != self.top_offset.to_c(): print("base", self.base_offset.to_c(), "top", self.top_offset.to_c()) @@ -112,7 +131,7 @@ def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" -def declare_variables(inst: Instruction, out: CWriter)->None: +def declare_variables(inst: Instruction, out: CWriter) -> None: variables = {"unused"} for uop in inst.uops: if isinstance(uop, Uop): @@ -133,6 +152,7 @@ def declare_variables(inst: Instruction, out: CWriter)->None: else: out.emit(f"{type}{var.name};\n") + def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens = 0 for tkn in tkn_iter: @@ -144,23 +164,39 @@ def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None: parens -= 1 out.emit(tkn) -def replace_deopt(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, unused: Stack, inst: Instruction) -> None: + +def replace_deopt( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction, +) -> None: out.emit_at("DEOPT_IF", tkn) out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon + next(tkn_iter) # Semi colon out.emit(", ") assert inst.family is not None out.emit(inst.family.name) out.emit(");\n") -def replace_error(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: + +def replace_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: out.emit_at("if ", tkn) out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "COMMA") label = next(tkn_iter).text - next(tkn_iter) # RPAREN - next(tkn_iter) # Semi colon + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon out.emit(") ") c_offset = stack.peek_offset.to_c() try: @@ -176,7 +212,15 @@ def replace_error(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, out.emit(label) out.emit(close) -def replace_decrefs(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: + +def replace_decrefs( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -193,7 +237,15 @@ def replace_decrefs(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uo else: out.emit(f"Py_DECREF({var.name});\n") -def replace_store_sp(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: + +def replace_store_sp( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -201,7 +253,15 @@ def replace_store_sp(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: U stack.flush(out) out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") -def replace_check_eval_breaker(out: CWriter, tkn: Token, tkn_iter: Iterator[Token], uop: Uop, stack: Stack, inst: Instruction) -> None: + +def replace_check_eval_breaker( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction, +) -> None: next(tkn_iter) next(tkn_iter) next(tkn_iter) @@ -209,16 +269,16 @@ def replace_check_eval_breaker(out: CWriter, tkn: Token, tkn_iter: Iterator[Toke out.emit_at("CHECK_EVAL_BREAKER();", tkn) - REPLACEMENT_FUNCTIONS = { - "DEOPT_IF" : replace_deopt, - "ERROR_IF" : replace_error, + "DEOPT_IF": replace_deopt, + "ERROR_IF": replace_error, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "STORE_SP": replace_store_sp, } -#Move this to formatter + +# Move this to formatter def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None: tkns = uop.body[1:-1] if not tkns: @@ -231,7 +291,10 @@ def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None else: out.emit(tkn) -def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces:bool) -> int: + +def write_uop( + uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool +) -> int: # out.emit(stack.as_comment() + "\n") if isinstance(uop, Skip): entries = "entries" if uop.size > 1 else "entry" @@ -256,7 +319,9 @@ def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruct else: type = f"uint{cache.size*16}_t " reader = f"read_u{cache.size*16}" - out.emit(f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n") + out.emit( + f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" + ) offset += cache.size emit_tokens(out, uop, stack, inst) if uop.properties.stores_sp: @@ -270,6 +335,7 @@ def write_uop(uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruct except SizeMismatch as ex: raise analysis_error(ex.args[0], uop.body[0]) + def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: return True @@ -281,7 +347,10 @@ def uses_this(inst: Instruction) -> bool: return True return False -def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO, lines: bool) -> None: + +def generate_tier1( + filenames: str, analysis: Analysis, outfile: TextIO, lines: bool +) -> None: write_header(filenames, outfile) out = CWriter(outfile, 2, lines) out.emit("\n") @@ -302,12 +371,13 @@ def generate_tier1(filenames: str, analysis: Analysis, outfile: TextIO, lines: b if inst.family is not None: out.emit( f"static_assert({inst.family.size} == {inst.size-1}" - ', "incorrect cache size");\n') + ', "incorrect cache size");\n' + ) declare_variables(inst, out) - offset = 1 # The instruction itself + offset = 1 # The instruction itself stack = Stack() for uop in inst.uops: - #Only emit braces if more than one uop + # Only emit braces if more than one uop offset = write_uop(uop, out, offset, stack, inst, len(inst.uops) > 1) out.start_line() if not inst.uops[-1].properties.always_exits: From 5e01a73eed974b7b025c29835ca6e37b89a45e81 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Nov 2023 03:08:06 +0000 Subject: [PATCH 36/40] Address review comments. --- Tools/cases_generator/analyzer.py | 92 ++++++++++++++----------------- Tools/cases_generator/mypy.ini | 2 + Tools/cases_generator/parser.py | 17 ------ 3 files changed, 42 insertions(+), 69 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index f5451698a2cb7e..32ed92b5dd9986 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -17,27 +17,22 @@ class Properties: stores_sp: bool def dump(self, indent: str) -> None: - print( - indent, - "escapes:", - self.escapes, - ", infallible:", - self.infallible, - ", deopts:", - self.deopts, - ", oparg:", - self.oparg, - ", jumps:", - self.jumps, - ", eval_breaker:", - self.ends_with_eval_breaker, - ", needs_this:", - self.needs_this, - ", always_exits:", - self.always_exits, - ", stores_sp:", - self.stores_sp, - sep="", + print(indent, end="") + text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()]) + print(indent, text, sep="") + + @staticmethod + def from_list(properties: list["Properties"]) -> "Properties": + return Properties( + any(p.escapes for p in properties), + all(p.infallible for p in properties), + any(p.deopts for p in properties), + any(p.oparg for p in properties), + any(p.jumps for p in properties), + any(p.ends_with_eval_breaker for p in properties), + any(p.needs_this for p in properties), + any(p.always_exits for p in properties), + any(p.stores_sp for p in properties), ) @@ -77,9 +72,9 @@ class StackItem: peek: bool = False def __str__(self) -> str: - cond = " if (" + self.condition + ")" if self.condition else "" - size = "[" + self.size + "]" if self.size != "1" else "" - type = "" if self.type is None else self.type + " " + cond = f" if ({self.condition})" if self.condition else "" + size = f"[{self.size}]" if self.size != "1" else "" + type = "" if self.type is None else f"{self.type} " return f"{type}{self.name}{size}{cond} {self.peek}" def is_array(self) -> bool: @@ -147,17 +142,7 @@ def properties(self) -> Properties: return self._properties def _compute_properties(self) -> Properties: - return Properties( - any(part.properties.escapes for part in self.uops), - all(part.properties.infallible for part in self.uops), - any(part.properties.deopts for part in self.uops), - any(part.properties.oparg for part in self.uops), - any(part.properties.jumps for part in self.uops), - any(part.properties.ends_with_eval_breaker for part in self.uops), - any(part.properties.needs_this for part in self.uops), - any(part.properties.always_exits for part in self.uops), - any(part.properties.stores_sp for part in self.uops), - ) + return Properties.from_list([part.properties for part in self.uops]) def dump(self, indent: str) -> None: print(indent, self.name, "=", ", ".join([op.name for op in self.uops])) @@ -172,6 +157,7 @@ def size(self) -> int: class PseudoInstruction: name: str targets: list[Instruction] + flags: list[str] def dump(self, indent: str) -> None: print(indent, self.name, "->", " or ".join([t.name for t in self.targets])) @@ -298,27 +284,27 @@ def always_exits(op: parser.InstDef) -> bool: def compute_properties(op: parser.InstDef) -> Properties: return Properties( - makes_escaping_api_call(op), - is_infallible(op), - variable_used(op, "DEOPT_IF"), - variable_used(op, "oparg"), - variable_used(op, "JUMPBY"), - eval_breaker_at_end(op), - variable_used(op, "this_instr"), - always_exits(op), - variable_used(op, "STORE_SP"), + escapes=makes_escaping_api_call(op), + infallible=is_infallible(op), + deopts=variable_used(op, "DEOPT_IF"), + oparg=variable_used(op, "oparg"), + jumps=variable_used(op, "JUMPBY"), + ends_with_eval_breaker=eval_breaker_at_end(op), + needs_this=variable_used(op, "this_instr"), + always_exits=always_exits(op), + stores_sp=variable_used(op, "STORE_SP"), ) def make_uop(name: str, op: parser.InstDef) -> Uop: return Uop( - name, - op.context, - op.annotations, - analyze_stack(op), - analyze_caches(op), - op.block.tokens, - compute_properties(op), + name=name, + context=op.context, + annotations=op.annotations, + stack=analyze_stack(op), + caches=analyze_caches(op), + body=op.block.tokens, + properties=compute_properties(op), ) @@ -389,7 +375,9 @@ def add_pseudo( pseudos: dict[str, PseudoInstruction], ) -> None: pseudos[pseudo.name] = PseudoInstruction( - pseudo.name, [instructions[target] for target in pseudo.targets] + pseudo.name, + [instructions[target] for target in pseudo.targets], + pseudo.flags, ) diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index e7175e263350b2..8e5a31851c596e 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -11,3 +11,5 @@ strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True +allow_redefinition = True +implicit_reexport = True diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index 6c573261bfe3bc..12173a61199700 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -17,9 +17,6 @@ END_MARKER = "// END BYTECODES //" -Context = Context - - def parse_files(filenames: list[str]) -> list[AstNode]: result: list[AstNode] = [] for filename in filenames: @@ -56,17 +53,3 @@ def parse_files(filenames: list[str]) -> list[AstNode]: f"Extra stuff at the end of {filename}", psr.next(True) ) return result - - -# Keep MyPy happy -__all__ = [ - "parse_files", - "Context", - "Macro", - "InstDef", - "Family", - "Pseudo", - "AstNode", - "StackEffect", - "CacheEffect", -] From 6aacaca727a223260d76e3cc83376294cf31dbd3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Nov 2023 03:09:34 +0000 Subject: [PATCH 37/40] Tidy --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 32ed92b5dd9986..0c81a0acb00788 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -201,7 +201,7 @@ def override_error( def convert_stack_item(item: parser.StackEffect) -> StackItem: - return StackItem(item.name, item.type, item.cond, (item.size if item.size else "1")) + return StackItem(item.name, item.type, item.cond, (item.size or "1")) def analyze_stack(op: parser.InstDef) -> StackEffect: From 19c9c1b6ab7ca2d95e5b577bb0aabe3a4444ad03 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Nov 2023 21:40:49 +0000 Subject: [PATCH 38/40] Add some explanatory comments --- Tools/cases_generator/stack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 2b92d501a6757b..803de7640cba1d 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -30,8 +30,10 @@ def push(self, item: StackItem) -> None: self.pushed.append(var_size(item)) def simplify(self) -> None: + "Remove matching values from both the popped and pushed list" if not self.popped or not self.pushed: return + # Sort the list so the lexically largest element is last. popped = sorted(self.popped) pushed = sorted(self.pushed) self.popped = [] @@ -42,6 +44,7 @@ def simplify(self) -> None: if pop == push: pass elif pop > push: + # if pop > push, there can be no element in pushed matching pop. self.popped.append(pop) pushed.append(push) else: From 8e385f58331006b5a9c5b31c2e939a533c837001 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Nov 2023 11:38:24 +0000 Subject: [PATCH 39/40] Address review comments --- Tools/cases_generator/analyzer.py | 60 ++++++++++++------------ Tools/cases_generator/stack.py | 1 - Tools/cases_generator/tier1_generator.py | 18 +++---- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 0c81a0acb00788..4eb184449d7410 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -24,28 +24,28 @@ def dump(self, indent: str) -> None: @staticmethod def from_list(properties: list["Properties"]) -> "Properties": return Properties( - any(p.escapes for p in properties), - all(p.infallible for p in properties), - any(p.deopts for p in properties), - any(p.oparg for p in properties), - any(p.jumps for p in properties), - any(p.ends_with_eval_breaker for p in properties), - any(p.needs_this for p in properties), - any(p.always_exits for p in properties), - any(p.stores_sp for p in properties), + escapes = any(p.escapes for p in properties), + infallible = all(p.infallible for p in properties), + deopts = any(p.deopts for p in properties), + oparg = any(p.oparg for p in properties), + jumps = any(p.jumps for p in properties), + ends_with_eval_breaker = any(p.ends_with_eval_breaker for p in properties), + needs_this = any(p.needs_this for p in properties), + always_exits = any(p.always_exits for p in properties), + stores_sp = any(p.stores_sp for p in properties), ) SKIP_PROPERTIES = Properties( - False, - True, - False, - False, - False, - False, - False, - False, - False, + escapes = False, + infallible = True, + deopts = False, + oparg = False, + jumps = False, + ends_with_eval_breaker = False, + needs_this = False, + always_exits = False, + stores_sp = False, ) @@ -130,7 +130,7 @@ def size(self) -> int: @dataclass class Instruction: name: str - uops: list[Part] + parts: list[Part] _properties: Properties | None is_target: bool = False family: Optional["Family"] = None @@ -142,15 +142,15 @@ def properties(self) -> Properties: return self._properties def _compute_properties(self) -> Properties: - return Properties.from_list([part.properties for part in self.uops]) + return Properties.from_list([part.properties for part in self.parts]) def dump(self, indent: str) -> None: - print(indent, self.name, "=", ", ".join([op.name for op in self.uops])) + print(indent, self.name, "=", ", ".join([part.name for part in self.parts])) self.properties.dump(" " + indent) @property def size(self) -> int: - return 1 + sum(uop.size for uop in self.uops) + return 1 + sum(part.size for part in self.parts) @dataclass @@ -241,15 +241,13 @@ def is_infallible(op: parser.InstDef) -> bool: from flags import makes_escaping_api_call -EXITS = set( - [ - "DISPATCH", - "GO_TO_INSTRUCTION", - "Py_UNREACHABLE", - "DISPATCH_INLINED", - "DISPATCH_GOTO", - ] -) +EXITS = { + "DISPATCH", + "GO_TO_INSTRUCTION", + "Py_UNREACHABLE", + "DISPATCH_INLINED", + "DISPATCH_GOTO", +} def eval_breaker_at_end(op: parser.InstDef) -> bool: diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 803de7640cba1d..9cd8e1a3b2edf8 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -15,7 +15,6 @@ def var_size(var: StackItem) -> str: return var.size -@dataclass class StackOffset: "The stack offset of the virtual base of the stack from the physical stack pointer" diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 2ff52857d0380a..eba926435d2415 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -55,9 +55,9 @@ class SizeMismatch(Exception): class Stack: def __init__(self) -> None: - self.top_offset: StackOffset = StackOffset() - self.base_offset: StackOffset = StackOffset() - self.peek_offset: StackOffset = StackOffset() + self.top_offset = StackOffset() + self.base_offset = StackOffset() + self.peek_offset = StackOffset() self.variables: list[StackItem] = [] self.defined: set[str] = set() @@ -133,7 +133,7 @@ def as_comment(self) -> str: def declare_variables(inst: Instruction, out: CWriter) -> None: variables = {"unused"} - for uop in inst.uops: + for uop in inst.parts: if isinstance(uop, Uop): for var in reversed(uop.stack.inputs): if var.name not in variables: @@ -339,7 +339,7 @@ def write_uop( def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: return True - for uop in inst.uops: + for uop in inst.parts: if isinstance(uop, Skip): continue for cache in uop.caches: @@ -376,13 +376,13 @@ def generate_tier1( declare_variables(inst, out) offset = 1 # The instruction itself stack = Stack() - for uop in inst.uops: + for part in inst.parts: # Only emit braces if more than one uop - offset = write_uop(uop, out, offset, stack, inst, len(inst.uops) > 1) + offset = write_uop(part, out, offset, stack, inst, len(inst.parts) > 1) out.start_line() - if not inst.uops[-1].properties.always_exits: + if not inst.parts[-1].properties.always_exits: stack.flush(out) - if inst.uops[-1].properties.ends_with_eval_breaker: + if inst.parts[-1].properties.ends_with_eval_breaker: out.emit("CHECK_EVAL_BREAKER();\n") out.emit("DISPATCH();\n") out.start_line() From bb6d42f99c548bd7ec308d7500e390d3ce876905 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Nov 2023 00:25:06 +0000 Subject: [PATCH 40/40] Rerun black --- Tools/cases_generator/analyzer.py | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4eb184449d7410..027f9861a1c0eb 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -24,28 +24,28 @@ def dump(self, indent: str) -> None: @staticmethod def from_list(properties: list["Properties"]) -> "Properties": return Properties( - escapes = any(p.escapes for p in properties), - infallible = all(p.infallible for p in properties), - deopts = any(p.deopts for p in properties), - oparg = any(p.oparg for p in properties), - jumps = any(p.jumps for p in properties), - ends_with_eval_breaker = any(p.ends_with_eval_breaker for p in properties), - needs_this = any(p.needs_this for p in properties), - always_exits = any(p.always_exits for p in properties), - stores_sp = any(p.stores_sp for p in properties), + escapes=any(p.escapes for p in properties), + infallible=all(p.infallible for p in properties), + deopts=any(p.deopts for p in properties), + oparg=any(p.oparg for p in properties), + jumps=any(p.jumps for p in properties), + ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties), + needs_this=any(p.needs_this for p in properties), + always_exits=any(p.always_exits for p in properties), + stores_sp=any(p.stores_sp for p in properties), ) SKIP_PROPERTIES = Properties( - escapes = False, - infallible = True, - deopts = False, - oparg = False, - jumps = False, - ends_with_eval_breaker = False, - needs_this = False, - always_exits = False, - stores_sp = False, + escapes=False, + infallible=True, + deopts=False, + oparg=False, + jumps=False, + ends_with_eval_breaker=False, + needs_this=False, + always_exits=False, + stores_sp=False, )