/* Execute compiled code */

/* XXX TO DO:

XXX speed up searching for keywords by using a dictionary

XXX document it!

*/

#define _PY_INTERPRETER

#include "Python.h"

#include "pycore_abstract.h" // _PyIndex_Check()

#include "pycore_call.h" // _PyObject_FastCallDictTstate()

#include "pycore_ceval.h" // _PyEval_SignalAsyncExc()

#include "pycore_code.h"

#include "pycore_function.h"

#include "pycore_long.h" // _PyLong_GetZero()

#include "pycore_object.h" // _PyObject_GC_TRACK()

#include "pycore_moduleobject.h" // PyModuleObject

#include "pycore_opcode.h" // EXTRA_CASES

#include "pycore_pyerrors.h" // _PyErr_Fetch()

#include "pycore_pymem.h" // _PyMem_IsPtrFreed()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_range.h" // _PyRangeIterObject

#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs

#include "pycore_sysmodule.h" // _PySys_Audit()

#include "pycore_tuple.h" // _PyTuple_ITEMS()

#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS

#include "pycore_dict.h"

#include "dictobject.h"

#include "pycore_frame.h"

#include "opcode.h"

#include "pydtrace.h"

#include "setobject.h"

#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#include <ctype.h>

#include <stdbool.h>

#ifdef Py_DEBUG

/* For debugging the interpreter: */

# define LLTRACE 1 /* Low-level trace feature */

#endif

#if !defined(Py_BUILD_CORE)

# error "ceval.c must be build with Py_BUILD_CORE define for best performance"

#endif

#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)

// GH-89279: The MSVC compiler does not inline these static inline functions

// in PGO build in _PyEval_EvalFrameDefault(), because this function is over

// the limit of PGO, and that limit cannot be configured.

// Define them as macros to make sure that they are always inlined by the

// preprocessor.

#undef Py_DECREF

#define Py_DECREF(arg) \

do { \

_Py_DECREF_STAT_INC(); \

PyObject *op = _PyObject_CAST(arg); \

if (--op->ob_refcnt == 0) { \

destructor dealloc = Py_TYPE(op)->tp_dealloc; \

(*dealloc)(op); \

} \

} while (0)

#undef Py_XDECREF

#define Py_XDECREF(arg) \

do { \

PyObject *xop = _PyObject_CAST(arg); \

if (xop != NULL) { \

Py_DECREF(xop); \

} \

} while (0)

#undef Py_IS_TYPE

#define Py_IS_TYPE(ob, type) \

(_PyObject_CAST(ob)->ob_type == (type))

#undef _Py_DECREF_SPECIALIZED

#define _Py_DECREF_SPECIALIZED(arg, dealloc) \

do { \

_Py_DECREF_STAT_INC(); \

PyObject *op = _PyObject_CAST(arg); \

if (--op->ob_refcnt == 0) { \

destructor d = (destructor)(dealloc); \

d(op); \

} \

} while (0)

#endif

// GH-89279: Similar to above, force inlining by using a macro.

#if defined(_MSC_VER) && SIZEOF_INT == 4

#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value)))

#else

#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL)

#endif

#define HEAD_LOCK(runtime) \

PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK)

#define HEAD_UNLOCK(runtime) \

PyThread_release_lock((runtime)->interpreters.mutex)

/* Forward declarations */

static PyObject *trace_call_function(

PyThreadState *tstate, PyObject *callable, PyObject **stack,

Py_ssize_t oparg, PyObject *kwnames);

static PyObject * do_call_core(

PyThreadState *tstate, PyObject *func,

PyObject *callargs, PyObject *kwdict, int use_tracing);

#ifdef LLTRACE

static void

dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer)

{

PyObject **stack_base = _PyFrame_Stackbase(frame);

PyObject *type, *value, *traceback;

PyErr_Fetch(&type, &value, &traceback);

printf(" stack=[");

for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) {

if (ptr != stack_base) {

printf(", ");

}

if (PyObject_Print(*ptr, stdout, 0) != 0) {

PyErr_Clear();

printf("<%s object at %p>",

Py_TYPE(*ptr)->tp_name, (void *)(*ptr));

}

}

printf("]\n");

fflush(stdout);

PyErr_Restore(type, value, traceback);

}

static void

lltrace_instruction(_PyInterpreterFrame *frame,

PyObject **stack_pointer,

_Py_CODEUNIT *next_instr)

{

dump_stack(frame, stack_pointer);

int oparg = _Py_OPARG(*next_instr);

int opcode = _Py_OPCODE(*next_instr);

const char *opname = _PyOpcode_OpName[opcode];

assert(opname != NULL);

int offset = (int)(next_instr - _PyCode_CODE(frame->f_code));

if (HAS_ARG(opcode)) {

printf("%d: %s %d\n", offset * 2, opname, oparg);

}

else {

printf("%d: %s\n", offset * 2, opname);

}

fflush(stdout);

}

static void

lltrace_resume_frame(_PyInterpreterFrame *frame)

{

PyObject *fobj = frame->f_funcobj;

if (fobj == NULL || !PyFunction_Check(fobj)) {

printf("\nResuming frame.");

return;

}

PyFunctionObject *f = (PyFunctionObject *)fobj;

PyObject *type, *value, *traceback;

PyErr_Fetch(&type, &value, &traceback);

PyObject *name = f->func_qualname;

if (name == NULL) {

name = f->func_name;

}

printf("\nResuming frame");

if (name) {

printf(" for ");

if (PyObject_Print(name, stdout, 0) < 0) {

PyErr_Clear();

}

}

if (f->func_module) {

printf(" in module ");

if (PyObject_Print(f->func_module, stdout, 0) < 0) {

PyErr_Clear();

}

}

printf("\n");

fflush(stdout);

PyErr_Restore(type, value, traceback);

}

#endif

static int call_trace(Py_tracefunc, PyObject *,

PyThreadState *, _PyInterpreterFrame *,

int, PyObject *);

static int call_trace_protected(Py_tracefunc, PyObject *,

PyThreadState *, _PyInterpreterFrame *,

int, PyObject *);

static void call_exc_trace(Py_tracefunc, PyObject *,

PyThreadState *, _PyInterpreterFrame *);

static int maybe_call_line_trace(Py_tracefunc, PyObject *,

PyThreadState *, _PyInterpreterFrame *, int);

static void maybe_dtrace_line(_PyInterpreterFrame *, PyTraceInfo *, int);

static void dtrace_function_entry(_PyInterpreterFrame *);

static void dtrace_function_return(_PyInterpreterFrame *);

static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *,

PyObject *, PyObject *, PyObject *);

static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);

static int import_all_from(PyThreadState *, PyObject *, PyObject *);

static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);

static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);

static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);

static int check_except_type_valid(PyThreadState *tstate, PyObject* right);

static int check_except_star_type_valid(PyThreadState *tstate, PyObject* right);

static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);

static void format_awaitable_error(PyThreadState *, PyTypeObject *, int);

static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);

static _PyInterpreterFrame *

_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,

PyObject *locals, PyObject* const* args,

size_t argcount, PyObject *kwnames);

static void

_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);

#define NAME_ERROR_MSG \

"name '%.200s' is not defined"

#define UNBOUNDLOCAL_ERROR_MSG \

"cannot access local variable '%s' where it is not associated with a value"

#define UNBOUNDFREE_ERROR_MSG \

"cannot access free variable '%s' where it is not associated with a" \

" value in enclosing scope"

#ifndef NDEBUG

/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and

PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen

when a thread continues to run after Python finalization, especially

daemon threads. */

static int

is_tstate_valid(PyThreadState *tstate)

{

assert(!_PyMem_IsPtrFreed(tstate));

assert(!_PyMem_IsPtrFreed(tstate->interp));

return 1;

}

#endif

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

int

Py_GetRecursionLimit(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

return interp->ceval.recursion_limit;

}

void

Py_SetRecursionLimit(int new_limit)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

interp->ceval.recursion_limit = new_limit;

for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) {

int depth = p->recursion_limit - p->recursion_remaining;

p->recursion_limit = new_limit;

p->recursion_remaining = new_limit - depth;

}

}

/* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall()

if the recursion_depth reaches recursion_limit. */

int

_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)

{

/* Check against global limit first. */

int depth = tstate->recursion_limit - tstate->recursion_remaining;

if (depth < tstate->interp->ceval.recursion_limit) {

tstate->recursion_limit = tstate->interp->ceval.recursion_limit;

tstate->recursion_remaining = tstate->recursion_limit - depth;

assert(tstate->recursion_remaining > 0);

return 0;

}

#ifdef USE_STACKCHECK

if (PyOS_CheckStack()) {

++tstate->recursion_remaining;

_PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");

return -1;

}

#endif

if (tstate->recursion_headroom) {

if (tstate->recursion_remaining < -50) {

/* Overflowing while handling an overflow. Give up. */

Py_FatalError("Cannot recover from stack overflow.");

}

}

else {

if (tstate->recursion_remaining <= 0) {

tstate->recursion_headroom++;

_PyErr_Format(tstate, PyExc_RecursionError,

"maximum recursion depth exceeded%s",

where);

tstate->recursion_headroom--;

++tstate->recursion_remaining;

return -1;

}

}

return 0;

}

static const binaryfunc binary_ops[] = {

[NB_ADD] = PyNumber_Add,

[NB_AND] = PyNumber_And,

[NB_FLOOR_DIVIDE] = PyNumber_FloorDivide,

[NB_LSHIFT] = PyNumber_Lshift,

[NB_MATRIX_MULTIPLY] = PyNumber_MatrixMultiply,

[NB_MULTIPLY] = PyNumber_Multiply,

[NB_REMAINDER] = PyNumber_Remainder,

[NB_OR] = PyNumber_Or,

[NB_POWER] = _PyNumber_PowerNoMod,

[NB_RSHIFT] = PyNumber_Rshift,

[NB_SUBTRACT] = PyNumber_Subtract,

[NB_TRUE_DIVIDE] = PyNumber_TrueDivide,

[NB_XOR] = PyNumber_Xor,

[NB_INPLACE_ADD] = PyNumber_InPlaceAdd,

[NB_INPLACE_AND] = PyNumber_InPlaceAnd,

[NB_INPLACE_FLOOR_DIVIDE] = PyNumber_InPlaceFloorDivide,

[NB_INPLACE_LSHIFT] = PyNumber_InPlaceLshift,

[NB_INPLACE_MATRIX_MULTIPLY] = PyNumber_InPlaceMatrixMultiply,

[NB_INPLACE_MULTIPLY] = PyNumber_InPlaceMultiply,

[NB_INPLACE_REMAINDER] = PyNumber_InPlaceRemainder,

[NB_INPLACE_OR] = PyNumber_InPlaceOr,

[NB_INPLACE_POWER] = _PyNumber_InPlacePowerNoMod,

[NB_INPLACE_RSHIFT] = PyNumber_InPlaceRshift,

[NB_INPLACE_SUBTRACT] = PyNumber_InPlaceSubtract,

[NB_INPLACE_TRUE_DIVIDE] = PyNumber_InPlaceTrueDivide,

[NB_INPLACE_XOR] = PyNumber_InPlaceXor,

};

// PEP 634: Structural Pattern Matching

// Return a tuple of values corresponding to keys, with error checks for

// duplicate/missing keys.

static PyObject*

match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys)

{

assert(PyTuple_CheckExact(keys));

Py_ssize_t nkeys = PyTuple_GET_SIZE(keys);

if (!nkeys) {

// No keys means no items.

return PyTuple_New(0);

}

PyObject *seen = NULL;

PyObject *dummy = NULL;

PyObject *values = NULL;

PyObject *get = NULL;

// We use the two argument form of map.get(key, default) for two reasons:

// - Atomically check for a key and get its value without error handling.

// - Don't cause key creation or resizing in dict subclasses like

// collections.defaultdict that define __missing__ (or similar).

int meth_found = _PyObject_GetMethod(map, &_Py_ID(get), &get);

if (get == NULL) {

goto fail;

}

seen = PySet_New(NULL);

if (seen == NULL) {

goto fail;

}

// dummy = object()

dummy = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type);

if (dummy == NULL) {

goto fail;

}

values = PyTuple_New(nkeys);

if (values == NULL) {

goto fail;

}

for (Py_ssize_t i = 0; i < nkeys; i++) {

PyObject *key = PyTuple_GET_ITEM(keys, i);

if (PySet_Contains(seen, key) || PySet_Add(seen, key)) {

if (!_PyErr_Occurred(tstate)) {

// Seen it before!

_PyErr_Format(tstate, PyExc_ValueError,

"mapping pattern checks duplicate key (%R)", key);

}

goto fail;

}

PyObject *args[] = { map, key, dummy };

PyObject *value = NULL;

if (meth_found) {

value = PyObject_Vectorcall(get, args, 3, NULL);

}

else {

value = PyObject_Vectorcall(get, &args[1], 2, NULL);

}

if (value == NULL) {

goto fail;

}

if (value == dummy) {

// key not in map!

Py_DECREF(value);

Py_DECREF(values);

// Return None:

Py_INCREF(Py_None);

values = Py_None;

goto done;

}

PyTuple_SET_ITEM(values, i, value);

}

// Success:

done:

Py_DECREF(get);

Py_DECREF(seen);

Py_DECREF(dummy);

return values;

fail:

Py_XDECREF(get);

Py_XDECREF(seen);

Py_XDECREF(dummy);

Py_XDECREF(values);

return NULL;

}

// Extract a named attribute from the subject, with additional bookkeeping to

// raise TypeErrors for repeated lookups. On failure, return NULL (with no

// error set). Use _PyErr_Occurred(tstate) to disambiguate.

static PyObject*

match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,

PyObject *name, PyObject *seen)

{

assert(PyUnicode_CheckExact(name));

assert(PySet_CheckExact(seen));

if (PySet_Contains(seen, name) || PySet_Add(seen, name)) {

if (!_PyErr_Occurred(tstate)) {

// Seen it before!

_PyErr_Format(tstate, PyExc_TypeError,

"%s() got multiple sub-patterns for attribute %R",

((PyTypeObject*)type)->tp_name, name);

}

return NULL;

}

PyObject *attr = PyObject_GetAttr(subject, name);

if (attr == NULL && _PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {

_PyErr_Clear(tstate);

}

return attr;

}

// On success (match), return a tuple of extracted attributes. On failure (no

// match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.

static PyObject*

match_class(PyThreadState *tstate, PyObject *subject, PyObject *type,

Py_ssize_t nargs, PyObject *kwargs)

{

if (!PyType_Check(type)) {

const char *e = "called match pattern must be a type";

_PyErr_Format(tstate, PyExc_TypeError, e);

return NULL;

}

assert(PyTuple_CheckExact(kwargs));

// First, an isinstance check:

if (PyObject_IsInstance(subject, type) <= 0) {

return NULL;

}

// So far so good:

PyObject *seen = PySet_New(NULL);

if (seen == NULL) {

return NULL;

}

PyObject *attrs = PyList_New(0);

if (attrs == NULL) {

Py_DECREF(seen);

return NULL;

}

// NOTE: From this point on, goto fail on failure:

PyObject *match_args = NULL;

// First, the positional subpatterns:

if (nargs) {

int match_self = 0;

match_args = PyObject_GetAttrString(type, "__match_args__");

if (match_args) {

if (!PyTuple_CheckExact(match_args)) {

const char *e = "%s.__match_args__ must be a tuple (got %s)";

_PyErr_Format(tstate, PyExc_TypeError, e,

((PyTypeObject *)type)->tp_name,

Py_TYPE(match_args)->tp_name);

goto fail;

}

}

else if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {

_PyErr_Clear(tstate);

// _Py_TPFLAGS_MATCH_SELF is only acknowledged if the type does not

// define __match_args__. This is natural behavior for subclasses:

// it's as if __match_args__ is some "magic" value that is lost as

// soon as they redefine it.

match_args = PyTuple_New(0);

match_self = PyType_HasFeature((PyTypeObject*)type,

_Py_TPFLAGS_MATCH_SELF);

}

else {

goto fail;

}

assert(PyTuple_CheckExact(match_args));

Py_ssize_t allowed = match_self ? 1 : PyTuple_GET_SIZE(match_args);

if (allowed < nargs) {

const char *plural = (allowed == 1) ? "" : "s";

_PyErr_Format(tstate, PyExc_TypeError,

"%s() accepts %d positional sub-pattern%s (%d given)",

((PyTypeObject*)type)->tp_name,

allowed, plural, nargs);

goto fail;

}

if (match_self) {

// Easy. Copy the subject itself, and move on to kwargs.

PyList_Append(attrs, subject);

}

else {

for (Py_ssize_t i = 0; i < nargs; i++) {

PyObject *name = PyTuple_GET_ITEM(match_args, i);

if (!PyUnicode_CheckExact(name)) {

_PyErr_Format(tstate, PyExc_TypeError,

"__match_args__ elements must be strings "

"(got %s)", Py_TYPE(name)->tp_name);

goto fail;

}

PyObject *attr = match_class_attr(tstate, subject, type, name,

seen);

if (attr == NULL) {

goto fail;

}

PyList_Append(attrs, attr);

Py_DECREF(attr);

}

}

Py_CLEAR(match_args);

}

// Finally, the keyword subpatterns:

for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwargs); i++) {

PyObject *name = PyTuple_GET_ITEM(kwargs, i);

PyObject *attr = match_class_attr(tstate, subject, type, name, seen);

if (attr == NULL) {

goto fail;

}

PyList_Append(attrs, attr);

Py_DECREF(attr);

}

Py_SETREF(attrs, PyList_AsTuple(attrs));

Py_DECREF(seen);

return attrs;

fail:

// We really don't care whether an error was raised or not... that's our

// caller's problem. All we know is that the match failed.

Py_XDECREF(match_args);

Py_DECREF(seen);

Py_DECREF(attrs);

return NULL;

}

static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);

static int exception_group_match(

PyObject* exc_value, PyObject *match_type,

PyObject **match, PyObject **rest);

static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);

PyObject *

PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)

{

PyThreadState *tstate = _PyThreadState_GET();

if (locals == NULL) {

locals = globals;

}

PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref

if (builtins == NULL) {

return NULL;

}

PyFrameConstructor desc = {

.fc_globals = globals,

.fc_builtins = builtins,

.fc_name = ((PyCodeObject *)co)->co_name,

.fc_qualname = ((PyCodeObject *)co)->co_name,

.fc_code = co,

.fc_defaults = NULL,

.fc_kwdefaults = NULL,

.fc_closure = NULL

};

PyFunctionObject *func = _PyFunction_FromConstructor(&desc);

if (func == NULL) {

return NULL;

}

EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY);

PyObject *res = _PyEval_Vector(tstate, func, locals, NULL, 0, NULL);

Py_DECREF(func);

return res;

}

/* Interpreter main loop */

PyObject *

PyEval_EvalFrame(PyFrameObject *f)

{

/* Function kept for backward compatibility */

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f->f_frame, 0);

}

PyObject *

PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)

{

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f->f_frame, throwflag);

}

/* Computed GOTOs, or

the-optimization-commonly-but-improperly-known-as-"threaded code"

using gcc's labels-as-values extension

(http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).

The traditional bytecode evaluation loop uses a "switch" statement, which

decent compilers will optimize as a single indirect branch instruction

combined with a lookup table of jump addresses. However, since the

indirect jump instruction is shared by all opcodes, the CPU will have a

hard time making the right prediction for where to jump next (actually,

it will be always wrong except in the uncommon case of a sequence of

several identical opcodes).

"Threaded code" in contrast, uses an explicit jump table and an explicit

indirect jump instruction at the end of each opcode. Since the jump

instruction is at a different address for each opcode, the CPU will make a

separate prediction for each of these instructions, which is equivalent to

predicting the second opcode of each opcode pair. These predictions have

a much better chance to turn out valid, especially in small bytecode loops.

A mispredicted branch on a modern CPU flushes the whole pipeline and

can cost several CPU cycles (depending on the pipeline depth),

and potentially many more instructions (depending on the pipeline width).

A correctly predicted branch, however, is nearly free.

At the time of this writing, the "threaded code" version is up to 15-20%

faster than the normal "switch" version, depending on the compiler and the

CPU architecture.

NOTE: care must be taken that the compiler doesn't try to "optimize" the

indirect jumps by sharing them between all opcodes. Such optimizations

can be disabled on gcc by using the -fno-gcse flag (or possibly

-fno-crossjumping).

*/

/* Use macros rather than inline functions, to make it as clear as possible

* to the C compiler that the tracing check is a simple test then branch.

* We want to be sure that the compiler knows this before it generates

* the CFG.

*/

#ifdef WITH_DTRACE

#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0)

#else

#define OR_DTRACE_LINE

#endif

#ifdef HAVE_COMPUTED_GOTOS

#ifndef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 1

#endif

#else

#if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS

#error "Computed gotos are not supported on this compiler."

#endif

#undef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 0

#endif

#ifdef Py_STATS

#define INSTRUCTION_START(op) \

do { \

frame->prev_instr = next_instr++; \

OPCODE_EXE_INC(op); \

if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \

lastopcode = op; \

} while (0)

#else

#define INSTRUCTION_START(op) (frame->prev_instr = next_instr++)

#endif

#if USE_COMPUTED_GOTOS

#define TARGET(op) TARGET_##op: INSTRUCTION_START(op);

#define DISPATCH_GOTO() goto *opcode_targets[opcode]

#else

#define TARGET(op) case op: INSTRUCTION_START(op);

#define DISPATCH_GOTO() goto dispatch_opcode

#endif

/* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */

#ifdef LLTRACE

#define PRE_DISPATCH_GOTO() if (lltrace) { \

lltrace_instruction(frame, stack_pointer, next_instr); }

#else

#define PRE_DISPATCH_GOTO() ((void)0)

#endif

#define NOTRACE_DISPATCH() \

{ \

NEXTOPARG(); \

PRE_DISPATCH_GOTO(); \

DISPATCH_GOTO(); \

}

/* Do interpreter dispatch accounting for tracing and instrumentation */

#define DISPATCH() \

{ \

NEXTOPARG(); \

PRE_DISPATCH_GOTO(); \

assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \

opcode |= cframe.use_tracing OR_DTRACE_LINE; \

DISPATCH_GOTO(); \

}

#define NOTRACE_DISPATCH_SAME_OPARG() \

{ \

opcode = _Py_OPCODE(*next_instr); \

PRE_DISPATCH_GOTO(); \

DISPATCH_GOTO(); \

}

#define CHECK_EVAL_BREAKER() \

_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \

if (_Py_atomic_load_relaxed_int32(eval_breaker)) { \

goto handle_eval_breaker; \

}

/* Tuple access macros */

#ifndef Py_DEBUG

#define GETITEM(v, i) PyTuple_GET_ITEM((v), (i))

#else

static inline PyObject *

GETITEM(PyObject *v, Py_ssize_t i) {

assert(PyTuple_Check(v));

assert(i >= 0);

assert(i < PyTuple_GET_SIZE(v));

return PyTuple_GET_ITEM(v, i);

}

#endif

/* Code access macros */

/* The integer overflow is checked by an assertion below. */

#define INSTR_OFFSET() ((int)(next_instr - first_instr))

#define NEXTOPARG() do { \

_Py_CODEUNIT word = *next_instr; \

opcode = _Py_OPCODE(word); \

oparg = _Py_OPARG(word); \

} while (0)

#define JUMPTO(x) (next_instr = first_instr + (x))

#define JUMPBY(x) (next_instr += (x))

/* Get opcode and oparg from original instructions, not quickened form. */

#define TRACING_NEXTOPARG() do { \

NEXTOPARG(); \

opcode = _PyOpcode_Deopt[opcode]; \

} while (0)

/* OpCode prediction macros

Some opcodes tend to come in pairs thus making it possible to

predict the second code when the first is run. For example,

COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE.

Verifying the prediction costs a single high-speed test of a register

variable against a constant. If the pairing was good, then the

processor's own internal branch predication has a high likelihood of

success, resulting in a nearly zero-overhead transition to the

next opcode. A successful prediction saves a trip through the eval-loop

including its unpredictable switch-case branch. Combined with the

processor's internal branch prediction, a successful PREDICT has the

effect of making the two opcodes run as if they were a single new opcode

with the bodies combined.

If collecting opcode statistics, your choices are to either keep the

predictions turned-on and interpret the results as if some opcodes

had been combined or turn-off predictions so that the opcode frequency

counter updates for both opcodes.

Opcode prediction is disabled with threaded code, since the latter allows

the CPU to record separate branch prediction information for each

opcode.

*/

#define PREDICT_ID(op) PRED_##op

#if USE_COMPUTED_GOTOS

#define PREDICT(op) if (0) goto PREDICT_ID(op)

#else

#define PREDICT(op) \

do { \

_Py_CODEUNIT word = *next_instr; \

opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \

if (opcode == op) { \

oparg = _Py_OPARG(word); \

INSTRUCTION_START(op); \

goto PREDICT_ID(op); \

} \

} while(0)

#endif

#define PREDICTED(op) PREDICT_ID(op):

/* Stack manipulation macros */

/* The stack can grow at most MAXINT deep, as co_nlocals and

co_stacksize are ints. */

#define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame)))

#define STACK_SIZE() (frame->f_code->co_stacksize)

#define EMPTY() (STACK_LEVEL() == 0)

#define TOP() (stack_pointer[-1])

#define SECOND() (stack_pointer[-2])

#define THIRD() (stack_pointer[-3])

#define FOURTH() (stack_pointer[-4])

#define PEEK(n) (stack_pointer[-(n)])

#define SET_TOP(v) (stack_pointer[-1] = (v))

#define SET_SECOND(v) (stack_pointer[-2] = (v))

#define BASIC_STACKADJ(n) (stack_pointer += n)

#define BASIC_PUSH(v) (*stack_pointer++ = (v))

#define BASIC_POP() (*--stack_pointer)

#ifdef Py_DEBUG

#define PUSH(v) do { \

BASIC_PUSH(v); \

assert(STACK_LEVEL() <= STACK_SIZE()); \

} while (0)

#define POP() (assert(STACK_LEVEL() > 0), BASIC_POP())

#define STACK_GROW(n) do { \

assert(n >= 0); \

BASIC_STACKADJ(n); \

assert(STACK_LEVEL() <= STACK_SIZE()); \

} while (0)

#define STACK_SHRINK(n) do { \

assert(n >= 0); \

assert(STACK_LEVEL() >= n); \

BASIC_STACKADJ(-(n)); \

} while (0)

#else

#define PUSH(v) BASIC_PUSH(v)

#define POP() BASIC_POP()

#define STACK_GROW(n) BASIC_STACKADJ(n)

#define STACK_SHRINK(n) BASIC_STACKADJ(-(n))

#endif

/* Local variable macros */

#define GETLOCAL(i) (frame->localsplus[i])

/* The SETLOCAL() macro must not DECREF the local variable in-place and

then store the new value; it must copy the old value to a temporary

value, then store the new value, and then DECREF the temporary value.

This is because it is possible that during the DECREF the frame is

accessed by other code (e.g. a __del__ method or gc.collect()) and the

variable would be pointing to already-freed memory. */

#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \

GETLOCAL(i) = value; \

Py_XDECREF(tmp); } while (0)

#define JUMP_TO_INSTRUCTION(op) goto PREDICT_ID(op)

#define DEOPT_IF(cond, instname) if (cond) { goto miss; }

#define GLOBALS() frame->f_globals

#define BUILTINS() frame->f_builtins

#define LOCALS() frame->f_locals

/* Shared opcode macros */

#define TRACE_FUNCTION_EXIT() \

if (cframe.use_tracing) { \

if (trace_function_exit(tstate, frame, retval)) { \

Py_DECREF(retval); \

goto exit_unwind; \

} \

}

#define DTRACE_FUNCTION_EXIT() \

if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \

dtrace_function_return(frame); \

}

#define TRACE_FUNCTION_UNWIND() \

if (cframe.use_tracing) { \

/* Since we are already unwinding, \

* we don't care if this raises */ \

trace_function_exit(tstate, frame, NULL); \

}

#define TRACE_FUNCTION_ENTRY() \

if (cframe.use_tracing) { \

_PyFrame_SetStackPointer(frame, stack_pointer); \

int err = trace_function_entry(tstate, frame); \

stack_pointer = _PyFrame_GetStackPointer(frame); \

if (err) { \

goto error; \

} \

}

#define TRACE_FUNCTION_THROW_ENTRY() \

if (cframe.use_tracing) { \

assert(frame->stacktop >= 0); \

if (trace_function_entry(tstate, frame)) { \

goto exit_unwind; \

} \

}

#define DTRACE_FUNCTION_ENTRY() \

if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \

dtrace_function_entry(frame); \

}

#define ADAPTIVE_COUNTER_IS_ZERO(cache) \

(cache)->counter < (1<<ADAPTIVE_BACKOFF_BITS)

#define DECREMENT_ADAPTIVE_COUNTER(cache) \

(cache)->counter -= (1<<ADAPTIVE_BACKOFF_BITS)

static int

trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame)

{

if (tstate->c_tracefunc != NULL) {

/* tstate->c_tracefunc, if defined, is a

function that will be called on *every* entry

to a code block. Its return value, if not

None, is a function that will be called at

the start of each executed line of code.

(Actually, the function must return itself

in order to continue tracing.) The trace

functions are called with three arguments:

a pointer to the current frame, a string

indicating why the function is called, and

an argument which depends on the situation.

The global trace function is also called

whenever an exception is detected. */

if (call_trace_protected(tstate->c_tracefunc,

tstate->c_traceobj,

tstate, frame,

PyTrace_CALL, Py_None)) {

/* Trace function raised an error */

return -1;

}

}

if (tstate->c_profilefunc != NULL) {

/* Similar for c_profilefunc, except it needn't

return itself and isn't called for "line" events */

if (call_trace_protected(tstate->c_profilefunc,

tstate->c_profileobj,

tstate, frame,

PyTrace_CALL, Py_None)) {

/* Profile function raised an error */

return -1;

}

}

return 0;

}

static int

trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *retval)

{

if (tstate->c_tracefunc) {

if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,

tstate, frame, PyTrace_RETURN, retval)) {

return -1;

}

}

if (tstate->c_profilefunc) {

if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,

tstate, frame, PyTrace_RETURN, retval)) {

return -1;

}

}

return 0;

}

static _PyInterpreterFrame *

pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame)

{

_PyInterpreterFrame *prev_frame = frame->previous;

_PyEvalFrameClearAndPop(tstate, frame);

return prev_frame;

}

/* It is only between the KW_NAMES instruction and the following CALL,

* that this has any meaning.

*/

typedef struct {

PyObject *kwnames;

} CallShape;

// GH-89279: Must be a macro to be sure it's inlined by MSVC.

#define is_method(stack_pointer, args) (PEEK((args)+2) != NULL)