/* Execute compiled code */

#define _PY_INTERPRETER

#include "Python.h"

#include "pycore_abstract.h" // _PyIndex_Check()

#include "pycore_audit.h" // _PySys_Audit()

#include "pycore_backoff.h"

#include "pycore_call.h" // _PyObject_CallNoArgs()

#include "pycore_cell.h" // PyCell_GetRef()

#include "pycore_ceval.h"

#include "pycore_code.h"

#include "pycore_dict.h"

#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS

#include "pycore_floatobject.h" // _PyFloat_ExactDealloc()

#include "pycore_frame.h"

#include "pycore_function.h"

#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter()

#include "pycore_import.h" // _PyImport_IsDefaultImportFunc()

#include "pycore_instruments.h"

#include "pycore_interpframe.h" // _PyFrame_SetStackPointer()

#include "pycore_interpolation.h" // _PyInterpolation_Build()

#include "pycore_intrinsics.h"

#include "pycore_jit.h"

#include "pycore_list.h" // _PyList_GetItemRef()

#include "pycore_long.h" // _PyLong_GetZero()

#include "pycore_moduleobject.h" // PyModuleObject

#include "pycore_object.h" // _PyObject_GC_TRACK()

#include "pycore_opcode_metadata.h" // EXTRA_CASES

#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*

#include "pycore_optimizer.h" // _PyUOpExecutor_Type

#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*

#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_range.h" // _PyRangeIterObject

#include "pycore_setobject.h" // _PySet_Update()

#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs

#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString()

#include "pycore_template.h" // _PyTemplate_Build()

#include "pycore_traceback.h" // _PyTraceBack_FromFrame

#include "pycore_tuple.h" // _PyTuple_ITEMS()

#include "pycore_uop_ids.h" // Uops

#include "dictobject.h"

#include "frameobject.h" // _PyInterpreterFrame_GetLine

#include "opcode.h"

#include "pydtrace.h"

#include "setobject.h"

#include "pycore_stackref.h"

#include <stdbool.h> // bool

#if !defined(Py_BUILD_CORE)

# error "ceval.c must be build with Py_BUILD_CORE define for best performance"

#endif

#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)

// GH-89279: The MSVC compiler does not inline these static inline functions

// in PGO build in _PyEval_EvalFrameDefault(), because this function is over

// the limit of PGO, and that limit cannot be configured.

// Define them as macros to make sure that they are always inlined by the

// preprocessor.

#undef Py_IS_TYPE

#define Py_IS_TYPE(ob, type) \

(_PyObject_CAST(ob)->ob_type == (type))

#undef Py_XDECREF

#define Py_XDECREF(arg) \

do { \

PyObject *xop = _PyObject_CAST(arg); \

if (xop != NULL) { \

Py_DECREF(xop); \

} \

} while (0)

#ifndef Py_GIL_DISABLED

#undef Py_DECREF

#define Py_DECREF(arg) \

do { \

PyObject *op = _PyObject_CAST(arg); \

if (_Py_IsImmortal(op)) { \

_Py_DECREF_IMMORTAL_STAT_INC(); \

break; \

} \

_Py_DECREF_STAT_INC(); \

if (--op->ob_refcnt == 0) { \

_PyReftracerTrack(op, PyRefTracer_DESTROY); \

destructor dealloc = Py_TYPE(op)->tp_dealloc; \

(*dealloc)(op); \

} \

} while (0)

#undef _Py_DECREF_SPECIALIZED

#define _Py_DECREF_SPECIALIZED(arg, dealloc) \

do { \

PyObject *op = _PyObject_CAST(arg); \

if (_Py_IsImmortal(op)) { \

_Py_DECREF_IMMORTAL_STAT_INC(); \

break; \

} \

_Py_DECREF_STAT_INC(); \

if (--op->ob_refcnt == 0) { \

_PyReftracerTrack(op, PyRefTracer_DESTROY); \

destructor d = (destructor)(dealloc); \

d(op); \

} \

} while (0)

#else // Py_GIL_DISABLED

#undef Py_DECREF

#define Py_DECREF(arg) \

do { \

PyObject *op = _PyObject_CAST(arg); \

uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \

if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \

_Py_DECREF_IMMORTAL_STAT_INC(); \

break; \

} \

_Py_DECREF_STAT_INC(); \

if (_Py_IsOwnedByCurrentThread(op)) { \

local--; \

_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \

if (local == 0) { \

_Py_MergeZeroLocalRefcount(op); \

} \

} \

else { \

_Py_DecRefShared(op); \

} \

} while (0)

#undef _Py_DECREF_SPECIALIZED

#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)

#endif

#endif

static void

check_invalid_reentrancy(void)

{

#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)

// In the free-threaded build, the interpreter must not be re-entered if

// the world-is-stopped. If so, that's a bug somewhere (quite likely in

// the painfully complex typeobject code).

PyInterpreterState *interp = _PyInterpreterState_GET();

assert(!interp->stoptheworld.world_stopped);

#endif

}

#ifdef Py_DEBUG

static void

dump_item(_PyStackRef item)

{

if (PyStackRef_IsNull(item)) {

printf("<NULL>");

return;

}

if (PyStackRef_IsTaggedInt(item)) {

printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));

return;

}

PyObject *obj = PyStackRef_AsPyObjectBorrow(item);

if (obj == NULL) {

printf("<nil>");

return;

}

// Don't call __repr__(), it might recurse into the interpreter.

printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);

}

static void

dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)

{

_PyFrame_SetStackPointer(frame, stack_pointer);

_PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);

_PyStackRef *stack_base = _PyFrame_Stackbase(frame);

PyObject *exc = PyErr_GetRaisedException();

printf(" locals=[");

for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {

if (ptr != locals_base) {

printf(", ");

}

dump_item(*ptr);

}

printf("]\n");

if (stack_pointer < stack_base) {

printf(" stack=%d\n", (int)(stack_pointer-stack_base));

}

else {

printf(" stack=[");

for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {

if (ptr != stack_base) {

printf(", ");

}

dump_item(*ptr);

}

printf("]\n");

}

fflush(stdout);

PyErr_SetRaisedException(exc);

_PyFrame_GetStackPointer(frame);

}

static void

lltrace_instruction(_PyInterpreterFrame *frame,

_PyStackRef *stack_pointer,

_Py_CODEUNIT *next_instr,

int opcode,

int oparg)

{

int offset = 0;

if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {

dump_stack(frame, stack_pointer);

offset = (int)(next_instr - _PyFrame_GetBytecode(frame));

}

const char *opname = _PyOpcode_OpName[opcode];

assert(opname != NULL);

if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {

printf("%d: %s %d\n", offset * 2, opname, oparg);

}

else {

printf("%d: %s\n", offset * 2, opname);

}

fflush(stdout);

}

static void

lltrace_resume_frame(_PyInterpreterFrame *frame)

{

PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);

if (!PyStackRef_CodeCheck(frame->f_executable) ||

fobj == NULL ||

!PyFunction_Check(fobj)

) {

printf("\nResuming frame.\n");

return;

}

PyFunctionObject *f = (PyFunctionObject *)fobj;

PyObject *exc = PyErr_GetRaisedException();

PyObject *name = f->func_qualname;

if (name == NULL) {

name = f->func_name;

}

printf("\nResuming frame");

if (name) {

printf(" for ");

if (PyObject_Print(name, stdout, 0) < 0) {

PyErr_Clear();

}

}

if (f->func_module) {

printf(" in module ");

if (PyObject_Print(f->func_module, stdout, 0) < 0) {

PyErr_Clear();

}

}

printf("\n");

fflush(stdout);

PyErr_SetRaisedException(exc);

}

static int

maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)

{

if (globals == NULL) {

return 0;

}

if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {

return 0;

}

int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));

if (r < 0) {

return -1;

}

int lltrace = r * 5; // Levels 1-4 only trace uops

if (!lltrace) {

// Can also be controlled by environment variable

char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");

if (python_lltrace != NULL && *python_lltrace >= '0') {

lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that

}

}

if (lltrace >= 5) {

lltrace_resume_frame(frame);

}

return lltrace;

}

#endif

static void monitor_reraise(PyThreadState *tstate,

_PyInterpreterFrame *frame,

_Py_CODEUNIT *instr);

static int monitor_stop_iteration(PyThreadState *tstate,

_PyInterpreterFrame *frame,

_Py_CODEUNIT *instr,

PyObject *value);

static void monitor_unwind(PyThreadState *tstate,

_PyInterpreterFrame *frame,

_Py_CODEUNIT *instr);

static int monitor_handled(PyThreadState *tstate,

_PyInterpreterFrame *frame,

_Py_CODEUNIT *instr, PyObject *exc);

static void monitor_throw(PyThreadState *tstate,

_PyInterpreterFrame *frame,

_Py_CODEUNIT *instr);

static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);

static _PyInterpreterFrame *

_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,

PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

int

Py_GetRecursionLimit(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

return interp->ceval.recursion_limit;

}

void

Py_SetRecursionLimit(int new_limit)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

_PyEval_StopTheWorld(interp);

interp->ceval.recursion_limit = new_limit;

_Py_FOR_EACH_TSTATE_BEGIN(interp, p) {

int depth = p->py_recursion_limit - p->py_recursion_remaining;

p->py_recursion_limit = new_limit;

p->py_recursion_remaining = new_limit - depth;

}

_Py_FOR_EACH_TSTATE_END(interp);

_PyEval_StartTheWorld(interp);

}

int

_Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count)

{

uintptr_t here_addr = _Py_get_machine_stack_pointer();

_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;

if (here_addr > _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES) {

return 0;

}

if (_tstate->c_stack_hard_limit == 0) {

_Py_InitializeRecursionLimits(tstate);

}

return here_addr <= _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES;

}

void

_Py_EnterRecursiveCallUnchecked(PyThreadState *tstate)

{

uintptr_t here_addr = _Py_get_machine_stack_pointer();

_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;

if (here_addr < _tstate->c_stack_hard_limit) {

Py_FatalError("Unchecked stack overflow.");

}

}

#if defined(__s390x__)

# define Py_C_STACK_SIZE 320000

#elif defined(_WIN32)

// Don't define Py_C_STACK_SIZE, ask the O/S

#elif defined(__ANDROID__)

# define Py_C_STACK_SIZE 1200000

#elif defined(__sparc__)

# define Py_C_STACK_SIZE 1600000

#elif defined(__hppa__) || defined(__powerpc64__)

# define Py_C_STACK_SIZE 2000000

#else

# define Py_C_STACK_SIZE 4000000

#endif

#if defined(__EMSCRIPTEN__)

// Temporary workaround to make `pthread_getattr_np` work on Emscripten.

// Emscripten 4.0.6 will contain a fix:

// https://github.com/emscripten-core/emscripten/pull/23887

#include "emscripten/stack.h"

#define pthread_attr_t workaround_pthread_attr_t

#define pthread_getattr_np workaround_pthread_getattr_np

#define pthread_attr_getguardsize workaround_pthread_attr_getguardsize

#define pthread_attr_getstack workaround_pthread_attr_getstack

#define pthread_attr_destroy workaround_pthread_attr_destroy

typedef struct {

void *_a_stackaddr;

size_t _a_stacksize, _a_guardsize;

} pthread_attr_t;

extern __attribute__((__visibility__("hidden"))) unsigned __default_guardsize;

// Modified version of pthread_getattr_np from the upstream PR.

int pthread_getattr_np(pthread_t thread, pthread_attr_t *attr) {

attr->_a_stackaddr = (void*)emscripten_stack_get_base();

attr->_a_stacksize = emscripten_stack_get_base() - emscripten_stack_get_end();

attr->_a_guardsize = __default_guardsize;

return 0;

}

// These three functions copied without any changes from Emscripten libc.

int pthread_attr_getguardsize(const pthread_attr_t *restrict a, size_t *restrict size)

{

*size = a->_a_guardsize;

return 0;

}

int pthread_attr_getstack(const pthread_attr_t *restrict a, void **restrict addr, size_t *restrict size)

{

/// XXX musl is not standard-conforming? It should not report EINVAL if _a_stackaddr is zero, and it should

/// report EINVAL if a is null: http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_attr_getstack.html

if (!a) return EINVAL;

// if (!a->_a_stackaddr)

// return EINVAL;

*size = a->_a_stacksize;

*addr = (void *)(a->_a_stackaddr - *size);

return 0;

}

int pthread_attr_destroy(pthread_attr_t *a)

{

return 0;

}

#endif

void

_Py_InitializeRecursionLimits(PyThreadState *tstate)

{

_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;

#ifdef WIN32

ULONG_PTR low, high;

GetCurrentThreadStackLimits(&low, &high);

_tstate->c_stack_top = (uintptr_t)high;

ULONG guarantee = 0;

SetThreadStackGuarantee(&guarantee);

_tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + PYOS_STACK_MARGIN_BYTES;

_tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + PYOS_STACK_MARGIN_BYTES;

#else

uintptr_t here_addr = _Py_get_machine_stack_pointer();

# if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && !defined(__NetBSD__)

size_t stack_size, guard_size;

void *stack_addr;

pthread_attr_t attr;

int err = pthread_getattr_np(pthread_self(), &attr);

if (err == 0) {

err = pthread_attr_getguardsize(&attr, &guard_size);

err |= pthread_attr_getstack(&attr, &stack_addr, &stack_size);

err |= pthread_attr_destroy(&attr);

}

if (err == 0) {

uintptr_t base = ((uintptr_t)stack_addr) + guard_size;

_tstate->c_stack_top = base + stack_size;

#ifdef _Py_THREAD_SANITIZER

// Thread sanitizer crashes if we use a bit more than half the stack.

_tstate->c_stack_soft_limit = base + (stack_size / 2);

#else

_tstate->c_stack_soft_limit = base + PYOS_STACK_MARGIN_BYTES * 2;

#endif

_tstate->c_stack_hard_limit = base + PYOS_STACK_MARGIN_BYTES;

assert(_tstate->c_stack_soft_limit < here_addr);

assert(here_addr < _tstate->c_stack_top);

return;

}

# endif

_tstate->c_stack_top = _Py_SIZE_ROUND_UP(here_addr, 4096);

_tstate->c_stack_soft_limit = _tstate->c_stack_top - Py_C_STACK_SIZE;

_tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + PYOS_STACK_MARGIN_BYTES);

#endif

}

/* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall()

if the recursion_depth reaches recursion_limit. */

int

_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)

{

_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;

uintptr_t here_addr = _Py_get_machine_stack_pointer();

assert(_tstate->c_stack_soft_limit != 0);

assert(_tstate->c_stack_hard_limit != 0);

if (here_addr < _tstate->c_stack_hard_limit) {

/* Overflowing while handling an overflow. Give up. */

int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024;

char buffer[80];

snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where);

Py_FatalError(buffer);

}

if (tstate->recursion_headroom) {

return 0;

}

else {

int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024;

tstate->recursion_headroom++;

_PyErr_Format(tstate, PyExc_RecursionError,

"Stack overflow (used %d kB)%s",

kbytes_used,

where);

tstate->recursion_headroom--;

return -1;

}

}

const binaryfunc _PyEval_BinaryOps[] = {

[NB_ADD] = PyNumber_Add,

[NB_AND] = PyNumber_And,

[NB_FLOOR_DIVIDE] = PyNumber_FloorDivide,

[NB_LSHIFT] = PyNumber_Lshift,

[NB_MATRIX_MULTIPLY] = PyNumber_MatrixMultiply,

[NB_MULTIPLY] = PyNumber_Multiply,

[NB_REMAINDER] = PyNumber_Remainder,

[NB_OR] = PyNumber_Or,

[NB_POWER] = _PyNumber_PowerNoMod,

[NB_RSHIFT] = PyNumber_Rshift,

[NB_SUBTRACT] = PyNumber_Subtract,

[NB_TRUE_DIVIDE] = PyNumber_TrueDivide,

[NB_XOR] = PyNumber_Xor,

[NB_INPLACE_ADD] = PyNumber_InPlaceAdd,

[NB_INPLACE_AND] = PyNumber_InPlaceAnd,

[NB_INPLACE_FLOOR_DIVIDE] = PyNumber_InPlaceFloorDivide,

[NB_INPLACE_LSHIFT] = PyNumber_InPlaceLshift,

[NB_INPLACE_MATRIX_MULTIPLY] = PyNumber_InPlaceMatrixMultiply,

[NB_INPLACE_MULTIPLY] = PyNumber_InPlaceMultiply,

[NB_INPLACE_REMAINDER] = PyNumber_InPlaceRemainder,

[NB_INPLACE_OR] = PyNumber_InPlaceOr,

[NB_INPLACE_POWER] = _PyNumber_InPlacePowerNoMod,

[NB_INPLACE_RSHIFT] = PyNumber_InPlaceRshift,

[NB_INPLACE_SUBTRACT] = PyNumber_InPlaceSubtract,

[NB_INPLACE_TRUE_DIVIDE] = PyNumber_InPlaceTrueDivide,

[NB_INPLACE_XOR] = PyNumber_InPlaceXor,

[NB_SUBSCR] = PyObject_GetItem,

};

const conversion_func _PyEval_ConversionFuncs[4] = {

[FVC_STR] = PyObject_Str,

[FVC_REPR] = PyObject_Repr,

[FVC_ASCII] = PyObject_ASCII

};

const _Py_SpecialMethod _Py_SpecialMethods[] = {

[SPECIAL___ENTER__] = {

.name = &_Py_ID(__enter__),

.error = (

"'%T' object does not support the context manager protocol "

"(missed __enter__ method)"

),

.error_suggestion = (

"'%T' object does not support the context manager protocol "

"(missed __enter__ method) but it supports the asynchronous "

"context manager protocol. Did you mean to use 'async with'?"

)

},

[SPECIAL___EXIT__] = {

.name = &_Py_ID(__exit__),

.error = (

"'%T' object does not support the context manager protocol "

"(missed __exit__ method)"

),

.error_suggestion = (

"'%T' object does not support the context manager protocol "

"(missed __exit__ method) but it supports the asynchronous "

"context manager protocol. Did you mean to use 'async with'?"

)

},

[SPECIAL___AENTER__] = {

.name = &_Py_ID(__aenter__),

.error = (

"'%T' object does not support the asynchronous "

"context manager protocol (missed __aenter__ method)"

),

.error_suggestion = (

"'%T' object does not support the asynchronous context manager "

"protocol (missed __aenter__ method) but it supports the context "

"manager protocol. Did you mean to use 'with'?"

)

},

[SPECIAL___AEXIT__] = {

.name = &_Py_ID(__aexit__),

.error = (

"'%T' object does not support the asynchronous "

"context manager protocol (missed __aexit__ method)"

),

.error_suggestion = (

"'%T' object does not support the asynchronous context manager "

"protocol (missed __aexit__ method) but it supports the context "

"manager protocol. Did you mean to use 'with'?"

)

}

};

const size_t _Py_FunctionAttributeOffsets[] = {

[MAKE_FUNCTION_CLOSURE] = offsetof(PyFunctionObject, func_closure),

[MAKE_FUNCTION_ANNOTATIONS] = offsetof(PyFunctionObject, func_annotations),

[MAKE_FUNCTION_KWDEFAULTS] = offsetof(PyFunctionObject, func_kwdefaults),

[MAKE_FUNCTION_DEFAULTS] = offsetof(PyFunctionObject, func_defaults),

[MAKE_FUNCTION_ANNOTATE] = offsetof(PyFunctionObject, func_annotate),

};

// PEP 634: Structural Pattern Matching

// Return a tuple of values corresponding to keys, with error checks for

// duplicate/missing keys.

PyObject *

_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys)

{

assert(PyTuple_CheckExact(keys));

Py_ssize_t nkeys = PyTuple_GET_SIZE(keys);

if (!nkeys) {

// No keys means no items.

return PyTuple_New(0);

}

PyObject *seen = NULL;

PyObject *dummy = NULL;

PyObject *values = NULL;

PyObject *get = NULL;

// We use the two argument form of map.get(key, default) for two reasons:

// - Atomically check for a key and get its value without error handling.

// - Don't cause key creation or resizing in dict subclasses like

// collections.defaultdict that define __missing__ (or similar).

int meth_found = _PyObject_GetMethod(map, &_Py_ID(get), &get);

if (get == NULL) {

goto fail;

}

seen = PySet_New(NULL);

if (seen == NULL) {

goto fail;

}

// dummy = object()

dummy = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type);

if (dummy == NULL) {

goto fail;

}

values = PyTuple_New(nkeys);

if (values == NULL) {

goto fail;

}

for (Py_ssize_t i = 0; i < nkeys; i++) {

PyObject *key = PyTuple_GET_ITEM(keys, i);

if (PySet_Contains(seen, key) || PySet_Add(seen, key)) {

if (!_PyErr_Occurred(tstate)) {

// Seen it before!

_PyErr_Format(tstate, PyExc_ValueError,

"mapping pattern checks duplicate key (%R)", key);

}

goto fail;

}

PyObject *args[] = { map, key, dummy };

PyObject *value = NULL;

if (meth_found) {

value = PyObject_Vectorcall(get, args, 3, NULL);

}

else {

value = PyObject_Vectorcall(get, &args[1], 2, NULL);

}

if (value == NULL) {

goto fail;

}

if (value == dummy) {

// key not in map!

Py_DECREF(value);

Py_DECREF(values);

// Return None:

values = Py_NewRef(Py_None);

goto done;

}

PyTuple_SET_ITEM(values, i, value);

}

// Success:

done:

Py_DECREF(get);

Py_DECREF(seen);

Py_DECREF(dummy);

return values;

fail:

Py_XDECREF(get);

Py_XDECREF(seen);

Py_XDECREF(dummy);

Py_XDECREF(values);

return NULL;

}

// Extract a named attribute from the subject, with additional bookkeeping to

// raise TypeErrors for repeated lookups. On failure, return NULL (with no

// error set). Use _PyErr_Occurred(tstate) to disambiguate.

static PyObject *

match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,

PyObject *name, PyObject *seen)

{

assert(PyUnicode_CheckExact(name));

assert(PySet_CheckExact(seen));

if (PySet_Contains(seen, name) || PySet_Add(seen, name)) {

if (!_PyErr_Occurred(tstate)) {

// Seen it before!

_PyErr_Format(tstate, PyExc_TypeError,

"%s() got multiple sub-patterns for attribute %R",

((PyTypeObject*)type)->tp_name, name);

}

return NULL;

}

PyObject *attr;

(void)PyObject_GetOptionalAttr(subject, name, &attr);

return attr;

}

// On success (match), return a tuple of extracted attributes. On failure (no

// match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.

PyObject*

_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type,

Py_ssize_t nargs, PyObject *kwargs)

{

if (!PyType_Check(type)) {

const char *e = "called match pattern must be a class";

_PyErr_Format(tstate, PyExc_TypeError, e);

return NULL;

}

assert(PyTuple_CheckExact(kwargs));

// First, an isinstance check:

if (PyObject_IsInstance(subject, type) <= 0) {

return NULL;

}

// So far so good:

PyObject *seen = PySet_New(NULL);

if (seen == NULL) {

return NULL;

}

PyObject *attrs = PyList_New(0);

if (attrs == NULL) {

Py_DECREF(seen);

return NULL;

}

// NOTE: From this point on, goto fail on failure:

PyObject *match_args = NULL;

// First, the positional subpatterns:

if (nargs) {

int match_self = 0;

if (PyObject_GetOptionalAttr(type, &_Py_ID(__match_args__), &match_args) < 0) {

goto fail;

}

if (match_args) {

if (!PyTuple_CheckExact(match_args)) {

const char *e = "%s.__match_args__ must be a tuple (got %s)";

_PyErr_Format(tstate, PyExc_TypeError, e,

((PyTypeObject *)type)->tp_name,

Py_TYPE(match_args)->tp_name);

goto fail;

}

}

else {

// _Py_TPFLAGS_MATCH_SELF is only acknowledged if the type does not

// define __match_args__. This is natural behavior for subclasses:

// it's as if __match_args__ is some "magic" value that is lost as

// soon as they redefine it.

match_args = PyTuple_New(0);

match_self = PyType_HasFeature((PyTypeObject*)type,

_Py_TPFLAGS_MATCH_SELF);

}

assert(PyTuple_CheckExact(match_args));

Py_ssize_t allowed = match_self ? 1 : PyTuple_GET_SIZE(match_args);

if (allowed < nargs) {

const char *plural = (allowed == 1) ? "" : "s";

_PyErr_Format(tstate, PyExc_TypeError,

"%s() accepts %d positional sub-pattern%s (%d given)",

((PyTypeObject*)type)->tp_name,

allowed, plural, nargs);

goto fail;

}

if (match_self) {

// Easy. Copy the subject itself, and move on to kwargs.

if (PyList_Append(attrs, subject) < 0) {

goto fail;

}

}

else {

for (Py_ssize_t i = 0; i < nargs; i++) {

PyObject *name = PyTuple_GET_ITEM(match_args, i);

if (!PyUnicode_CheckExact(name)) {

_PyErr_Format(tstate, PyExc_TypeError,

"__match_args__ elements must be strings "

"(got %s)", Py_TYPE(name)->tp_name);

goto fail;

}

PyObject *attr = match_class_attr(tstate, subject, type, name,

seen);

if (attr == NULL) {

goto fail;

}

if (PyList_Append(attrs, attr) < 0) {

Py_DECREF(attr);

goto fail;

}

Py_DECREF(attr);

}

}

Py_CLEAR(match_args);

}

// Finally, the keyword subpatterns:

for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwargs); i++) {

PyObject *name = PyTuple_GET_ITEM(kwargs, i);

PyObject *attr = match_class_attr(tstate, subject, type, name, seen);

if (attr == NULL) {

goto fail;

}

if (PyList_Append(attrs, attr) < 0) {

Py_DECREF(attr);

goto fail;

}

Py_DECREF(attr);

}

Py_SETREF(attrs, PyList_AsTuple(attrs));

Py_DECREF(seen);

return attrs;

fail:

// We really don't care whether an error was raised or not... that's our

// caller's problem. All we know is that the match failed.

Py_XDECREF(match_args);

Py_DECREF(seen);

Py_DECREF(attrs);

return NULL;

}

static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);

PyObject *

PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)

{

PyThreadState *tstate = _PyThreadState_GET();

if (locals == NULL) {

locals = globals;

}

PyObject *builtins = _PyDict_LoadBuiltinsFromGlobals(globals);

if (builtins == NULL) {

return NULL;

}

PyFrameConstructor desc = {

.fc_globals = globals,

.fc_builtins = builtins,

.fc_name = ((PyCodeObject *)co)->co_name,

.fc_qualname = ((PyCodeObject *)co)->co_name,

.fc_code = co,

.fc_defaults = NULL,

.fc_kwdefaults = NULL,

.fc_closure = NULL

};

PyFunctionObject *func = _PyFunction_FromConstructor(&desc);

_Py_DECREF_BUILTINS(builtins);

if (func == NULL) {

return NULL;

}

EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY);

PyObject *res = _PyEval_Vector(tstate, func, locals, NULL, 0, NULL);

Py_DECREF(func);

return res;

}

/* Interpreter main loop */

PyObject *

PyEval_EvalFrame(PyFrameObject *f)

{

/* Function kept for backward compatibility */

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f->f_frame, 0);

}

PyObject *

PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)

{

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f->f_frame, throwflag);

}

#include "ceval_macros.h"

int _Py_CheckRecursiveCallPy(

PyThreadState *tstate)

{

if (tstate->recursion_headroom) {

if (tstate->py_recursion_remaining < -50) {

/* Overflowing while handling an overflow. Give up. */

Py_FatalError("Cannot recover from Python stack overflow.");

}

}

else {

if (tstate->py_recursion_remaining <= 0) {

tstate->recursion_headroom++;

_PyErr_Format(tstate, PyExc_RecursionError,

"maximum recursion depth exceeded");

tstate->recursion_headroom--;

return -1;

}

}

return 0;

}

static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {

/* Put a NOP at the start, so that the IP points into

* the code, rather than before it */

{ .op.code = NOP, .op.arg = 0 },

{ .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */

{ .op.code = NOP, .op.arg = 0 },

{ .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */

{ .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START }

};

#ifdef Py_DEBUG

extern void _PyUOpPrint(const _PyUOpInstruction *uop);

#endif

/* Disable unused label warnings. They are handy for debugging, even

if computed gotos aren't used. */

/* TBD - what about other compilers? */

#if defined(__GNUC__) || defined(__clang__)

# pragma GCC diagnostic push

# pragma GCC diagnostic ignored "-Wunused-label"

#elif defined(_MSC_VER) /* MS_WINDOWS */

# pragma warning(push)

# pragma warning(disable:4102)

#endif

PyObject **

_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch)

{

PyObject **result;

if (nargs > MAX_STACKREF_SCRATCH) {

// +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set.

result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *));

if (result == NULL) {

return NULL;

}

result++;

}

else {

result = scratch;

}

for (int i = 0; i < nargs; i++) {

result[i] = PyStackRef_AsPyObjectBorrow(input[i]);

}

return result;

}

void

_PyObjectArray_Free(PyObject **array, PyObject **scratch)

{

if (array != scratch) {

PyMem_Free(array);

}

}

/* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC.

*/

#if (defined(_MSC_VER) && \

(_MSC_VER < 1943) && \

defined(_Py_USING_PGO))

#define DO_NOT_OPTIMIZE_INTERP_LOOP

#endif

#ifdef DO_NOT_OPTIMIZE_INTERP_LOOP

# pragma optimize("t", off)

/* This setting is reversed below following _PyEval_EvalFrameDefault */

#endif

#if Py_TAIL_CALL_INTERP

#include "opcode_targets.h"

#include "generated_cases.c.h"

#endif

#if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__)

/*

* gh-129987: The SLP autovectorizer can cause poor code generation for

* opcode dispatch in some GCC versions (observed in GCCs 12 through 15,

* probably caused by https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115777),

* negating any benefit we get from vectorization elsewhere in the

* interpreter loop. Disabling it significantly affected older GCC versions

* (prior to GCC 9, 40% performance drop), so we have to selectively disable

* it.

*/

#define DONT_SLP_VECTORIZE __attribute__((optimize ("no-tree-slp-vectorize")))

#else

#define DONT_SLP_VECTORIZE