/* Execute compiled code */

/* XXX TO DO:

XXX speed up searching for keywords by using a dictionary

XXX document it!

*/

/* enable more aggressive intra-module optimizations, where available */

#define PY_LOCAL_AGGRESSIVE

#include "Python.h"

#include "pycore_call.h"

#include "pycore_ceval.h"

#include "pycore_code.h"

#include "pycore_object.h"

#include "pycore_pyerrors.h"

#include "pycore_pylifecycle.h"

#include "pycore_pystate.h"

#include "pycore_tupleobject.h"

#include "code.h"

#include "dictobject.h"

#include "frameobject.h"

#include "opcode.h"

#include "pydtrace.h"

#include "setobject.h"

#include "structmember.h"

#include <ctype.h>

#ifdef Py_DEBUG

/* For debugging the interpreter: */

#define LLTRACE 1 /* Low-level trace feature */

#define CHECKEXC 1 /* Double-check exception checking */

#endif

#if !defined(Py_BUILD_CORE)

# error "ceval.c must be build with Py_BUILD_CORE define for best performance"

#endif

/* Forward declarations */

Py_LOCAL_INLINE(PyObject *) call_function(

PyThreadState *tstate, PyObject ***pp_stack,

Py_ssize_t oparg, PyObject *kwnames);

static PyObject * do_call_core(

PyThreadState *tstate, PyObject *func,

PyObject *callargs, PyObject *kwdict);

#ifdef LLTRACE

static int lltrace;

static int prtrace(PyThreadState *, PyObject *, const char *);

#endif

static int call_trace(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *,

int, PyObject *);

static int call_trace_protected(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *,

int, PyObject *);

static void call_exc_trace(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *);

static int maybe_call_line_trace(Py_tracefunc, PyObject *,

PyThreadState *, PyFrameObject *,

int *, int *, int *);

static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *);

static void dtrace_function_entry(PyFrameObject *);

static void dtrace_function_return(PyFrameObject *);

static PyObject * cmp_outcome(PyThreadState *, int, PyObject *, PyObject *);

static PyObject * import_name(PyThreadState *, PyFrameObject *,

PyObject *, PyObject *, PyObject *);

static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);

static int import_all_from(PyThreadState *, PyObject *, PyObject *);

static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);

static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);

static PyObject * unicode_concatenate(PyThreadState *, PyObject *, PyObject *,

PyFrameObject *, const _Py_CODEUNIT *);

static PyObject * special_lookup(PyThreadState *, PyObject *, _Py_Identifier *);

static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);

static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);

static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int);

#define NAME_ERROR_MSG \

"name '%.200s' is not defined"

#define UNBOUNDLOCAL_ERROR_MSG \

"local variable '%.200s' referenced before assignment"

#define UNBOUNDFREE_ERROR_MSG \

"free variable '%.200s' referenced before assignment" \

" in enclosing scope"

/* Dynamic execution profile */

#ifdef DYNAMIC_EXECUTION_PROFILE

#ifdef DXPAIRS

static long dxpairs[257][256];

#define dxp dxpairs[256]

#else

static long dxp[256];

#endif

#endif

/* per opcode cache */

#ifdef Py_DEBUG

// --with-pydebug is used to find memory leak. opcache makes it harder.

// So we disable opcache when Py_DEBUG is defined.

// See bpo-37146

#define OPCACHE_MIN_RUNS 0 /* disable opcache */

#else

#define OPCACHE_MIN_RUNS 1024 /* create opcache when code executed this time */

#endif

#define OPCACHE_STATS 0 /* Enable stats */

#if OPCACHE_STATS

static size_t opcache_code_objects = 0;

static size_t opcache_code_objects_extra_mem = 0;

static size_t opcache_global_opts = 0;

static size_t opcache_global_hits = 0;

static size_t opcache_global_misses = 0;

#endif

#define GIL_REQUEST _Py_atomic_load_relaxed(&ceval->gil_drop_request)

/* This can set eval_breaker to 0 even though gil_drop_request became

1. We believe this is all right because the eval loop will release

the GIL eventually anyway. */

#define COMPUTE_EVAL_BREAKER(ceval) \

_Py_atomic_store_relaxed( \

&(ceval)->eval_breaker, \

GIL_REQUEST | \

_Py_atomic_load_relaxed(&(ceval)->signals_pending) | \

_Py_atomic_load_relaxed(&(ceval)->pending.calls_to_do) | \

(ceval)->pending.async_exc)

#define SET_GIL_DROP_REQUEST(ceval) \

do { \

_Py_atomic_store_relaxed(&(ceval)->gil_drop_request, 1); \

_Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \

} while (0)

#define RESET_GIL_DROP_REQUEST(ceval) \

do { \

_Py_atomic_store_relaxed(&(ceval)->gil_drop_request, 0); \

COMPUTE_EVAL_BREAKER(ceval); \

} while (0)

/* Pending calls are only modified under pending_lock */

#define SIGNAL_PENDING_CALLS(ceval) \

do { \

_Py_atomic_store_relaxed(&(ceval)->pending.calls_to_do, 1); \

_Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \

} while (0)

#define UNSIGNAL_PENDING_CALLS(ceval) \

do { \

_Py_atomic_store_relaxed(&(ceval)->pending.calls_to_do, 0); \

COMPUTE_EVAL_BREAKER(ceval); \

} while (0)

#define SIGNAL_PENDING_SIGNALS(ceval) \

do { \

_Py_atomic_store_relaxed(&(ceval)->signals_pending, 1); \

_Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \

} while (0)

#define UNSIGNAL_PENDING_SIGNALS(ceval) \

do { \

_Py_atomic_store_relaxed(&(ceval)->signals_pending, 0); \

COMPUTE_EVAL_BREAKER(ceval); \

} while (0)

#define SIGNAL_ASYNC_EXC(ceval) \

do { \

(ceval)->pending.async_exc = 1; \

_Py_atomic_store_relaxed(&(ceval)->eval_breaker, 1); \

} while (0)

#define UNSIGNAL_ASYNC_EXC(ceval) \

do { \

(ceval)->pending.async_exc = 0; \

COMPUTE_EVAL_BREAKER(ceval); \

} while (0)

#ifdef HAVE_ERRNO_H

#include <errno.h>

#endif

#include "pythread.h"

#include "ceval_gil.h"

int

PyEval_ThreadsInitialized(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

return gil_created(&runtime->ceval.gil);

}

void

PyEval_InitThreads(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

struct _gil_runtime_state *gil = &ceval->gil;

if (gil_created(gil)) {

return;

}

PyThread_init_thread();

create_gil(gil);

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

take_gil(ceval, tstate);

struct _pending_calls *pending = &ceval->pending;

pending->lock = PyThread_allocate_lock();

if (pending->lock == NULL) {

Py_FatalError("Can't initialize threads for pending calls");

}

}

void

_PyEval_FiniThreads(struct _ceval_runtime_state *ceval)

{

struct _gil_runtime_state *gil = &ceval->gil;

if (!gil_created(gil)) {

return;

}

destroy_gil(gil);

assert(!gil_created(gil));

struct _pending_calls *pending = &ceval->pending;

if (pending->lock != NULL) {

PyThread_free_lock(pending->lock);

pending->lock = NULL;

}

}

static inline void

exit_thread_if_finalizing(PyThreadState *tstate)

{

_PyRuntimeState *runtime = tstate->interp->runtime;

/* _Py_Finalizing is protected by the GIL */

if (runtime->finalizing != NULL && !_Py_CURRENTLY_FINALIZING(runtime, tstate)) {

drop_gil(&runtime->ceval, tstate);

PyThread_exit_thread();

}

}

void

_PyEval_Fini(void)

{

#if OPCACHE_STATS

fprintf(stderr, "-- Opcode cache number of objects = %zd\n",

opcache_code_objects);

fprintf(stderr, "-- Opcode cache total extra mem = %zd\n",

opcache_code_objects_extra_mem);

fprintf(stderr, "\n");

fprintf(stderr, "-- Opcode cache LOAD_GLOBAL hits = %zd (%d%%)\n",

opcache_global_hits,

(int) (100.0 * opcache_global_hits /

(opcache_global_hits + opcache_global_misses)));

fprintf(stderr, "-- Opcode cache LOAD_GLOBAL misses = %zd (%d%%)\n",

opcache_global_misses,

(int) (100.0 * opcache_global_misses /

(opcache_global_hits + opcache_global_misses)));

fprintf(stderr, "-- Opcode cache LOAD_GLOBAL opts = %zd\n",

opcache_global_opts);

fprintf(stderr, "\n");

#endif

}

void

PyEval_AcquireLock(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

if (tstate == NULL) {

Py_FatalError("PyEval_AcquireLock: current thread state is NULL");

}

take_gil(ceval, tstate);

exit_thread_if_finalizing(tstate);

}

void

PyEval_ReleaseLock(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

/* This function must succeed when the current thread state is NULL.

We therefore avoid PyThreadState_Get() which dumps a fatal error

in debug mode.

*/

drop_gil(&runtime->ceval, tstate);

}

void

PyEval_AcquireThread(PyThreadState *tstate)

{

if (tstate == NULL) {

Py_FatalError("PyEval_AcquireThread: NULL new thread state");

}

_PyRuntimeState *runtime = tstate->interp->runtime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

/* Check someone has called PyEval_InitThreads() to create the lock */

assert(gil_created(&ceval->gil));

take_gil(ceval, tstate);

exit_thread_if_finalizing(tstate);

if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {

Py_FatalError("PyEval_AcquireThread: non-NULL old thread state");

}

}

void

PyEval_ReleaseThread(PyThreadState *tstate)

{

if (tstate == NULL) {

Py_FatalError("PyEval_ReleaseThread: NULL thread state");

}

_PyRuntimeState *runtime = tstate->interp->runtime;

PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);

if (new_tstate != tstate) {

Py_FatalError("PyEval_ReleaseThread: wrong thread state");

}

drop_gil(&runtime->ceval, tstate);

}

/* This function is called from PyOS_AfterFork_Child to destroy all threads

* which are not running in the child process, and clear internal locks

* which might be held by those threads.

*/

void

_PyEval_ReInitThreads(_PyRuntimeState *runtime)

{

struct _ceval_runtime_state *ceval = &runtime->ceval;

if (!gil_created(&ceval->gil)) {

return;

}

recreate_gil(&ceval->gil);

PyThreadState *current_tstate = _PyRuntimeState_GetThreadState(runtime);

take_gil(ceval, current_tstate);

struct _pending_calls *pending = &ceval->pending;

pending->lock = PyThread_allocate_lock();

if (pending->lock == NULL) {

Py_FatalError("Can't initialize threads for pending calls");

}

/* Destroy all threads except the current one */

_PyThreadState_DeleteExcept(runtime, current_tstate);

}

/* This function is used to signal that async exceptions are waiting to be

raised. */

void

_PyEval_SignalAsyncExc(struct _ceval_runtime_state *ceval)

{

SIGNAL_ASYNC_EXC(ceval);

}

PyThreadState *

PyEval_SaveThread(void)

{

_PyRuntimeState *runtime = &_PyRuntime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);

if (tstate == NULL) {

Py_FatalError("PyEval_SaveThread: NULL tstate");

}

assert(gil_created(&ceval->gil));

drop_gil(ceval, tstate);

return tstate;

}

void

PyEval_RestoreThread(PyThreadState *tstate)

{

_PyRuntimeState *runtime = tstate->interp->runtime;

struct _ceval_runtime_state *ceval = &runtime->ceval;

if (tstate == NULL) {

Py_FatalError("PyEval_RestoreThread: NULL tstate");

}

assert(gil_created(&ceval->gil));

int err = errno;

take_gil(ceval, tstate);

exit_thread_if_finalizing(tstate);

errno = err;

_PyThreadState_Swap(&runtime->gilstate, tstate);

}

/* Mechanism whereby asynchronously executing callbacks (e.g. UNIX

signal handlers or Mac I/O completion routines) can schedule calls

to a function to be called synchronously.

The synchronous function is called with one void* argument.

It should return 0 for success or -1 for failure -- failure should

be accompanied by an exception.

If registry succeeds, the registry function returns 0; if it fails

(e.g. due to too many pending calls) it returns -1 (without setting

an exception condition).

Note that because registry may occur from within signal handlers,

or other asynchronous events, calling malloc() is unsafe!

Any thread can schedule pending calls, but only the main thread

will execute them.

There is no facility to schedule calls to a particular thread, but

that should be easy to change, should that ever be required. In

that case, the static variables here should go into the python

threadstate.

*/

void

_PyEval_SignalReceived(struct _ceval_runtime_state *ceval)

{

/* bpo-30703: Function called when the C signal handler of Python gets a

signal. We cannot queue a callback using Py_AddPendingCall() since

that function is not async-signal-safe. */

SIGNAL_PENDING_SIGNALS(ceval);

}

/* Push one item onto the queue while holding the lock. */

static int

_push_pending_call(struct _pending_calls *pending,

int (*func)(void *), void *arg)

{

int i = pending->last;

int j = (i + 1) % NPENDINGCALLS;

if (j == pending->first) {

return -1; /* Queue full */

}

pending->calls[i].func = func;

pending->calls[i].arg = arg;

pending->last = j;

return 0;

}

/* Pop one item off the queue while holding the lock. */

static void

_pop_pending_call(struct _pending_calls *pending,

int (**func)(void *), void **arg)

{

int i = pending->first;

if (i == pending->last) {

return; /* Queue empty */

}

*func = pending->calls[i].func;

*arg = pending->calls[i].arg;

pending->first = (i + 1) % NPENDINGCALLS;

}

/* This implementation is thread-safe. It allows

scheduling to be made from any thread, and even from an executing

callback.

*/

int

_PyEval_AddPendingCall(PyThreadState *tstate,

struct _ceval_runtime_state *ceval,

int (*func)(void *), void *arg)

{

struct _pending_calls *pending = &ceval->pending;

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

if (pending->finishing) {

PyThread_release_lock(pending->lock);

PyObject *exc, *val, *tb;

_PyErr_Fetch(tstate, &exc, &val, &tb);

_PyErr_SetString(tstate, PyExc_SystemError,

"Py_AddPendingCall: cannot add pending calls "

"(Python shutting down)");

_PyErr_Print(tstate);

_PyErr_Restore(tstate, exc, val, tb);

return -1;

}

int result = _push_pending_call(pending, func, arg);

PyThread_release_lock(pending->lock);

/* signal main loop */

SIGNAL_PENDING_CALLS(ceval);

return result;

}

int

Py_AddPendingCall(int (*func)(void *), void *arg)

{

_PyRuntimeState *runtime = &_PyRuntime;

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

return _PyEval_AddPendingCall(tstate, &runtime->ceval, func, arg);

}

static int

handle_signals(_PyRuntimeState *runtime)

{

/* Only handle signals on main thread. PyEval_InitThreads must

* have been called already.

*/

if (PyThread_get_thread_ident() != runtime->main_thread) {

return 0;

}

/*

* Ensure that the thread isn't currently running some other

* interpreter.

*/

PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp;

if (interp != runtime->interpreters.main) {

return 0;

}

struct _ceval_runtime_state *ceval = &runtime->ceval;

UNSIGNAL_PENDING_SIGNALS(ceval);

if (_PyErr_CheckSignals() < 0) {

SIGNAL_PENDING_SIGNALS(ceval); /* We're not done yet */

return -1;

}

return 0;

}

static int

make_pending_calls(_PyRuntimeState *runtime)

{

static int busy = 0;

/* only service pending calls on main thread */

if (PyThread_get_thread_ident() != runtime->main_thread) {

return 0;

}

/* don't perform recursive pending calls */

if (busy) {

return 0;

}

busy = 1;

struct _ceval_runtime_state *ceval = &runtime->ceval;

/* unsignal before starting to call callbacks, so that any callback

added in-between re-signals */

UNSIGNAL_PENDING_CALLS(ceval);

int res = 0;

/* perform a bounded number of calls, in case of recursion */

struct _pending_calls *pending = &ceval->pending;

for (int i=0; i<NPENDINGCALLS; i++) {

int (*func)(void *) = NULL;

void *arg = NULL;

/* pop one item off the queue while holding the lock */

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

_pop_pending_call(pending, &func, &arg);

PyThread_release_lock(pending->lock);

/* having released the lock, perform the callback */

if (func == NULL) {

break;

}

res = func(arg);

if (res) {

goto error;

}

}

busy = 0;

return res;

error:

busy = 0;

SIGNAL_PENDING_CALLS(ceval);

return res;

}

void

_Py_FinishPendingCalls(_PyRuntimeState *runtime)

{

assert(PyGILState_Check());

PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);

struct _pending_calls *pending = &runtime->ceval.pending;

PyThread_acquire_lock(pending->lock, WAIT_LOCK);

pending->finishing = 1;

PyThread_release_lock(pending->lock);

if (!_Py_atomic_load_relaxed(&(pending->calls_to_do))) {

return;

}

if (make_pending_calls(runtime) < 0) {

PyObject *exc, *val, *tb;

_PyErr_Fetch(tstate, &exc, &val, &tb);

PyErr_BadInternalCall();

_PyErr_ChainExceptions(exc, val, tb);

_PyErr_Print(tstate);

}

}

/* Py_MakePendingCalls() is a simple wrapper for the sake

of backward-compatibility. */

int

Py_MakePendingCalls(void)

{

assert(PyGILState_Check());

/* Python signal handler doesn't really queue a callback: it only signals

that a signal was received, see _PyEval_SignalReceived(). */

_PyRuntimeState *runtime = &_PyRuntime;

int res = handle_signals(runtime);

if (res != 0) {

return res;

}

res = make_pending_calls(runtime);

if (res != 0) {

return res;

}

return 0;

}

/* The interpreter's recursion limit */

#ifndef Py_DEFAULT_RECURSION_LIMIT

#define Py_DEFAULT_RECURSION_LIMIT 1000

#endif

int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;

void

_PyEval_Initialize(struct _ceval_runtime_state *state)

{

state->recursion_limit = Py_DEFAULT_RECURSION_LIMIT;

_Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;

_gil_initialize(&state->gil);

}

int

Py_GetRecursionLimit(void)

{

struct _ceval_runtime_state *ceval = &_PyRuntime.ceval;

return ceval->recursion_limit;

}

void

Py_SetRecursionLimit(int new_limit)

{

struct _ceval_runtime_state *ceval = &_PyRuntime.ceval;

ceval->recursion_limit = new_limit;

_Py_CheckRecursionLimit = ceval->recursion_limit;

}

/* The function _Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall()

if the recursion_depth reaches _Py_CheckRecursionLimit.

If USE_STACKCHECK, the macro decrements _Py_CheckRecursionLimit

to guarantee that _Py_CheckRecursiveCall() is regularly called.

Without USE_STACKCHECK, there is no need for this. */

int

_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)

{

_PyRuntimeState *runtime = tstate->interp->runtime;

int recursion_limit = runtime->ceval.recursion_limit;

#ifdef USE_STACKCHECK

tstate->stackcheck_counter = 0;

if (PyOS_CheckStack()) {

--tstate->recursion_depth;

_PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");

return -1;

}

/* Needed for ABI backwards-compatibility (see bpo-31857) */

_Py_CheckRecursionLimit = recursion_limit;

#endif

if (tstate->recursion_critical)

/* Somebody asked that we don't check for recursion. */

return 0;

if (tstate->overflowed) {

if (tstate->recursion_depth > recursion_limit + 50) {

/* Overflowing while handling an overflow. Give up. */

Py_FatalError("Cannot recover from stack overflow.");

}

return 0;

}

if (tstate->recursion_depth > recursion_limit) {

--tstate->recursion_depth;

tstate->overflowed = 1;

_PyErr_Format(tstate, PyExc_RecursionError,

"maximum recursion depth exceeded%s",

where);

return -1;

}

return 0;

}

static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);

static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);

#define _Py_TracingPossible(ceval) ((ceval)->tracing_possible)

PyObject *

PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)

{

return PyEval_EvalCodeEx(co,

globals, locals,

(PyObject **)NULL, 0,

(PyObject **)NULL, 0,

(PyObject **)NULL, 0,

NULL, NULL);

}

/* Interpreter main loop */

PyObject *

PyEval_EvalFrame(PyFrameObject *f)

{

/* This is for backward compatibility with extension modules that

used this API; core interpreter code should call

PyEval_EvalFrameEx() */

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f, 0);

}

PyObject *

PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)

{

PyThreadState *tstate = _PyThreadState_GET();

return _PyEval_EvalFrame(tstate, f, throwflag);

}

PyObject* _Py_HOT_FUNCTION

_PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)

{

#ifdef DXPAIRS

int lastopcode = 0;

#endif

PyObject **stack_pointer; /* Next free slot in value stack */

const _Py_CODEUNIT *next_instr;

int opcode; /* Current opcode */

int oparg; /* Current opcode argument, if any */

PyObject **fastlocals, **freevars;

PyObject *retval = NULL; /* Return value */

_PyRuntimeState * const runtime = &_PyRuntime;

PyThreadState * const tstate = _PyRuntimeState_GetThreadState(runtime);

struct _ceval_runtime_state * const ceval = &runtime->ceval;

_Py_atomic_int * const eval_breaker = &ceval->eval_breaker;

PyCodeObject *co;

/* when tracing we set things up so that

not (instr_lb <= current_bytecode_offset < instr_ub)

is true when the line being executed has changed. The

initial values are such as to make this false the first

time it is tested. */

int instr_ub = -1, instr_lb = 0, instr_prev = -1;

const _Py_CODEUNIT *first_instr;

PyObject *names;

PyObject *consts;

_PyOpcache *co_opcache;

#ifdef LLTRACE

_Py_IDENTIFIER(__ltrace__);

#endif

/* Computed GOTOs, or

the-optimization-commonly-but-improperly-known-as-"threaded code"

using gcc's labels-as-values extension

(http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).

The traditional bytecode evaluation loop uses a "switch" statement, which

decent compilers will optimize as a single indirect branch instruction

combined with a lookup table of jump addresses. However, since the

indirect jump instruction is shared by all opcodes, the CPU will have a

hard time making the right prediction for where to jump next (actually,

it will be always wrong except in the uncommon case of a sequence of

several identical opcodes).

"Threaded code" in contrast, uses an explicit jump table and an explicit

indirect jump instruction at the end of each opcode. Since the jump

instruction is at a different address for each opcode, the CPU will make a

separate prediction for each of these instructions, which is equivalent to

predicting the second opcode of each opcode pair. These predictions have

a much better chance to turn out valid, especially in small bytecode loops.

A mispredicted branch on a modern CPU flushes the whole pipeline and

can cost several CPU cycles (depending on the pipeline depth),

and potentially many more instructions (depending on the pipeline width).

A correctly predicted branch, however, is nearly free.

At the time of this writing, the "threaded code" version is up to 15-20%

faster than the normal "switch" version, depending on the compiler and the

CPU architecture.

We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,

because it would render the measurements invalid.

NOTE: care must be taken that the compiler doesn't try to "optimize" the

indirect jumps by sharing them between all opcodes. Such optimizations

can be disabled on gcc by using the -fno-gcse flag (or possibly

-fno-crossjumping).

*/

#ifdef DYNAMIC_EXECUTION_PROFILE

#undef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 0

#endif

#ifdef HAVE_COMPUTED_GOTOS

#ifndef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 1

#endif

#else

#if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS

#error "Computed gotos are not supported on this compiler."

#endif

#undef USE_COMPUTED_GOTOS

#define USE_COMPUTED_GOTOS 0

#endif

#if USE_COMPUTED_GOTOS

/* Import the static jump table */

#include "opcode_targets.h"

#define TARGET(op) \

op: \

TARGET_##op

#ifdef LLTRACE

#define FAST_DISPATCH() \

{ \

if (!lltrace && !_Py_TracingPossible(ceval) && !PyDTrace_LINE_ENABLED()) { \

f->f_lasti = INSTR_OFFSET(); \

NEXTOPARG(); \

goto *opcode_targets[opcode]; \

} \

goto fast_next_opcode; \

}

#else

#define FAST_DISPATCH() \

{ \

if (!_Py_TracingPossible(ceval) && !PyDTrace_LINE_ENABLED()) { \

f->f_lasti = INSTR_OFFSET(); \

NEXTOPARG(); \

goto *opcode_targets[opcode]; \

} \

goto fast_next_opcode; \

}

#endif

#define DISPATCH() \

{ \

if (!_Py_atomic_load_relaxed(eval_breaker)) { \

FAST_DISPATCH(); \

} \

continue; \

}

#else

#define TARGET(op) op

#define FAST_DISPATCH() goto fast_next_opcode

#define DISPATCH() continue

#endif

/* Tuple access macros */

#ifndef Py_DEBUG

#define GETITEM(v, i) PyTuple_GET_ITEM((PyTupleObject *)(v), (i))

#else

#define GETITEM(v, i) PyTuple_GetItem((v), (i))

#endif

/* Code access macros */

/* The integer overflow is checked by an assertion below. */

#define INSTR_OFFSET() \

(sizeof(_Py_CODEUNIT) * (int)(next_instr - first_instr))

#define NEXTOPARG() do { \

_Py_CODEUNIT word = *next_instr; \

opcode = _Py_OPCODE(word); \

oparg = _Py_OPARG(word); \

next_instr++; \

} while (0)

#define JUMPTO(x) (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT))

#define JUMPBY(x) (next_instr += (x) / sizeof(_Py_CODEUNIT))

/* OpCode prediction macros

Some opcodes tend to come in pairs thus making it possible to

predict the second code when the first is run. For example,

COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE.

Verifying the prediction costs a single high-speed test of a register

variable against a constant. If the pairing was good, then the

processor's own internal branch predication has a high likelihood of

success, resulting in a nearly zero-overhead transition to the

next opcode. A successful prediction saves a trip through the eval-loop

including its unpredictable switch-case branch. Combined with the

processor's internal branch prediction, a successful PREDICT has the

effect of making the two opcodes run as if they were a single new opcode

with the bodies combined.

If collecting opcode statistics, your choices are to either keep the

predictions turned-on and interpret the results as if some opcodes

had been combined or turn-off predictions so that the opcode frequency

counter updates for both opcodes.

Opcode prediction is disabled with threaded code, since the latter allows

the CPU to record separate branch prediction information for each

opcode.

*/

#if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS

#define PREDICT(op) if (0) goto PRED_##op

#else

#define PREDICT(op) \

do{ \

_Py_CODEUNIT word = *next_instr; \

opcode = _Py_OPCODE(word); \

if (opcode == op){ \

oparg = _Py_OPARG(word); \

next_instr++; \

goto PRED_##op; \

} \

} while(0)

#endif

#define PREDICTED(op) PRED_##op:

/* Stack manipulation macros */

/* The stack can grow at most MAXINT deep, as co_nlocals and

co_stacksize are ints. */

#define STACK_LEVEL() ((int)(stack_pointer - f->f_valuestack))

#define EMPTY() (STACK_LEVEL() == 0)

#define TOP() (stack_pointer[-1])

#define SECOND() (stack_pointer[-2])

#define THIRD() (stack_pointer[-3])

#define FOURTH() (stack_pointer[-4])

#define PEEK(n) (stack_pointer[-(n)])

#define SET_TOP(v) (stack_pointer[-1] = (v))

#define SET_SECOND(v) (stack_pointer[-2] = (v))

#define SET_THIRD(v) (stack_pointer[-3] = (v))

#define SET_FOURTH(v) (stack_pointer[-4] = (v))

#define SET_VALUE(n, v) (stack_pointer[-(n)] = (v))

#define BASIC_STACKADJ(n) (stack_pointer += n)

#define BASIC_PUSH(v) (*stack_pointer++ = (v))

#define BASIC_POP() (*--stack_pointer)

#ifdef LLTRACE

#define PUSH(v) { (void)(BASIC_PUSH(v), \

lltrace && prtrace(tstate, TOP(), "push")); \

assert(STACK_LEVEL() <= co->co_stacksize); }

#define POP() ((void)(lltrace && prtrace(tstate, TOP(), "pop")), \

BASIC_POP())

#define STACK_GROW(n) do { \

assert(n >= 0); \

(void)(BASIC_STACKADJ(n), \

lltrace && prtrace(tstate, TOP(), "stackadj")); \

assert(STACK_LEVEL() <= co->co_stacksize); \

} while (0)

#define STACK_SHRINK(n) do { \

assert(n >= 0); \

(void)(lltrace && prtrace(tstate, TOP(), "stackadj")); \

(void)(BASIC_STACKADJ(-n)); \

assert(STACK_LEVEL() <= co->co_stacksize); \

} while (0)

#define EXT_POP(STACK_POINTER) ((void)(lltrace && \

prtrace(tstate, (STACK_POINTER)[-1], "ext_pop")), \

*--(STACK_POINTER))

#else

#define PUSH(v) BASIC_PUSH(v)

#define POP() BASIC_POP()

#define STACK_GROW(n) BASIC_STACKADJ(n)

#define STACK_SHRINK(n) BASIC_STACKADJ(-n)

#define EXT_POP(STACK_POINTER) (*--(STACK_POINTER))

#endif

/* Local variable macros */

#define GETLOCAL(i) (fastlocals[i])

/* The SETLOCAL() macro must not DECREF the local variable in-place and