#include "Python.h"

#ifdef _Py_TIER2

#include "opcode.h"

#include "pycore_interp.h"

#include "pycore_backoff.h"

#include "pycore_bitutils.h" // _Py_popcount32()

#include "pycore_code.h" // _Py_GetBaseCodeUnit

#include "pycore_function.h" // _PyFunction_LookupByVersion()

#include "pycore_interpframe.h"

#include "pycore_object.h" // _PyObject_GC_UNTRACK()

#include "pycore_opcode_metadata.h" // _PyOpcode_OpName[]

#include "pycore_opcode_utils.h" // MAX_REAL_OPCODE

#include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_tuple.h" // _PyTuple_FromArraySteal

#include "pycore_unicodeobject.h" // _PyUnicode_FromASCII

#include "pycore_uop_ids.h"

#include "pycore_jit.h"

#include <stdbool.h>

#include <stdint.h>

#include <stddef.h>

#define NEED_OPCODE_METADATA

#include "pycore_uop_metadata.h" // Uop tables

#undef NEED_OPCODE_METADATA

#define MAX_EXECUTORS_SIZE 256

#define _PyExecutorObject_CAST(op) ((_PyExecutorObject *)(op))

static bool

has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr)

{

if (instr->op.code == ENTER_EXECUTOR) {

return true;

}

if (code->co_executors == NULL) {

return true;

}

return code->co_executors->size < MAX_EXECUTORS_SIZE;

}

static int32_t

get_index_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr)

{

if (instr->op.code == ENTER_EXECUTOR) {

return instr->op.arg;

}

_PyExecutorArray *old = code->co_executors;

int size = 0;

int capacity = 0;

if (old != NULL) {

size = old->size;

capacity = old->capacity;

assert(size < MAX_EXECUTORS_SIZE);

}

assert(size <= capacity);

if (size == capacity) {

/* Array is full. Grow array */

int new_capacity = capacity ? capacity * 2 : 4;

_PyExecutorArray *new = PyMem_Realloc(

old,

offsetof(_PyExecutorArray, executors) +

new_capacity * sizeof(_PyExecutorObject *));

if (new == NULL) {

return -1;

}

new->capacity = new_capacity;

new->size = size;

code->co_executors = new;

}

assert(size < code->co_executors->capacity);

return size;

}

static void

insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorObject *executor)

{

Py_INCREF(executor);

if (instr->op.code == ENTER_EXECUTOR) {

assert(index == instr->op.arg);

_Py_ExecutorDetach(code->co_executors->executors[index]);

}

else {

assert(code->co_executors->size == index);

assert(code->co_executors->capacity > index);

code->co_executors->size++;

}

executor->vm_data.opcode = instr->op.code;

executor->vm_data.oparg = instr->op.arg;

executor->vm_data.code = code;

executor->vm_data.index = (int)(instr - _PyCode_CODE(code));

code->co_executors->executors[index] = executor;

assert(index < MAX_EXECUTORS_SIZE);

instr->op.code = ENTER_EXECUTOR;

instr->op.arg = index;

}

static _PyExecutorObject *

make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies);

static int

uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr,

_PyExecutorObject **exec_ptr, int curr_stackentries,

bool progress_needed);

/* Returns 1 if optimized, 0 if not optimized, and -1 for an error.

* If optimized, *executor_ptr contains a new reference to the executor

*/

int

_PyOptimizer_Optimize(

_PyInterpreterFrame *frame, _Py_CODEUNIT *start,

_PyExecutorObject **executor_ptr, int chain_depth)

{

_PyStackRef *stack_pointer = frame->stackpointer;

assert(_PyInterpreterState_GET()->jit);

// The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must*

// make progress in order to avoid infinite loops or excessively-long

// side-exit chains. We can only insert the executor into the bytecode if

// this is true, since a deopt won't infinitely re-enter the executor:

chain_depth %= MAX_CHAIN_DEPTH;

bool progress_needed = chain_depth == 0;

PyCodeObject *code = _PyFrame_GetCode(frame);

assert(PyCode_Check(code));

if (progress_needed && !has_space_for_executor(code, start)) {

return 0;

}

int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed);

if (err <= 0) {

return err;

}

assert(*executor_ptr != NULL);

if (progress_needed) {

int index = get_index_for_executor(code, start);

if (index < 0) {

/* Out of memory. Don't raise and assume that the

* error will show up elsewhere.

*

* If an optimizer has already produced an executor,

* it might get confused by the executor disappearing,

* but there is not much we can do about that here. */

Py_DECREF(*executor_ptr);

return 0;

}

insert_executor(code, start, index, *executor_ptr);

}

else {

(*executor_ptr)->vm_data.code = NULL;

}

(*executor_ptr)->vm_data.chain_depth = chain_depth;

assert((*executor_ptr)->vm_data.valid);

return 1;

}

static _PyExecutorObject *

get_executor_lock_held(PyCodeObject *code, int offset)

{

int code_len = (int)Py_SIZE(code);

for (int i = 0 ; i < code_len;) {

if (_PyCode_CODE(code)[i].op.code == ENTER_EXECUTOR && i*2 == offset) {

int oparg = _PyCode_CODE(code)[i].op.arg;

_PyExecutorObject *res = code->co_executors->executors[oparg];

Py_INCREF(res);

return res;

}

i += _PyInstruction_GetLength(code, i);

}

PyErr_SetString(PyExc_ValueError, "no executor at given byte offset");

return NULL;

}

_PyExecutorObject *

_Py_GetExecutor(PyCodeObject *code, int offset)

{

_PyExecutorObject *executor;

Py_BEGIN_CRITICAL_SECTION(code);

executor = get_executor_lock_held(code, offset);

Py_END_CRITICAL_SECTION();

return executor;

}

static PyObject *

is_valid(PyObject *self, PyObject *Py_UNUSED(ignored))

{

return PyBool_FromLong(((_PyExecutorObject *)self)->vm_data.valid);

}

static PyObject *

get_opcode(PyObject *self, PyObject *Py_UNUSED(ignored))

{

return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.opcode);

}

static PyObject *

get_oparg(PyObject *self, PyObject *Py_UNUSED(ignored))

{

return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.oparg);

}

///////////////////// Experimental UOp Optimizer /////////////////////

static int executor_clear(PyObject *executor);

static void unlink_executor(_PyExecutorObject *executor);

static void

free_executor(_PyExecutorObject *self)

{

#ifdef _Py_JIT

_PyJIT_Free(self);

#endif

PyObject_GC_Del(self);

}

void

_Py_ClearExecutorDeletionList(PyInterpreterState *interp)

{

_PyRuntimeState *runtime = &_PyRuntime;

HEAD_LOCK(runtime);

PyThreadState* ts = PyInterpreterState_ThreadHead(interp);

HEAD_UNLOCK(runtime);

while (ts) {

_PyExecutorObject *current = (_PyExecutorObject *)ts->current_executor;

if (current != NULL) {

/* Anything in this list will be unlinked, so we can reuse the

* linked field as a reachability marker. */

current->vm_data.linked = 1;

}

HEAD_LOCK(runtime);

ts = PyThreadState_Next(ts);

HEAD_UNLOCK(runtime);

}

_PyExecutorObject **prev_to_next_ptr = &interp->executor_deletion_list_head;

_PyExecutorObject *exec = *prev_to_next_ptr;

while (exec != NULL) {

if (exec->vm_data.linked) {

// This executor is currently executing

exec->vm_data.linked = 0;

prev_to_next_ptr = &exec->vm_data.links.next;

}

else {

*prev_to_next_ptr = exec->vm_data.links.next;

free_executor(exec);

}

exec = *prev_to_next_ptr;

}

interp->executor_deletion_list_remaining_capacity = EXECUTOR_DELETE_LIST_MAX;

}

static void

add_to_pending_deletion_list(_PyExecutorObject *self)

{

PyInterpreterState *interp = PyInterpreterState_Get();

self->vm_data.links.next = interp->executor_deletion_list_head;

interp->executor_deletion_list_head = self;

if (interp->executor_deletion_list_remaining_capacity > 0) {

interp->executor_deletion_list_remaining_capacity--;

}

else {

_Py_ClearExecutorDeletionList(interp);

}

}

static void

uop_dealloc(PyObject *op) {

_PyExecutorObject *self = _PyExecutorObject_CAST(op);

_PyObject_GC_UNTRACK(self);

assert(self->vm_data.code == NULL);

unlink_executor(self);

// Once unlinked it becomes impossible to invalidate an executor, so do it here.

self->vm_data.valid = 0;

add_to_pending_deletion_list(self);

}

const char *

_PyUOpName(int index)

{

if (index < 0 || index > MAX_UOP_ID) {

return NULL;

}

return _PyOpcode_uop_name[index];

}

#ifdef Py_DEBUG

void

_PyUOpPrint(const _PyUOpInstruction *uop)

{

const char *name = _PyUOpName(uop->opcode);

if (name == NULL) {

printf("<uop %d>", uop->opcode);

}

else {

printf("%s", name);

}

switch(uop->format) {

case UOP_FORMAT_TARGET:

printf(" (%d, target=%d, operand0=%#" PRIx64 ", operand1=%#" PRIx64,

uop->oparg,

uop->target,

(uint64_t)uop->operand0,

(uint64_t)uop->operand1);

break;

case UOP_FORMAT_JUMP:

printf(" (%d, jump_target=%d, operand0=%#" PRIx64 ", operand1=%#" PRIx64,

uop->oparg,

uop->jump_target,

(uint64_t)uop->operand0,

(uint64_t)uop->operand1);

break;

default:

printf(" (%d, Unknown format)", uop->oparg);

}

if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) {

printf(", error_target=%d", uop->error_target);

}

printf(")");

}

#endif

static Py_ssize_t

uop_len(PyObject *op)

{

_PyExecutorObject *self = _PyExecutorObject_CAST(op);

return self->code_size;

}

static PyObject *

uop_item(PyObject *op, Py_ssize_t index)

{

_PyExecutorObject *self = _PyExecutorObject_CAST(op);

Py_ssize_t len = uop_len(op);

if (index < 0 || index >= len) {

PyErr_SetNone(PyExc_IndexError);

return NULL;

}

const char *name = _PyUOpName(self->trace[index].opcode);

if (name == NULL) {

name = "<nil>";

}

PyObject *oname = _PyUnicode_FromASCII(name, strlen(name));

if (oname == NULL) {

return NULL;

}

PyObject *oparg = PyLong_FromUnsignedLong(self->trace[index].oparg);

if (oparg == NULL) {

Py_DECREF(oname);

return NULL;

}

PyObject *target = PyLong_FromUnsignedLong(self->trace[index].target);

if (oparg == NULL) {

Py_DECREF(oparg);

Py_DECREF(oname);

return NULL;

}

PyObject *operand = PyLong_FromUnsignedLongLong(self->trace[index].operand0);

if (operand == NULL) {

Py_DECREF(target);

Py_DECREF(oparg);

Py_DECREF(oname);

return NULL;

}

PyObject *args[4] = { oname, oparg, target, operand };

return _PyTuple_FromArraySteal(args, 4);

}

PySequenceMethods uop_as_sequence = {

.sq_length = uop_len,

.sq_item = uop_item,

};

static int

executor_traverse(PyObject *o, visitproc visit, void *arg)

{

_PyExecutorObject *executor = _PyExecutorObject_CAST(o);

for (uint32_t i = 0; i < executor->exit_count; i++) {

Py_VISIT(executor->exits[i].executor);

}

return 0;

}

static PyObject *

get_jit_code(PyObject *self, PyObject *Py_UNUSED(ignored))

{

#ifndef _Py_JIT

PyErr_SetString(PyExc_RuntimeError, "JIT support not enabled.");

return NULL;

#else

_PyExecutorObject *executor = _PyExecutorObject_CAST(self);

if (executor->jit_code == NULL || executor->jit_size == 0) {

Py_RETURN_NONE;

}

return PyBytes_FromStringAndSize(executor->jit_code, executor->jit_size);

#endif

}

static PyMethodDef uop_executor_methods[] = {

{ "is_valid", is_valid, METH_NOARGS, NULL },

{ "get_jit_code", get_jit_code, METH_NOARGS, NULL},

{ "get_opcode", get_opcode, METH_NOARGS, NULL },

{ "get_oparg", get_oparg, METH_NOARGS, NULL },

{ NULL, NULL },

};

static int

executor_is_gc(PyObject *o)

{

return !_Py_IsImmortal(o);

}

PyTypeObject _PyUOpExecutor_Type = {

PyVarObject_HEAD_INIT(&PyType_Type, 0)

.tp_name = "uop_executor",

.tp_basicsize = offsetof(_PyExecutorObject, exits),

.tp_itemsize = 1,

.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC,

.tp_dealloc = uop_dealloc,

.tp_as_sequence = &uop_as_sequence,

.tp_methods = uop_executor_methods,

.tp_traverse = executor_traverse,

.tp_clear = executor_clear,

.tp_is_gc = executor_is_gc,

};

/* TO DO -- Generate these tables */

static const uint16_t

_PyUOp_Replacements[MAX_UOP_ID + 1] = {

[_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE,

[_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST,

[_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE,

[_FOR_ITER] = _FOR_ITER_TIER_TWO,

[_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO,

};

static const uint8_t

is_for_iter_test[MAX_UOP_ID + 1] = {

[_GUARD_NOT_EXHAUSTED_RANGE] = 1,

[_GUARD_NOT_EXHAUSTED_LIST] = 1,

[_GUARD_NOT_EXHAUSTED_TUPLE] = 1,

[_FOR_ITER_TIER_TWO] = 1,

};

static const uint16_t

BRANCH_TO_GUARD[4][2] = {

[POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP,

[POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_FALSE_POP,

[POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_FALSE_POP,

[POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_TRUE_POP,

[POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NOT_NONE_POP,

[POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NONE_POP,

[POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NONE_POP,

[POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP,

};

#define CONFIDENCE_RANGE 1000

#define CONFIDENCE_CUTOFF 333

#ifdef Py_DEBUG

#define DPRINTF(level, ...) \

if (lltrace >= (level)) { printf(__VA_ARGS__); }

#else

#define DPRINTF(level, ...)

#endif

static inline int

add_to_trace(

_PyUOpInstruction *trace,

int trace_length,

uint16_t opcode,

uint16_t oparg,

uint64_t operand,

uint32_t target)

{

trace[trace_length].opcode = opcode;

trace[trace_length].format = UOP_FORMAT_TARGET;

trace[trace_length].target = target;

trace[trace_length].oparg = oparg;

trace[trace_length].operand0 = operand;

#ifdef Py_STATS

trace[trace_length].execution_count = 0;

#endif

return trace_length + 1;

}

#ifdef Py_DEBUG

#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \

assert(trace_length < max_length); \

trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \

if (lltrace >= 2) { \

printf("%4d ADD_TO_TRACE: ", trace_length); \

_PyUOpPrint(&trace[trace_length-1]); \

printf("\n"); \

}

#else

#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \

assert(trace_length < max_length); \

trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET));

#endif

#define INSTR_IP(INSTR, CODE) \

((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive)))

// Reserve space for n uops

#define RESERVE_RAW(n, opname) \

if (trace_length + (n) > max_length) { \

DPRINTF(2, "No room for %s (need %d, got %d)\n", \

(opname), (n), max_length - trace_length); \

OPT_STAT_INC(trace_too_long); \

goto done; \

}

// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE

#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode))

// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE)

#define TRACE_STACK_PUSH() \

if (trace_stack_depth >= TRACE_STACK_SIZE) { \

DPRINTF(2, "Trace stack overflow\n"); \

OPT_STAT_INC(trace_stack_overflow); \

return 0; \

} \

assert(func == NULL || func->func_code == (PyObject *)code); \

trace_stack[trace_stack_depth].func = func; \

trace_stack[trace_stack_depth].code = code; \

trace_stack[trace_stack_depth].instr = instr; \

trace_stack_depth++;

#define TRACE_STACK_POP() \

if (trace_stack_depth <= 0) { \

Py_FatalError("Trace stack underflow\n"); \

} \

trace_stack_depth--; \

func = trace_stack[trace_stack_depth].func; \

code = trace_stack[trace_stack_depth].code; \

assert(func == NULL || func->func_code == (PyObject *)code); \

instr = trace_stack[trace_stack_depth].instr;

/* Returns the length of the trace on success,

* 0 if it failed to produce a worthwhile trace,

* and -1 on an error.

*/

static int

translate_bytecode_to_trace(

_PyInterpreterFrame *frame,

_Py_CODEUNIT *instr,

_PyUOpInstruction *trace,

int buffer_size,

_PyBloomFilter *dependencies, bool progress_needed)

{

bool first = true;

PyCodeObject *code = _PyFrame_GetCode(frame);

PyFunctionObject *func = _PyFrame_GetFunction(frame);

assert(PyFunction_Check(func));

PyCodeObject *initial_code = code;

_Py_BloomFilter_Add(dependencies, initial_code);

_Py_CODEUNIT *initial_instr = instr;

int trace_length = 0;

// Leave space for possible trailing _EXIT_TRACE

int max_length = buffer_size-2;

struct {

PyFunctionObject *func;

PyCodeObject *code;

_Py_CODEUNIT *instr;

} trace_stack[TRACE_STACK_SIZE];

int trace_stack_depth = 0;

int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions

bool jump_seen = false;

#ifdef Py_DEBUG

char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");

int lltrace = 0;

if (python_lltrace != NULL && *python_lltrace >= '0') {

lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that

}

#endif

DPRINTF(2,

"Optimizing %s (%s:%d) at byte offset %d\n",

PyUnicode_AsUTF8(code->co_qualname),

PyUnicode_AsUTF8(code->co_filename),

code->co_firstlineno,

2 * INSTR_IP(initial_instr, code));

ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code));

ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0);

uint32_t target = 0;

for (;;) {

target = INSTR_IP(instr, code);

// Need space for _DEOPT

max_length--;

uint32_t opcode = instr->op.code;

uint32_t oparg = instr->op.arg;

if (!first && instr == initial_instr) {

// We have looped around to the start:

RESERVE(1);

ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0);

goto done;

}

DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg);

if (opcode == EXTENDED_ARG) {

instr++;

opcode = instr->op.code;

oparg = (oparg << 8) | instr->op.arg;

if (opcode == EXTENDED_ARG) {

instr--;

goto done;

}

}

if (opcode == ENTER_EXECUTOR) {

// We have a couple of options here. We *could* peek "underneath"

// this executor and continue tracing, which could give us a longer,

// more optimizeable trace (at the expense of lots of duplicated

// tier two code). Instead, we choose to just end here and stitch to

// the other trace, which allows a side-exit traces to rejoin the

// "main" trace periodically (and also helps protect us against

// pathological behavior where the amount of tier two code explodes

// for a medium-length, branchy code path). This seems to work

// better in practice, but in the future we could be smarter about

// what we do here:

goto done;

}

assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG);

RESERVE_RAW(2, "_CHECK_VALIDITY");

ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target);

if (!OPCODE_HAS_NO_SAVE_IP(opcode)) {

RESERVE_RAW(2, "_SET_IP");

ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target);

}

/* Special case the first instruction,

* so that we can guarantee forward progress */

if (first && progress_needed) {

assert(first);

if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {

opcode = _PyOpcode_Deopt[opcode];

}

assert(!OPCODE_HAS_EXIT(opcode));

assert(!OPCODE_HAS_DEOPT(opcode));

}

if (OPCODE_HAS_EXIT(opcode)) {

// Make space for side exit and final _EXIT_TRACE:

RESERVE_RAW(2, "_EXIT_TRACE");

max_length--;

}

if (OPCODE_HAS_ERROR(opcode)) {

// Make space for error stub and final _EXIT_TRACE:

RESERVE_RAW(2, "_ERROR_POP_N");

max_length--;

}

switch (opcode) {

case POP_JUMP_IF_NONE:

case POP_JUMP_IF_NOT_NONE:

case POP_JUMP_IF_FALSE:

case POP_JUMP_IF_TRUE:

{

RESERVE(1);

int counter = instr[1].cache;

int bitcount = _Py_popcount32(counter);

int jump_likely = bitcount > 8;

/* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%.

For values in between, adjust proportionally. */

if (jump_likely) {

confidence = confidence * bitcount / 16;

}

else {

confidence = confidence * (16 - bitcount) / 16;

}

uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely];

DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n",

target, _PyOpcode_OpName[opcode], oparg,

counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode));

if (confidence < CONFIDENCE_CUTOFF) {

DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF);

OPT_STAT_INC(low_confidence);

goto done;

}

_Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];

_Py_CODEUNIT *target_instr = next_instr + oparg;

if (jump_likely) {

DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n",

instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code));

instr = target_instr;

ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code));

goto top;

}

ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code));

break;

}

case JUMP_BACKWARD:

case JUMP_BACKWARD_JIT:

ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target);

_Py_FALLTHROUGH;

case JUMP_BACKWARD_NO_INTERRUPT:

{

instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg;

if (jump_seen) {

OPT_STAT_INC(inner_loop);

DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n");

goto done;

}

jump_seen = true;

goto top;

}

case JUMP_FORWARD:

{

RESERVE(0);

// This will emit two _SET_IP instructions; leave it to the optimizer

instr += oparg;

break;

}

case RESUME:

/* Use a special tier 2 version of RESUME_CHECK to allow traces to

* start with RESUME_CHECK */

ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);

break;

default:

{

const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];

if (expansion->nuops > 0) {

// Reserve space for nuops (+ _SET_IP + _EXIT_TRACE)

int nuops = expansion->nuops;

RESERVE(nuops + 1); /* One extra for exit */

int16_t last_op = expansion->uops[nuops-1].uop;

if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) {

// Check for trace stack underflow now:

// We can't bail e.g. in the middle of

// LOAD_CONST + _RETURN_VALUE.

if (trace_stack_depth == 0) {

DPRINTF(2, "Trace stack underflow\n");

OPT_STAT_INC(trace_stack_underflow);

return 0;

}

}

uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM

for (int i = 0; i < nuops; i++) {

oparg = orig_oparg;

uint32_t uop = expansion->uops[i].uop;

uint64_t operand = 0;

// Add one to account for the actual opcode/oparg pair:

int offset = expansion->uops[i].offset + 1;

switch (expansion->uops[i].size) {

case OPARG_SIMPLE:

assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD);

break;

case OPARG_CACHE_1:

operand = read_u16(&instr[offset].cache);

break;

case OPARG_CACHE_2:

operand = read_u32(&instr[offset].cache);

break;

case OPARG_CACHE_4:

operand = read_u64(&instr[offset].cache);

break;

case OPARG_TOP: // First half of super-instr

oparg = orig_oparg >> 4;

break;

case OPARG_BOTTOM: // Second half of super-instr

oparg = orig_oparg & 0xF;

break;

case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset

oparg = offset;

assert(uop == _SAVE_RETURN_OFFSET);

break;

case OPARG_REPLACED:

uop = _PyUOp_Replacements[uop];

assert(uop != 0);

#ifdef Py_DEBUG

{

uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255);

uint32_t jump_target = next_inst + oparg;

assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR);

assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER);

}

#endif

break;

case OPERAND1_1:

assert(trace[trace_length-1].opcode == uop);

operand = read_u16(&instr[offset].cache);

trace[trace_length-1].operand1 = operand;

continue;

case OPERAND1_2:

assert(trace[trace_length-1].opcode == uop);

operand = read_u32(&instr[offset].cache);

trace[trace_length-1].operand1 = operand;

continue;

case OPERAND1_4:

assert(trace[trace_length-1].opcode == uop);

operand = read_u64(&instr[offset].cache);

trace[trace_length-1].operand1 = operand;

continue;

default:

fprintf(stderr,

"opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n",

opcode, oparg, nuops, i,

expansion->uops[i].size,

expansion->uops[i].offset);

Py_FatalError("garbled expansion");

}

if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) {

TRACE_STACK_POP();

/* Set the operand to the function or code object returned to,

* to assist optimization passes. (See _PUSH_FRAME below.)

*/

if (func != NULL) {

operand = (uintptr_t)func;

}

else if (code != NULL) {

operand = (uintptr_t)code | 1;

}

else {

operand = 0;

}

ADD_TO_TRACE(uop, oparg, operand, target);

DPRINTF(2,

"Returning to %s (%s:%d) at byte offset %d\n",

PyUnicode_AsUTF8(code->co_qualname),

PyUnicode_AsUTF8(code->co_filename),

code->co_firstlineno,

2 * INSTR_IP(instr, code));

goto top;

}

if (uop == _PUSH_FRAME) {

assert(i + 1 == nuops);

if (opcode == FOR_ITER_GEN ||

opcode == LOAD_ATTR_PROPERTY ||

opcode == BINARY_OP_SUBSCR_GETITEM ||

opcode == SEND_GEN)

{

DPRINTF(2, "Bailing due to dynamic target\n");

OPT_STAT_INC(unknown_callee);

return 0;

}

assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW);

int func_version_offset =

offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT)

// Add one to account for the actual opcode/oparg pair:

+ 1;

uint32_t func_version = read_u32(&instr[func_version_offset].cache);

PyCodeObject *new_code = NULL;

PyFunctionObject *new_func =

_PyFunction_LookupByVersion(func_version, (PyObject **) &new_code);

DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n",

(int)func_version, new_func, new_code);

if (new_code != NULL) {

if (new_code == code) {

// Recursive call, bail (we could be here forever).

DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n",

PyUnicode_AsUTF8(new_code->co_qualname),

PyUnicode_AsUTF8(new_code->co_filename),

new_code->co_firstlineno);

OPT_STAT_INC(recursive_call);

ADD_TO_TRACE(uop, oparg, 0, target);

ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0);

goto done;

}

if (new_code->co_version != func_version) {

// func.__code__ was updated.

// Perhaps it may happen again, so don't bother tracing.

// TODO: Reason about this -- is it better to bail or not?

DPRINTF(2, "Bailing because co_version != func_version\n");

ADD_TO_TRACE(uop, oparg, 0, target);

ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0);

goto done;

}

// Increment IP to the return address

instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1;

TRACE_STACK_PUSH();

_Py_BloomFilter_Add(dependencies, new_code);

/* Set the operand to the callee's function or code object,

* to assist optimization passes.

* We prefer setting it to the function (for remove_globals())

* but if that's not available but the code is available,

* use the code, setting the low bit so the optimizer knows.

*/

if (new_func != NULL) {

operand = (uintptr_t)new_func;

}

else if (new_code != NULL) {

operand = (uintptr_t)new_code | 1;

}

else {

operand = 0;

}

ADD_TO_TRACE(uop, oparg, operand, target);

code = new_code;

func = new_func;

instr = _PyCode_CODE(code);

DPRINTF(2,

"Continuing in %s (%s:%d) at byte offset %d\n",

PyUnicode_AsUTF8(code->co_qualname),

PyUnicode_AsUTF8(code->co_filename),

code->co_firstlineno,

2 * INSTR_IP(instr, code));

goto top;

}

DPRINTF(2, "Bail, new_code == NULL\n");

OPT_STAT_INC(unknown_callee);

return 0;

}

if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) {

assert(i + 1 == nuops);

_Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];

assert(next_instr->op.code == STORE_FAST);

operand = next_instr->op.arg;

// Skip the STORE_FAST:

instr++;

}

// All other instructions

ADD_TO_TRACE(uop, oparg, operand, target);

}

break;

}

DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]);

OPT_UNSUPPORTED_OPCODE(opcode);

goto done; // Break out of loop

} // End default

} // End switch (opcode)

instr++;

// Add cache size for opcode

instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];

if (opcode == CALL_LIST_APPEND) {

assert(instr->op.code == POP_TOP);

instr++;

}

top:

// Jump here after _PUSH_FRAME or likely branches.

first = false;

} // End for (;;)

done:

while (trace_stack_depth > 0) {

TRACE_STACK_POP();

}

assert(code == initial_code);

// Skip short traces where we can't even translate a single instruction:

if (first) {

OPT_STAT_INC(trace_too_short);

DPRINTF(2,

"No trace for %s (%s:%d) at byte offset %d (no progress)\n",

PyUnicode_AsUTF8(code->co_qualname),

PyUnicode_AsUTF8(code->co_filename),

code->co_firstlineno,

2 * INSTR_IP(initial_instr, code));

return 0;

}

if (!is_terminator(&trace[trace_length-1])) {

/* Allow space for _EXIT_TRACE */

max_length += 2;

ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target);

}

DPRINTF(1,

"Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n",

PyUnicode_AsUTF8(code->co_qualname),

PyUnicode_AsUTF8(code->co_filename),

code->co_firstlineno,

2 * INSTR_IP(initial_instr, code),

trace_length);

OPT_HIST(trace_length, trace_length_hist);

return trace_length;

}

#undef RESERVE

#undef RESERVE_RAW

#undef INSTR_IP

#undef ADD_TO_TRACE

#undef DPRINTF

#define UNSET_BIT(array, bit) (array[(bit)>>5] &= ~(1<<((bit)&31)))

#define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31)))

#define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31)))

/* Count the number of unused uops and exits

*/

static int

count_exits(_PyUOpInstruction *buffer, int length)

{

int exit_count = 0;

for (int i = 0; i < length; i++) {

int opcode = buffer[i].opcode;

if (opcode == _EXIT_TRACE) {

exit_count++;