From 0d4d7e4e285d8ce6ddc6fc694ca74f7591c60f21 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 10 Jun 2020 08:38:15 +0100 Subject: [PATCH 01/10] Merge gen and frame state variables into one. --- Include/cpython/frameobject.h | 23 ++++++++++- Include/genobject.h | 2 - Lib/test/test_generators.py | 2 +- Lib/test/test_sys.py | 2 +- Lib/test/test_yield_from.py | 3 ++ Modules/_xxsubinterpretersmodule.c | 2 +- Objects/frameobject.c | 49 ++++++++++++------------ Objects/genobject.c | 61 ++++++++++++++++++++---------- Python/ceval.c | 16 ++++++-- 9 files changed, 106 insertions(+), 54 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index c76fbe0616cb25..8a1b1207abddbe 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,6 +4,16 @@ # error "this header file must not be included directly" #endif +/* These values are chosen so that all tests involve comparing to zero. */ +typedef enum _framestate { + FRAME_CREATED = -2, + FRAME_SUSPENDED = -1, + FRAME_EXECUTING = 0, + FRAME_RETURNED = 1, + FRAME_RAISED = 2, + FRAME_CLEARED = 3 +} PyFrameState; + typedef struct { int b_type; /* what kind of block this is */ int b_handler; /* where to jump to find handler */ @@ -37,11 +47,22 @@ struct _frame { bytecode index. */ int f_lineno; /* Current line number */ int f_iblock; /* index in f_blockstack */ - char f_executing; /* whether the frame is still executing */ + PyFrameState f_state; /* What state the frame is in */ PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */ PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ }; +static inline int _PyFrameIsRunnable(struct _frame *f) { + return f->f_state < FRAME_EXECUTING; +} + +static inline int _PyFrameIsExecuting(struct _frame *f) { + return f->f_state == FRAME_EXECUTING; +} + +static inline int _PyFrameHasCompleted(struct _frame *f) { + return f->f_state > FRAME_EXECUTING; +} /* Standard object interface */ diff --git a/Include/genobject.h b/Include/genobject.h index 8ffd15646f084e..a76dc92e811c42 100644 --- a/Include/genobject.h +++ b/Include/genobject.h @@ -16,8 +16,6 @@ extern "C" { PyObject_HEAD \ /* Note: gi_frame can be NULL if the generator is "finished" */ \ PyFrameObject *prefix##_frame; \ - /* True if generator is being executed. */ \ - char prefix##_running; \ /* The code object backing the generator */ \ PyObject *prefix##_code; \ /* List of weak reference. */ \ diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 3bf152280868e8..a634ccd517399e 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -881,7 +881,7 @@ def b(): >>> i.gi_running = 42 Traceback (most recent call last): ... -AttributeError: readonly attribute +AttributeError: attribute 'gi_running' of 'generator' objects is not writable >>> def g(): ... yield me.gi_running >>> me = g() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index aaba6630ff4396..433ed2bab75581 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1253,7 +1253,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('Pb2PPP4P')) + check(get_gen(), size('P2PPP4P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 4735ef4bee3b36..d105d8c6eb513c 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -938,6 +938,9 @@ def two(): res.append(g1.throw(MyErr)) except StopIteration: pass + except: + self.assertEqual(res, [0, 1, 2, 3]) + raise # Check with close class MyIt(object): def __iter__(self): diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 8a6fce9e0b4bd9..4fb8e22b5bed19 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1847,7 +1847,7 @@ _is_running(PyInterpreterState *interp) return 0; } - int executing = (int)(frame->f_executing); + int executing = _PyFrameIsExecuting(frame); Py_DECREF(frame); return executing; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7c2bce36158607..c83408390d5886 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -352,33 +352,32 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } - /* Upon the 'call' trace event of a new frame, f->f_lasti is -1 and - * f->f_trace is NULL, check first on the first condition. - * Forbidding jumps from the 'call' event of a new frame is a side effect - * of allowing to set f_lineno only from trace functions. */ - if (f->f_lasti == -1) { - PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' trace event of a new frame"); - return -1; - } - - /* You can only do this from within a trace function, not via - * _getframe or similar hackery. */ - if (!f->f_trace) { - PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a trace function"); - return -1; - } - /* Forbid jumps upon a 'return' trace event (except after executing a - * YIELD_VALUE or YIELD_FROM opcode, f_stacktop is not NULL in that case) + * YIELD_VALUE or YIELD_FROM opcode) * and upon an 'exception' trace event. * Jumps from 'call' trace events have already been forbidden above for new * frames, so this check does not change anything for 'call' events. */ - if (f->f_stacktop == NULL) { - PyErr_SetString(PyExc_ValueError, + switch(f->f_state) { + case FRAME_CREATED: + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + case FRAME_RETURNED: + case FRAME_RAISED: + case FRAME_CLEARED: + PyErr_SetString(PyExc_ValueError, "can only jump from a 'line' trace event"); - return -1; + return -1; + case FRAME_EXECUTING: + case FRAME_SUSPENDED: + /* You can only do this from within a trace function, not via + * _getframe or similar hackery. */ + if (!f->f_trace) { + PyErr_Format(PyExc_ValueError, + "f_lineno can only be set by a trace function"); + return -1; + } + break; } int new_lineno; @@ -665,7 +664,7 @@ frame_tp_clear(PyFrameObject *f) */ PyObject **oldtop = f->f_stacktop; f->f_stacktop = NULL; - f->f_executing = 0; + f->f_state = FRAME_CLEARED; Py_CLEAR(f->f_trace); @@ -687,7 +686,7 @@ frame_tp_clear(PyFrameObject *f) static PyObject * frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { - if (f->f_executing) { + if (_PyFrameIsExecuting(f)) { PyErr_SetString(PyExc_RuntimeError, "cannot clear an executing frame"); return NULL; @@ -927,7 +926,7 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code, f->f_lasti = -1; f->f_lineno = code->co_firstlineno; f->f_iblock = 0; - f->f_executing = 0; + f->f_state = FRAME_CREATED; f->f_gen = NULL; f->f_trace_opcodes = 0; f->f_trace_lines = 1; diff --git a/Objects/genobject.c b/Objects/genobject.c index a379fa6088e16a..9f9206443a2955 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -47,7 +47,7 @@ _PyGen_Finalize(PyObject *self) PyObject *res = NULL; PyObject *error_type, *error_value, *error_traceback; - if (gen->gi_frame == NULL || gen->gi_frame->f_stacktop == NULL) { + if (gen->gi_frame == NULL || _PyFrameHasCompleted(gen->gi_frame)) { /* Generator isn't paused, so no need to close */ return; } @@ -143,7 +143,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) PyFrameObject *f = gen->gi_frame; PyObject *result; - if (gen->gi_running) { + if (f != NULL && _PyFrameIsExecuting(f)) { const char *msg = "generator already executing"; if (PyCoro_CheckExact(gen)) { msg = "coroutine already executing"; @@ -154,7 +154,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) PyErr_SetString(PyExc_ValueError, msg); return NULL; } - if (f == NULL || f->f_stacktop == NULL) { + if (f == NULL || _PyFrameHasCompleted(f)) { if (PyCoro_CheckExact(gen) && !closing) { /* `gen` is an exhausted coroutine: raise an error, except when called from gen_close(), which should @@ -176,6 +176,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) return NULL; } + assert(_PyFrameIsRunnable(f)); if (f->f_lasti == -1) { if (arg && arg != Py_None) { const char *msg = "can't send non-None value to a " @@ -203,7 +204,6 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) assert(f->f_back == NULL); f->f_back = tstate->frame; - gen->gi_running = 1; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -215,7 +215,6 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) result = _PyEval_EvalFrame(tstate, f, exc); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; - gen->gi_running = 0; /* Don't keep the reference to f_back any longer than necessary. It * may keep a chain of frames alive or it could create a reference @@ -225,7 +224,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ - if (result && f->f_stacktop == NULL) { + if (result && _PyFrameHasCompleted(f)) { if (result == Py_None) { /* Delay exception instantiation if we can */ if (PyAsyncGen_CheckExact(gen)) { @@ -264,7 +263,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) _PyErr_FormatFromCause(PyExc_RuntimeError, "%s", msg); } - if (!result || f->f_stacktop == NULL) { + if (!result || _PyFrameHasCompleted(f)) { /* generator can't be rerun, so release the frame */ /* first clean reference cycle through stored exception traceback */ _PyErr_ClearExcState(&gen->gi_exc_state); @@ -356,9 +355,10 @@ gen_close(PyGenObject *gen, PyObject *args) int err = 0; if (yf) { - gen->gi_running = 1; + PyFrameState state = gen->gi_frame->f_state; + gen->gi_frame->f_state = FRAME_EXECUTING; err = gen_close_iter(yf); - gen->gi_running = 0; + gen->gi_frame->f_state = state; Py_DECREF(yf); } if (err == 0) @@ -405,9 +405,10 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, We have to allow some awaits to work it through, hence the `close_on_genexit` parameter here. */ - gen->gi_running = 1; + PyFrameState state = gen->gi_frame->f_state; + gen->gi_frame->f_state = FRAME_EXECUTING; err = gen_close_iter(yf); - gen->gi_running = 0; + gen->gi_frame->f_state = state; Py_DECREF(yf); if (err < 0) return gen_send_ex(gen, Py_None, 1, 0); @@ -418,7 +419,6 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyThreadState *tstate = _PyThreadState_GET(); PyFrameObject *f = tstate->frame; - gen->gi_running = 1; /* Since we are fast-tracking things by skipping the eval loop, we need to update the current frame so the stack trace will be reported correctly to the user. */ @@ -427,10 +427,12 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, tstate->frame = gen->gi_frame; /* Close the generator that we are currently iterating with 'yield from' or awaiting on with 'await'. */ + PyFrameState state = gen->gi_frame->f_state; + gen->gi_frame->f_state = FRAME_EXECUTING; ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); tstate->frame = f; - gen->gi_running = 0; + gen->gi_frame->f_state = state; } else { /* `yf` is an iterator or a coroutine-like object. */ PyObject *meth; @@ -442,9 +444,10 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, Py_DECREF(yf); goto throw_here; } - gen->gi_running = 1; + PyFrameState state = gen->gi_frame->f_state; + gen->gi_frame->f_state = FRAME_EXECUTING; ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); - gen->gi_running = 0; + gen->gi_frame->f_state = state; Py_DECREF(meth); } Py_DECREF(yf); @@ -701,6 +704,17 @@ gen_getyieldfrom(PyGenObject *gen, void *Py_UNUSED(ignored)) return yf; } + +static PyObject * +gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) +{ + if (gen->gi_frame == NULL) { + Py_INCREF(Py_False); + return Py_False; + } + return PyBool_FromLong(_PyFrameIsExecuting(gen->gi_frame)); +} + static PyGetSetDef gen_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the generator")}, @@ -708,12 +722,12 @@ static PyGetSetDef gen_getsetlist[] = { PyDoc_STR("qualified name of the generator")}, {"gi_yieldfrom", (getter)gen_getyieldfrom, NULL, PyDoc_STR("object being iterated by yield from, or None")}, + {"gi_running", (getter)gen_getrunning, NULL, NULL}, {NULL} /* Sentinel */ }; static PyMemberDef gen_memberlist[] = { {"gi_frame", T_OBJECT, offsetof(PyGenObject, gi_frame), READONLY}, - {"gi_running", T_BOOL, offsetof(PyGenObject, gi_running), READONLY}, {"gi_code", T_OBJECT, offsetof(PyGenObject, gi_code), READONLY}, {NULL} /* Sentinel */ }; @@ -791,7 +805,6 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, f->f_gen = (PyObject *) gen; Py_INCREF(f->f_code); gen->gi_code = (PyObject *)(f->f_code); - gen->gi_running = 0; gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_type = NULL; gen->gi_exc_state.exc_value = NULL; @@ -921,6 +934,16 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) return yf; } +static PyObject * +cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) +{ + if (coro->cr_frame == NULL) { + Py_INCREF(Py_False); + return Py_False; + } + return PyBool_FromLong(_PyFrameIsExecuting(coro->cr_frame)); +} + static PyGetSetDef coro_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the coroutine")}, @@ -928,12 +951,12 @@ static PyGetSetDef coro_getsetlist[] = { PyDoc_STR("qualified name of the coroutine")}, {"cr_await", (getter)coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, + {"cr_running", (getter)cr_getrunning, NULL, NULL}, {NULL} /* Sentinel */ }; static PyMemberDef coro_memberlist[] = { {"cr_frame", T_OBJECT, offsetof(PyCoroObject, cr_frame), READONLY}, - {"cr_running", T_BOOL, offsetof(PyCoroObject, cr_running), READONLY}, {"cr_code", T_OBJECT, offsetof(PyCoroObject, cr_code), READONLY}, {"cr_origin", T_OBJECT, offsetof(PyCoroObject, cr_origin), READONLY}, {NULL} /* Sentinel */ @@ -1828,7 +1851,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) return NULL; } - if (f == NULL || f->f_stacktop == NULL) { + if (f == NULL || _PyFrameHasCompleted(f)) { o->agt_state = AWAITABLE_STATE_CLOSED; PyErr_SetNone(PyExc_StopIteration); return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 0386929a5b2b37..2cdf2881ea8471 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1349,10 +1349,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) assert(f->f_lasti % sizeof(_Py_CODEUNIT) == 0); next_instr += f->f_lasti / sizeof(_Py_CODEUNIT) + 1; } + f->f_state = FRAME_EXECUTING; stack_pointer = f->f_stacktop; assert(stack_pointer != NULL); f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ - f->f_executing = 1; + f->f_state = FRAME_EXECUTING; if (co->co_opcache_flag < OPCACHE_MIN_RUNS) { co->co_opcache_flag++; @@ -2076,6 +2077,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) retval = POP(); assert(f->f_iblock == 0); assert(EMPTY()); + f->f_state = FRAME_RETURNED; goto exiting; } @@ -2246,6 +2248,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) /* and repeat... */ assert(f->f_lasti >= (int)sizeof(_Py_CODEUNIT)); f->f_lasti -= sizeof(_Py_CODEUNIT); + f->f_state = FRAME_SUSPENDED; goto exiting; } @@ -2263,6 +2266,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) } f->f_stacktop = stack_pointer; + f->f_state = FRAME_SUSPENDED; goto exiting; } @@ -3762,10 +3766,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) /* Log traceback info. */ PyTraceBack_Here(f); - if (tstate->c_tracefunc != NULL) + if (tstate->c_tracefunc != NULL) { + /* Temporarily set frame state to RAISED for benefit of frame.setlineno */ + assert(f->f_state == FRAME_EXECUTING); + f->f_state = FRAME_RAISED; call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f); - + f->f_state = FRAME_EXECUTING; + } exception_unwind: /* Unwind stacks if an exception occurred */ while (f->f_iblock > 0) { @@ -3842,6 +3850,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) Py_XDECREF(o); } + f->f_state = FRAME_RAISED; exiting: if (tstate->use_tracing) { if (tstate->c_tracefunc) { @@ -3863,7 +3872,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) if (PyDTrace_FUNCTION_RETURN_ENABLED()) dtrace_function_return(f); _Py_LeaveRecursiveCall(tstate); - f->f_executing = 0; tstate->frame = f->f_back; return _Py_CheckFunctionResult(tstate, NULL, retval, __func__); From a7e7f75bd19ebfbd0315932f1a3a807c0d1649fc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jun 2020 13:45:12 +0100 Subject: [PATCH 02/10] Replace stack pointer with depth in PyFrameObject. Makes code easier to read and saves a word of memory. --- Include/cpython/frameobject.h | 5 +---- Lib/test/test_sys.py | 2 +- Objects/frameobject.c | 29 ++++++++++++----------------- Objects/genobject.c | 11 +++++++---- Python/ceval.c | 19 +++++++------------ 5 files changed, 28 insertions(+), 38 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 8a1b1207abddbe..fe8f1f7f617644 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -28,11 +28,8 @@ struct _frame { PyObject *f_globals; /* global symbol table (PyDictObject) */ PyObject *f_locals; /* local symbol table (any mapping) */ PyObject **f_valuestack; /* points after the last local */ - /* Next free slot in f_valuestack. Frame creation sets to f_valuestack. - Frame evaluation usually NULLs it, but a frame that yields sets it - to the current stack top. */ - PyObject **f_stacktop; PyObject *f_trace; /* Trace function */ + int f_stackdepth; /* Depth of value stack */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 433ed2bab75581..7a5df46b3e3fdf 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1236,7 +1236,7 @@ class C(object): pass nfrees = len(x.f_code.co_freevars) extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ ncells + nfrees - 1 - check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + check(x, vsize('4Pi2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass check(func, size('13P')) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c83408390d5886..26723818d0c4f7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -300,7 +300,8 @@ first_line_not_before(int *lines, int len, int line) static void frame_stack_pop(PyFrameObject *f) { - PyObject *v = (*--f->f_stacktop); + f->f_stackdepth--; + PyObject *v = f->f_valuestack[f->f_stackdepth]; Py_DECREF(v); } @@ -310,7 +311,7 @@ frame_block_unwind(PyFrameObject *f) assert(f->f_iblock > 0); f->f_iblock--; PyTryBlock *b = &f->f_blockstack[f->f_iblock]; - intptr_t delta = (f->f_stacktop - f->f_valuestack) - b->b_level; + intptr_t delta = f->f_stackdepth - b->b_level; while (delta > 0) { frame_stack_pop(f); delta--; @@ -584,11 +585,10 @@ frame_dealloc(PyFrameObject *f) } /* Free stack */ - if (f->f_stacktop != NULL) { - for (PyObject **p = valuestack; p < f->f_stacktop; p++) { - Py_XDECREF(*p); - } + for (int i = 0; i < f->f_stackdepth; i++) { + Py_XDECREF(f->f_valuestack[i]); } + f->f_stackdepth = 0; Py_XDECREF(f->f_back); Py_DECREF(f->f_builtins); @@ -646,10 +646,8 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) } /* stack */ - if (f->f_stacktop != NULL) { - for (PyObject **p = f->f_valuestack; p < f->f_stacktop; p++) { - Py_VISIT(*p); - } + for (int i = 0; i < f->f_stackdepth; i++) { + Py_VISIT(f->f_valuestack[i]); } return 0; } @@ -662,8 +660,6 @@ frame_tp_clear(PyFrameObject *f) * frame may also point to this frame, believe itself to still be * active, and try cleaning up this frame again. */ - PyObject **oldtop = f->f_stacktop; - f->f_stacktop = NULL; f->f_state = FRAME_CLEARED; Py_CLEAR(f->f_trace); @@ -675,11 +671,10 @@ frame_tp_clear(PyFrameObject *f) } /* stack */ - if (oldtop != NULL) { - for (PyObject **p = f->f_valuestack; p < oldtop; p++) { - Py_CLEAR(*p); - } + for (int i = 0; i < f->f_stackdepth; i++) { + Py_CLEAR(f->f_valuestack[i]); } + f->f_stackdepth = 0; return 0; } @@ -897,7 +892,7 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code, return NULL; } - f->f_stacktop = f->f_valuestack; + f->f_stackdepth = 0; f->f_builtins = builtins; Py_XINCREF(back); f->f_back = back; diff --git a/Objects/genobject.c b/Objects/genobject.c index 9f9206443a2955..6eac94c087e82b 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -195,7 +195,8 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) /* Push arg onto the frame's value stack */ result = arg ? arg : Py_None; Py_INCREF(result); - *(f->f_stacktop++) = result; + gen->gi_frame->f_valuestack[gen->gi_frame->f_stackdepth] = result; + gen->gi_frame->f_stackdepth++; } /* Generators always return to their most recent caller, not @@ -326,7 +327,7 @@ _PyGen_yf(PyGenObject *gen) PyObject *yf = NULL; PyFrameObject *f = gen->gi_frame; - if (f && f->f_stacktop) { + if (f) { PyObject *bytecode = f->f_code->co_code; unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); @@ -340,7 +341,8 @@ _PyGen_yf(PyGenObject *gen) if (code[f->f_lasti + sizeof(_Py_CODEUNIT)] != YIELD_FROM) return NULL; - yf = f->f_stacktop[-1]; + assert(f->f_stackdepth > 0); + yf = f->f_valuestack[f->f_stackdepth-1]; Py_INCREF(yf); } @@ -454,7 +456,8 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, if (!ret) { PyObject *val; /* Pop subiterator from stack */ - ret = *(--gen->gi_frame->f_stacktop); + gen->gi_frame->f_stackdepth--; + ret = gen->gi_frame->f_valuestack[gen->gi_frame->f_stackdepth]; assert(ret == yf); Py_DECREF(ret); /* Termination repetition of YIELD_FROM */ diff --git a/Python/ceval.c b/Python/ceval.c index 2cdf2881ea8471..3989ed4531e1c4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1349,10 +1349,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) assert(f->f_lasti % sizeof(_Py_CODEUNIT) == 0); next_instr += f->f_lasti / sizeof(_Py_CODEUNIT) + 1; } - f->f_state = FRAME_EXECUTING; - stack_pointer = f->f_stacktop; - assert(stack_pointer != NULL); - f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */ + stack_pointer = f->f_valuestack + f->f_stackdepth; + /* To help with debugging, set f->f_stackdepth to -1. + * Update when returning or calling trace function */ + f->f_stackdepth = -1; f->f_state = FRAME_EXECUTING; if (co->co_opcache_flag < OPCACHE_MIN_RUNS) { @@ -1441,7 +1441,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) int err; /* see maybe_call_line_trace for expository comments */ - f->f_stacktop = stack_pointer; + f->f_stackdepth = stack_pointer-f->f_valuestack; err = maybe_call_line_trace(tstate->c_tracefunc, tstate->c_traceobj, @@ -1449,10 +1449,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) &instr_lb, &instr_ub, &instr_prev); /* Reload possibly changed frame fields */ JUMPTO(f->f_lasti); - if (f->f_stacktop != NULL) { - stack_pointer = f->f_stacktop; - f->f_stacktop = NULL; - } + stack_pointer = f->f_valuestack+f->f_stackdepth; if (err) /* trace function raised an exception */ goto error; @@ -2244,7 +2241,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) DISPATCH(); } /* receiver remains on stack, retval is value to be yielded */ - f->f_stacktop = stack_pointer; /* and repeat... */ assert(f->f_lasti >= (int)sizeof(_Py_CODEUNIT)); f->f_lasti -= sizeof(_Py_CODEUNIT); @@ -2264,8 +2260,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) } retval = w; } - - f->f_stacktop = stack_pointer; f->f_state = FRAME_SUSPENDED; goto exiting; } @@ -3852,6 +3846,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) f->f_state = FRAME_RAISED; exiting: + f->f_stackdepth = stack_pointer-f->f_valuestack; if (tstate->use_tracing) { if (tstate->c_tracefunc) { if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, From fe005138da9e2752b648ac936587f76d45c4a748 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 15 Jun 2020 13:30:42 +0100 Subject: [PATCH 03/10] Tweak handling of f_stackdepth. --- Objects/frameobject.c | 2 ++ Python/ceval.c | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 26723818d0c4f7..ede5b96af1c4ce 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -300,6 +300,7 @@ first_line_not_before(int *lines, int len, int line) static void frame_stack_pop(PyFrameObject *f) { + assert(f->f_stackdepth >= 0); f->f_stackdepth--; PyObject *v = f->f_valuestack[f->f_stackdepth]; Py_DECREF(v); @@ -308,6 +309,7 @@ frame_stack_pop(PyFrameObject *f) static void frame_block_unwind(PyFrameObject *f) { + assert(f->f_stackdepth >= 0); assert(f->f_iblock > 0); f->f_iblock--; PyTryBlock *b = &f->f_blockstack[f->f_iblock]; diff --git a/Python/ceval.c b/Python/ceval.c index 3989ed4531e1c4..ae8cfd2bb60b02 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1350,8 +1350,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) next_instr += f->f_lasti / sizeof(_Py_CODEUNIT) + 1; } stack_pointer = f->f_valuestack + f->f_stackdepth; - /* To help with debugging, set f->f_stackdepth to -1. - * Update when returning or calling trace function */ + /* Set f->f_stackdepth to -1. + * Update when returning or calling trace function. + Having f_stackdepth <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. + */ f->f_stackdepth = -1; f->f_state = FRAME_EXECUTING; @@ -1450,6 +1454,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) /* Reload possibly changed frame fields */ JUMPTO(f->f_lasti); stack_pointer = f->f_valuestack+f->f_stackdepth; + f->f_stackdepth = -1; if (err) /* trace function raised an exception */ goto error; @@ -2075,6 +2080,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) assert(f->f_iblock == 0); assert(EMPTY()); f->f_state = FRAME_RETURNED; + f->f_stackdepth = 0; goto exiting; } @@ -2245,6 +2251,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) assert(f->f_lasti >= (int)sizeof(_Py_CODEUNIT)); f->f_lasti -= sizeof(_Py_CODEUNIT); f->f_state = FRAME_SUSPENDED; + f->f_stackdepth = stack_pointer-f->f_valuestack; goto exiting; } @@ -2261,6 +2268,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) retval = w; } f->f_state = FRAME_SUSPENDED; + f->f_stackdepth = stack_pointer-f->f_valuestack; goto exiting; } @@ -3843,10 +3851,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) PyObject *o = POP(); Py_XDECREF(o); } - + f->f_stackdepth = 0; f->f_state = FRAME_RAISED; exiting: - f->f_stackdepth = stack_pointer-f->f_valuestack; if (tstate->use_tracing) { if (tstate->c_tracefunc) { if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, From eeb0e4dc612c54bc47c9d6d0427ea5a77ceaf465 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jul 2020 15:50:42 +0100 Subject: [PATCH 04/10] Change _PyFrameIsExecuting to _PyFrame_IsExecuting for improved legibility. --- Include/cpython/frameobject.h | 4 ++-- Modules/_xxsubinterpretersmodule.c | 2 +- Objects/frameobject.c | 2 +- Objects/genobject.c | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index fe8f1f7f617644..a990f830f38a64 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -49,11 +49,11 @@ struct _frame { PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ }; -static inline int _PyFrameIsRunnable(struct _frame *f) { +static inline int _PyFrame_IsExecutingRunnable(struct _frame *f) { return f->f_state < FRAME_EXECUTING; } -static inline int _PyFrameIsExecuting(struct _frame *f) { +static inline int _PyFrame_IsExecuting(struct _frame *f) { return f->f_state == FRAME_EXECUTING; } diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 4fb8e22b5bed19..8943ebf455df4b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1847,7 +1847,7 @@ _is_running(PyInterpreterState *interp) return 0; } - int executing = _PyFrameIsExecuting(frame); + int executing = _PyFrame_IsExecuting(frame); Py_DECREF(frame); return executing; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ede5b96af1c4ce..f3bef9953e8be7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -683,7 +683,7 @@ frame_tp_clear(PyFrameObject *f) static PyObject * frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { - if (_PyFrameIsExecuting(f)) { + if (_PyFrame_IsExecuting(f)) { PyErr_SetString(PyExc_RuntimeError, "cannot clear an executing frame"); return NULL; diff --git a/Objects/genobject.c b/Objects/genobject.c index 6eac94c087e82b..19b4230dff53ed 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -143,7 +143,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) PyFrameObject *f = gen->gi_frame; PyObject *result; - if (f != NULL && _PyFrameIsExecuting(f)) { + if (f != NULL && _PyFrame_IsExecuting(f)) { const char *msg = "generator already executing"; if (PyCoro_CheckExact(gen)) { msg = "coroutine already executing"; @@ -715,7 +715,7 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) Py_INCREF(Py_False); return Py_False; } - return PyBool_FromLong(_PyFrameIsExecuting(gen->gi_frame)); + return PyBool_FromLong(_PyFrame_IsExecuting(gen->gi_frame)); } static PyGetSetDef gen_getsetlist[] = { @@ -944,7 +944,7 @@ cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) Py_INCREF(Py_False); return Py_False; } - return PyBool_FromLong(_PyFrameIsExecuting(coro->cr_frame)); + return PyBool_FromLong(_PyFrame_IsExecuting(coro->cr_frame)); } static PyGetSetDef coro_getsetlist[] = { From 286f281ac89a9e870288733a90595df5ad567719 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jul 2020 16:11:15 +0100 Subject: [PATCH 05/10] Reword comment in frame_setlineno for clarity. --- Objects/frameobject.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f3bef9953e8be7..2e1af955bc43b8 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -355,11 +355,14 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } - /* Forbid jumps upon a 'return' trace event (except after executing a - * YIELD_VALUE or YIELD_FROM opcode) - * and upon an 'exception' trace event. - * Jumps from 'call' trace events have already been forbidden above for new - * frames, so this check does not change anything for 'call' events. */ + /* + * This code preserves the historical restrictions on + * setting the line number of a frame. + * Jumps are forbidden on a 'return' trace event (except after a yield). + * Jumps from 'call' trace events are also forbidden. + * In addition, jumps are forbidden when not tracing, + * as this is a debugging feature. + */ switch(f->f_state) { case FRAME_CREATED: PyErr_Format(PyExc_ValueError, From 39d09966656a3c5b8debf466addb99c4933331d5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jul 2020 16:15:45 +0100 Subject: [PATCH 06/10] Fix messed up replace. --- Include/cpython/frameobject.h | 2 +- Objects/genobject.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index a990f830f38a64..e02fed84e5d21b 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -49,7 +49,7 @@ struct _frame { PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ }; -static inline int _PyFrame_IsExecutingRunnable(struct _frame *f) { +static inline int _PyFrame_IsRunnable(struct _frame *f) { return f->f_state < FRAME_EXECUTING; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 19b4230dff53ed..36070dceab4d3d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -176,7 +176,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) return NULL; } - assert(_PyFrameIsRunnable(f)); + assert(_PyFrame_IsRunnable(f)); if (f->f_lasti == -1) { if (arg && arg != Py_None) { const char *msg = "can't send non-None value to a " From 15a965e01d51b90d917f8d6d584de56d84aad0ba Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jul 2020 16:40:40 +0100 Subject: [PATCH 07/10] Change f_state to use a single byte. --- Include/cpython/frameobject.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index e02fed84e5d21b..37ee7d1fad61a7 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,15 +4,19 @@ # error "this header file must not be included directly" #endif -/* These values are chosen so that all tests involve comparing to zero. */ -typedef enum _framestate { +/* These values are chosen so that the inline functions below all + * compare f_state to zero. + */ +enum _framestate { FRAME_CREATED = -2, FRAME_SUSPENDED = -1, FRAME_EXECUTING = 0, FRAME_RETURNED = 1, FRAME_RAISED = 2, FRAME_CLEARED = 3 -} PyFrameState; +}; + +typedef signed char PyFrameState; typedef struct { int b_type; /* what kind of block this is */ From a22412c5430b3e604fb2d7f765733692f0da7373 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jul 2020 17:00:39 +0100 Subject: [PATCH 08/10] minor tidy ups to genobject.c --- Objects/genobject.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 36070dceab4d3d..acff593f6e12d2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -456,6 +456,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, if (!ret) { PyObject *val; /* Pop subiterator from stack */ + assert(gen->gi_frame->f_stackdepth > 0); gen->gi_frame->f_stackdepth--; ret = gen->gi_frame->f_valuestack[gen->gi_frame->f_stackdepth]; assert(ret == yf); @@ -712,8 +713,7 @@ static PyObject * gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) { if (gen->gi_frame == NULL) { - Py_INCREF(Py_False); - return Py_False; + Py_RETURN_FALSE; } return PyBool_FromLong(_PyFrame_IsExecuting(gen->gi_frame)); } @@ -941,8 +941,7 @@ static PyObject * cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) { if (coro->cr_frame == NULL) { - Py_INCREF(Py_False); - return Py_False; + Py_RETURN_FALSE; } return PyBool_FromLong(_PyFrame_IsExecuting(coro->cr_frame)); } From 38792f240769fd4e707185898f8e4358fab67c27 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Jul 2020 17:09:41 +0100 Subject: [PATCH 09/10] Add FRAME_UNWINDING to mark unwinding state. --- Include/cpython/frameobject.h | 5 +++-- Objects/frameobject.c | 1 + Python/ceval.c | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 37ee7d1fad61a7..63240b5b6d5ccb 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -12,8 +12,9 @@ enum _framestate { FRAME_SUSPENDED = -1, FRAME_EXECUTING = 0, FRAME_RETURNED = 1, - FRAME_RAISED = 2, - FRAME_CLEARED = 3 + FRAME_UNWINDING = 2, + FRAME_RAISED = 3, + FRAME_CLEARED = 4 }; typedef signed char PyFrameState; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2e1af955bc43b8..8838b807462e4a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -369,6 +369,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore "can't jump from the 'call' trace event of a new frame"); return -1; case FRAME_RETURNED: + case FRAME_UNWINDING: case FRAME_RAISED: case FRAME_CLEARED: PyErr_SetString(PyExc_ValueError, diff --git a/Python/ceval.c b/Python/ceval.c index ae8cfd2bb60b02..f747faaebf024a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3769,14 +3769,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) PyTraceBack_Here(f); if (tstate->c_tracefunc != NULL) { - /* Temporarily set frame state to RAISED for benefit of frame.setlineno */ + /* Make sure state is set to FRAME_EXECUTING for tracing */ assert(f->f_state == FRAME_EXECUTING); - f->f_state = FRAME_RAISED; + f->f_state = FRAME_UNWINDING; call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f); - f->f_state = FRAME_EXECUTING; } exception_unwind: + f->f_state = FRAME_UNWINDING; /* Unwind stacks if an exception occurred */ while (f->f_iblock > 0) { /* Pop the current block. */ @@ -3835,6 +3835,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) } } /* Resume normal execution */ + f->f_state = FRAME_EXECUTING; goto main_loop; } } /* unwind stack */ From 21a8f2b952f8961b6be5dcc20cb0d30e9fc3fa3c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 15 Jul 2020 15:45:36 +0100 Subject: [PATCH 10/10] Make state changes around call to _gen_throw symmetrical. --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index acff593f6e12d2..809838a4cd2f3b 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -433,8 +433,8 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, gen->gi_frame->f_state = FRAME_EXECUTING; ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); - tstate->frame = f; gen->gi_frame->f_state = state; + tstate->frame = f; } else { /* `yf` is an iterator or a coroutine-like object. */ PyObject *meth;