Thanks to visit codestin.com
Credit goes to github.com

Skip to content

bpo-43846: Use less stack for large literals and calls #25403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
markshannon merged 6 commits into from
Apr 15, 2021
Merged

bpo-43846: Use less stack for large literals and calls #25403

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
67c7d87
Modify compiler to reduce stack consumption for large expressions.
markshannon Apr 14, 2021
a5afaad
Add more tests for stack usage.
markshannon Apr 14, 2021
8c3cc5c
Add NEWS item.
markshannon Apr 14, 2021
87b7b6d
Remove assertion and raise SystemError for truly excessive stack use.
markshannon Apr 14, 2021
a523a6e
Clarify comments for STACK_USE_GUIDELINE and MAX_ALLOWED_STACK_USE.
markshannon Apr 14, 2021
95feeb8
Clean up starunpack_helper. Make sure all tests against STACK_USE_GUI…
markshannon Apr 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions Lib/test/test_compile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,32 @@ def test_if_else(self):
def test_binop(self):
self.check_stack_size("x + " * self.N + "x")

def test_list(self):
self.check_stack_size("[" + "x, " * self.N + "x]")

def test_tuple(self):
self.check_stack_size("(" + "x, " * self.N + "x)")

def test_set(self):
self.check_stack_size("{" + "x, " * self.N + "x}")

def test_dict(self):
self.check_stack_size("{" + "x:x, " * self.N + "x:x}")

def test_func_args(self):
self.check_stack_size("f(" + "x, " * self.N + ")")

def test_func_kwargs(self):
kwargs = (f'a{i}=x' for i in range(self.N))
self.check_stack_size("f(" + ", ".join(kwargs) + ")")

def test_func_args(self):
self.check_stack_size("o.m(" + "x, " * self.N + ")")

def test_meth_kwargs(self):
kwargs = (f'a{i}=x' for i in range(self.N))
self.check_stack_size("o.m(" + ", ".join(kwargs) + ")")

def test_func_and(self):
code = "def f(x):\n"
code += " x and x\n" * self.N
Expand Down
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Core and Builtins/2021-04-14-13-53-08.bpo-43846.2jO97c.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Data stack usage is much reduced for large literal and call expressions.
187 changes: 141 additions & 46 deletions Python/compile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,30 @@
#define COMP_SETCOMP 2
#define COMP_DICTCOMP 3

/* A soft limit for stack use, to avoid excessive
* memory use for large constants, etc.
*
* The value 30 is plucked out of thin air.
* Code that could use more stack than this is
* rare, so the exact value is unimportant.
*/
#define STACK_USE_GUIDELINE 30

/* If we exceed this limit, it should
* be considered a compiler bug.
* Currently it should be impossible
* to exceed STACK_USE_GUIDELINE * 100,
* as 100 is the maximum parse depth.
* For performance reasons we will
* want to reduce this to a
* few hundred in the future.
*
* NOTE: Whatever MAX_ALLOWED_STACK_USE is
* set to, it should never restrict what Python
* we can write, just how we compile it.
*/
#define MAX_ALLOWED_STACK_USE (STACK_USE_GUIDELINE * 100)

#define IS_TOP_LEVEL_AWAIT(c) ( \
(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \
&& (c->u->u_ste->ste_type == ModuleBlock))
Expand Down Expand Up @@ -1403,7 +1427,7 @@ compiler_addop_name(struct compiler *c, int opcode, PyObject *dict,
*/

static int
compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
compiler_addop_i_line(struct compiler *c, int opcode, Py_ssize_t oparg, int lineno)
{
struct instr *i;
int off;
Expand All @@ -1424,10 +1448,22 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
i = &c->u->u_curblock->b_instr[off];
i->i_opcode = opcode;
i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
i->i_lineno = c->u->u_lineno;
i->i_lineno = lineno;
return 1;
}

static int
compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
{
return compiler_addop_i_line(c, opcode, oparg, c->u->u_lineno);
}

static int
compiler_addop_i_noline(struct compiler *c, int opcode, Py_ssize_t oparg)
{
return compiler_addop_i_line(c, opcode, oparg, -1);
}

static int add_jump_to_block(basicblock *b, int opcode, int lineno, basicblock *target)
{
assert(HAS_ARG(opcode));
Expand Down Expand Up @@ -1527,6 +1563,11 @@ compiler_addop_j_noline(struct compiler *c, int opcode, basicblock *b)
return 0; \
}

#define ADDOP_I_NOLINE(C, OP, O) { \
if (!compiler_addop_i_noline((C), (OP), (O))) \
return 0; \
}

#define ADDOP_JUMP(C, OP, O) { \
if (!compiler_addop_j((C), (OP), (O))) \
return 0; \
Expand Down Expand Up @@ -3741,31 +3782,7 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed,
seen_star = 1;
}
}
if (seen_star) {
seen_star = 0;
for (i = 0; i < n; i++) {
expr_ty elt = asdl_seq_GET(elts, i);
if (elt->kind == Starred_kind) {
if (seen_star == 0) {
ADDOP_I(c, build, i+pushed);
seen_star = 1;
}
VISIT(c, expr, elt->v.Starred.value);
ADDOP_I(c, extend, 1);
}
else {
VISIT(c, expr, elt);
if (seen_star) {
ADDOP_I(c, add, 1);
}
}
}
assert(seen_star);
if (tuple) {
ADDOP(c, LIST_TO_TUPLE);
}
}
else {
if (!seen_star && n+pushed < STACK_USE_GUIDELINE) {
for (i = 0; i < n; i++) {
expr_ty elt = asdl_seq_GET(elts, i);
VISIT(c, expr, elt);
Expand All @@ -3775,6 +3792,35 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed,
} else {
ADDOP_I(c, build, n+pushed);
}
return 1;
}
if (n+pushed >= STACK_USE_GUIDELINE) {
ADDOP_I(c, build, pushed);
seen_star = 1;
}
else {
seen_star = 0;
}
Copy link
Copy Markdown
Member

@gvanrossum gvanrossum Apr 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, I suppose you could just set seen_star = 0 here and emit the push in the for-loop below when i+n exceeds the limit (and it hasn't been emitted yet). FWIW I wish during this phase the variable was called something like 'build_emitted', since that is really what it's keeping track of.

Copy link
Copy Markdown
Member Author

@markshannon markshannon Apr 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
I used sequence_built and also cleaned up the surrounding code a bit.

for (i = 0; i < n; i++) {
expr_ty elt = asdl_seq_GET(elts, i);
if (elt->kind == Starred_kind) {
if (seen_star == 0) {
ADDOP_I(c, build, i+pushed);
seen_star = 1;
}
VISIT(c, expr, elt->v.Starred.value);
ADDOP_I(c, extend, 1);
}
else {
VISIT(c, expr, elt);
if (seen_star) {
ADDOP_I(c, add, 1);
}
}
}
assert(seen_star);
if (tuple) {
ADDOP(c, LIST_TO_TUPLE);
}
return 1;
}
Expand Down Expand Up @@ -3874,7 +3920,8 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end
{
Py_ssize_t i, n = end - begin;
PyObject *keys, *key;
if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) {
int big = n > STACK_USE_GUIDELINE/2;
if (n > 1 && !big && are_all_items_const(e->v.Dict.keys, begin, end)) {
for (i = begin; i < end; i++) {
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
}
Expand All @@ -3889,12 +3936,19 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end
}
ADDOP_LOAD_CONST_NEW(c, keys);
ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
return 1;
}
else {
for (i = begin; i < end; i++) {
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
if (big) {
ADDOP_I(c, BUILD_MAP, 0);
}
for (i = begin; i < end; i++) {
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
if (big) {
ADDOP_I(c, MAP_ADD, 1);
}
}
if (!big) {
ADDOP_I(c, BUILD_MAP, n);
}
return 1;
Expand Down Expand Up @@ -3930,7 +3984,7 @@ compiler_dict(struct compiler *c, expr_ty e)
ADDOP_I(c, DICT_UPDATE, 1);
}
else {
if (elements == 0xFFFF) {
if (elements > STACK_USE_GUIDELINE/2) {
if (!compiler_subdict(c, e, i - elements, i + 1)) {
return 0;
}
Expand Down Expand Up @@ -4126,11 +4180,15 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e)
/* Check that the call node is an attribute access, and that
the call doesn't have keyword parameters. */
if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load ||
asdl_seq_LEN(e->v.Call.keywords))
asdl_seq_LEN(e->v.Call.keywords)) {
return -1;

/* Check that there are no *varargs types of arguments. */
}
/* Check that there aren't too many arguments */
argsl = asdl_seq_LEN(args);
if (argsl >= STACK_USE_GUIDELINE) {
return -1;
}
/* Check that there are no *varargs types of arguments. */
for (i = 0; i < argsl; i++) {
expr_ty elt = asdl_seq_GET(args, i);
if (elt->kind == Starred_kind) {
Expand Down Expand Up @@ -4192,9 +4250,29 @@ compiler_call(struct compiler *c, expr_ty e)
static int
compiler_joined_str(struct compiler *c, expr_ty e)
{
VISIT_SEQ(c, expr, e->v.JoinedStr.values);
if (asdl_seq_LEN(e->v.JoinedStr.values) != 1)
ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values));

Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values);
if (value_count > STACK_USE_GUIDELINE) {
Copy link
Copy Markdown
Member

@gvanrossum gvanrossum Apr 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe use >= for consistency with earlier locations?

Copy link
Copy Markdown
Member Author

@markshannon markshannon Apr 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've used > throughout. It seems to fit better with STACK_USE_GUIDELINE being the maximum stack use.

ADDOP_LOAD_CONST_NEW(c, _PyUnicode_FromASCII("", 0));
PyObject *join = _PyUnicode_FromASCII("join", 4);
if (join == NULL) {
return 0;
}
ADDOP_NAME(c, LOAD_METHOD, join, names);
Py_DECREF(join);
ADDOP_I(c, BUILD_LIST, 0);
for (Py_ssize_t i = 0; i < asdl_seq_LEN(e->v.JoinedStr.values); i++) {
VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i));
ADDOP_I(c, LIST_APPEND, 1);
}
ADDOP_I(c, CALL_METHOD, 1);
}
else {
VISIT_SEQ(c, expr, e->v.JoinedStr.values);
if (asdl_seq_LEN(e->v.JoinedStr.values) != 1) {
ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values));
}
}
return 1;
}

Expand Down Expand Up @@ -4251,7 +4329,8 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be
keyword_ty kw;
PyObject *keys, *key;
assert(n > 0);
if (n > 1) {
int big = n > STACK_USE_GUIDELINE/2;
Copy link
Copy Markdown
Member

@gvanrossum gvanrossum Apr 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ditto?

if (n > 1 && !big) {
for (i = begin; i < end; i++) {
kw = asdl_seq_GET(keywords, i);
VISIT(c, expr, kw->value);
Expand All @@ -4267,14 +4346,20 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be
}
ADDOP_LOAD_CONST_NEW(c, keys);
ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
return 1;
}
else {
/* a for loop only executes once */
for (i = begin; i < end; i++) {
kw = asdl_seq_GET(keywords, i);
ADDOP_LOAD_CONST(c, kw->arg);
VISIT(c, expr, kw->value);
if (big) {
ADDOP_I_NOLINE(c, BUILD_MAP, 0);
}
for (i = begin; i < end; i++) {
kw = asdl_seq_GET(keywords, i);
ADDOP_LOAD_CONST(c, kw->arg);
VISIT(c, expr, kw->value);
if (big) {
ADDOP_I_NOLINE(c, MAP_ADD, 1);
}
}
if (!big) {
ADDOP_I(c, BUILD_MAP, n);
}
return 1;
Expand All @@ -4296,6 +4381,9 @@ compiler_call_helper(struct compiler *c,
nelts = asdl_seq_LEN(args);
nkwelts = asdl_seq_LEN(keywords);

if (nelts + nkwelts >= STACK_USE_GUIDELINE) {
goto ex_call;
}
for (i = 0; i < nelts; i++) {
expr_ty elt = asdl_seq_GET(args, i);
if (elt->kind == Starred_kind) {
Expand Down Expand Up @@ -6603,6 +6691,13 @@ makecode(struct compiler *c, struct assembler *a, PyObject *consts)
Py_DECREF(consts);
goto error;
}
if (maxdepth > MAX_ALLOWED_STACK_USE) {
PyErr_Format(PyExc_SystemError,
"excessive stack use: stack is %d deep",
maxdepth);
Py_DECREF(consts);
goto error;
}
co = PyCode_NewWithPosOnlyArgs(posonlyargcount+posorkeywordargcount,
posonlyargcount, kwonlyargcount, nlocals_int,
maxdepth, flags, a->a_bytecode, consts, names,
Expand Down