From 53e6f412f005f8e2033b47a9c2ef212a99a881c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 17:17:00 +0200 Subject: [PATCH 1/3] gh-108308: Replace PyDict_GetItem() with PyDict_GetItemRef() Replace PyDict_GetItem() calls with PyDict_GetItemRef() to handle errors. pycore_init_builtins() now checks for _PyType_Lookup() failure. --- Python/assemble.c | 20 ++++++++++++++++---- Python/compile.c | 24 ++++++++++++++++++++---- Python/flowgraph.c | 6 +++++- Python/pylifecycle.c | 26 ++++++++++++++++++++------ 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/Python/assemble.c b/Python/assemble.c index b7012534d6cc4e..1d27edad0818a6 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -466,7 +466,7 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); -static void +static int compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, PyObject *names, PyObject *kinds) { @@ -481,7 +481,11 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, if (PyDict_Contains(umd->u_fasthidden, k)) { kind |= CO_FAST_HIDDEN; } - if (PyDict_GetItem(umd->u_cellvars, k) != NULL) { + int has_cell = PyDict_Contains(umd->u_cellvars, k); + if (has_cell < 0) { + return -1; + } + if (has_cell) { kind |= CO_FAST_CELL; } _Py_set_localsplus_info(offset, k, kind, names, kinds); @@ -492,7 +496,11 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, int numdropped = 0; pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &k, &v)) { - if (PyDict_GetItem(umd->u_varnames, k) != NULL) { + int has_name = PyDict_Contains(umd->u_varnames, k); + if (has_name < 0) { + return -1; + } + if (has_name) { // Skip cells that are already covered by locals. numdropped += 1; continue; @@ -512,6 +520,7 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, assert(offset < nlocalsplus); _Py_set_localsplus_info(offset, k, CO_FAST_FREE, names, kinds); } + return 0; } static PyCodeObject * @@ -556,7 +565,10 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ if (localspluskinds == NULL) { goto error; } - compute_localsplus_info(umd, nlocalsplus, localsplusnames, localspluskinds); + if (compute_localsplus_info(umd, nlocalsplus, + localsplusnames, localspluskinds) < 0) { + goto error; + } struct _PyCodeConstructor con = { .filename = filename, diff --git a/Python/compile.c b/Python/compile.c index 3260dba57eac8f..032ce21ddebaba 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4194,9 +4194,20 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste) || - (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True)) + if (_PyST_IsFunctionLike(c->u->u_ste)) { optype = OP_FAST; + } + else { + PyObject *item; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, + &item) < 0) { + return ERROR; + } + if (item == Py_True) { + optype = OP_FAST; + } + Py_XDECREF(item); + } break; case GLOBAL_IMPLICIT: if (_PyST_IsFunctionLike(c->u->u_ste)) @@ -5518,8 +5529,13 @@ push_inlined_comprehension_state(struct compiler *c, location loc, if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { if (!_PyST_IsFunctionLike(c->u->u_ste)) { // non-function scope: override this name to use fast locals - PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); - if (orig != Py_True) { + PyObject *orig; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { + return ERROR; + } + int orig_is_true = (orig == Py_True); + Py_XDECREF(orig); + if (!orig_is_true) { if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { return ERROR; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 719ed92105074e..7811f1065dacce 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2404,12 +2404,16 @@ build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) PyObject *varname, *cellindex; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); + PyObject *varindex; + if (PyDict_GetItemRef(umd->u_varnames, varname, &varindex) < 0) { + return NULL; + } if (varindex != NULL) { assert(PyLong_AS_LONG(cellindex) < INT_MAX); assert(PyLong_AS_LONG(varindex) < INT_MAX); int oldindex = (int)PyLong_AS_LONG(cellindex); int argoffset = (int)PyLong_AS_LONG(varindex); + Py_DECREF(varindex); fixed[oldindex] = argoffset; } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9837e0f0d52e6d..440b09612b41d2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -762,18 +762,32 @@ pycore_init_builtins(PyThreadState *tstate) } interp->builtins = Py_NewRef(builtins_dict); - PyObject *isinstance = PyDict_GetItem(builtins_dict, &_Py_ID(isinstance)); - assert(isinstance); + PyObject *isinstance; + if (PyDict_GetItemRef(builtins_dict, &_Py_ID(isinstance), &isinstance) != 1) { + goto error; + } interp->callable_cache.isinstance = isinstance; - PyObject *len = PyDict_GetItem(builtins_dict, &_Py_ID(len)); - assert(len); + Py_DECREF(isinstance); + + PyObject *len; + if (PyDict_GetItemRef(builtins_dict, &_Py_ID(len), &len) != 1) { + goto error; + } interp->callable_cache.len = len; + Py_DECREF(len); + PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); - assert(list_append); + if (list_append == NULL) { + goto error; + } interp->callable_cache.list_append = list_append; + PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); - assert(object__getattribute__); + if (object__getattribute__ == NULL) { + goto error; + } interp->callable_cache.object__getattribute__ = object__getattribute__; + if (_PyBuiltins_AddExceptions(bimod) < 0) { return _PyStatus_ERR("failed to add exceptions to builtins"); } From 2dd33ae34436771635943e6ec3aa9eb6499a5020 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 20:44:32 +0200 Subject: [PATCH 2/3] Address Serhiy's review --- Python/assemble.c | 49 ++++++++++++++++++++++++++++++---------------- Python/compile.c | 12 ++++++++---- Python/flowgraph.c | 28 +++++++++++++++++--------- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Python/assemble.c b/Python/assemble.c index 1d27edad0818a6..4f66cf294e38c9 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -18,7 +18,7 @@ #define ERROR -1 #define RETURN_IF_ERROR(X) \ - if ((X) == -1) { \ + if ((X) < 0) { \ return ERROR; \ } @@ -448,13 +448,17 @@ static PyObject * dict_keys_inorder(PyObject *dict, Py_ssize_t offset) { PyObject *tuple, *k, *v; - Py_ssize_t i, pos = 0, size = PyDict_GET_SIZE(dict); + Py_ssize_t pos = 0, size = PyDict_GET_SIZE(dict); tuple = PyTuple_New(size); if (tuple == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyLong_AS_LONG(v); + Py_ssize_t i = PyLong_AsSsize_t(v); + if (i == -1 && PyErr_Occurred()) { + Py_DECREF(tuple); + return NULL; + } assert((i - offset) < size); assert((i - offset) >= 0); PyTuple_SET_ITEM(tuple, i - offset, Py_NewRef(k)); @@ -473,21 +477,27 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_varnames, &pos, &k, &v)) { - int offset = (int)PyLong_AS_LONG(v); + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); assert(offset < nlocalsplus); + // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_Contains(umd->u_fasthidden, k)) { + int has_key = PyDict_Contains(umd->u_fasthidden, k); + RETURN_IF_ERROR(has_key); + if (has_key) { kind |= CO_FAST_HIDDEN; } - int has_cell = PyDict_Contains(umd->u_cellvars, k); - if (has_cell < 0) { - return -1; - } - if (has_cell) { + + has_key = PyDict_Contains(umd->u_cellvars, k); + RETURN_IF_ERROR(has_key); + if (has_key) { kind |= CO_FAST_CELL; } + _Py_set_localsplus_info(offset, k, kind, names, kinds); } int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); @@ -497,15 +507,17 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &k, &v)) { int has_name = PyDict_Contains(umd->u_varnames, k); - if (has_name < 0) { - return -1; - } + RETURN_IF_ERROR(has_name); if (has_name) { // Skip cells that are already covered by locals. numdropped += 1; continue; } - int offset = (int)PyLong_AS_LONG(v); + + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); offset += nlocals - numdropped; assert(offset < nlocalsplus); @@ -514,13 +526,16 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, pos = 0; while (PyDict_Next(umd->u_freevars, &pos, &k, &v)) { - int offset = (int)PyLong_AS_LONG(v); + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); offset += nlocals - numdropped; assert(offset < nlocalsplus); _Py_set_localsplus_info(offset, k, CO_FAST_FREE, names, kinds); } - return 0; + return SUCCESS; } static PyCodeObject * @@ -566,7 +581,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ goto error; } if (compute_localsplus_info(umd, nlocalsplus, - localsplusnames, localspluskinds) < 0) { + localsplusnames, localspluskinds) == ERROR) { goto error; } diff --git a/Python/compile.c b/Python/compile.c index 032ce21ddebaba..7418f509c87b14 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4201,7 +4201,7 @@ compiler_nameop(struct compiler *c, location loc, PyObject *item; if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, &item) < 0) { - return ERROR; + goto error; } if (item == Py_True) { optype = OP_FAST; @@ -4232,7 +4232,7 @@ compiler_nameop(struct compiler *c, location loc, op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { - return ERROR; + goto error; } } else if (c->u->u_ste->ste_can_see_class_scope) { @@ -4240,7 +4240,7 @@ compiler_nameop(struct compiler *c, location loc, // First load the classdict if (compiler_addop_o(c->u, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - return ERROR; + goto error; } } else { @@ -4267,7 +4267,7 @@ compiler_nameop(struct compiler *c, location loc, // First load the classdict if (compiler_addop_o(c->u, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - return ERROR; + goto error; } } else { op = LOAD_GLOBAL; @@ -4301,6 +4301,10 @@ compiler_nameop(struct compiler *c, location loc, arg <<= 1; } return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); + +error: + Py_DECREF(mangled); + return ERROR; } static int diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 7811f1065dacce..e620e7cf1b9e96 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2406,19 +2406,29 @@ build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { PyObject *varindex; if (PyDict_GetItemRef(umd->u_varnames, varname, &varindex) < 0) { - return NULL; + goto error; } - if (varindex != NULL) { - assert(PyLong_AS_LONG(cellindex) < INT_MAX); - assert(PyLong_AS_LONG(varindex) < INT_MAX); - int oldindex = (int)PyLong_AS_LONG(cellindex); - int argoffset = (int)PyLong_AS_LONG(varindex); - Py_DECREF(varindex); - fixed[oldindex] = argoffset; + if (varindex == NULL) { + continue; } - } + int argoffset = _PyLong_AsInt(varindex); + Py_DECREF(varindex); + if (argoffset == -1 && PyErr_Occurred()) { + goto error; + } + + int oldindex = _PyLong_AsInt(cellindex); + if (oldindex == -1 && PyErr_Occurred()) { + goto error; + } + fixed[oldindex] = argoffset; + } return fixed; + +error: + PyMem_Free(fixed); + return NULL; } #define IS_GENERATOR(CF) \ From 5c770565418818446dc1eaf1e9ac0ed8bc183083 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Aug 2023 20:46:35 +0200 Subject: [PATCH 3/3] Use PyDict_GetItemWithError() in pylifecycle.c --- Python/pylifecycle.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 440b09612b41d2..8c321fbcfe4104 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -762,19 +762,17 @@ pycore_init_builtins(PyThreadState *tstate) } interp->builtins = Py_NewRef(builtins_dict); - PyObject *isinstance; - if (PyDict_GetItemRef(builtins_dict, &_Py_ID(isinstance), &isinstance) != 1) { + PyObject *isinstance = PyDict_GetItemWithError(builtins_dict, &_Py_ID(isinstance)); + if (!isinstance) { goto error; } interp->callable_cache.isinstance = isinstance; - Py_DECREF(isinstance); - PyObject *len; - if (PyDict_GetItemRef(builtins_dict, &_Py_ID(len), &len) != 1) { + PyObject *len = PyDict_GetItemWithError(builtins_dict, &_Py_ID(len)); + if (!len) { goto error; } interp->callable_cache.len = len; - Py_DECREF(len); PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); if (list_append == NULL) {