From 7c3e5881e8177dff321a790022dff4e1c7064e7d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 13:06:41 -0600 Subject: [PATCH 1/8] Make tp_bases immortal for static builtin types. --- Objects/typeobject.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2fbcafe91aadc6..45c13ab0a32d92 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -268,12 +268,6 @@ clear_tp_dict(PyTypeObject *self) static inline PyObject * lookup_tp_bases(PyTypeObject *self) { - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - return state->tp_bases; - } return self->tp_bases; } @@ -288,11 +282,18 @@ static inline void set_tp_bases(PyTypeObject *self, PyObject *bases) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - state->tp_bases = bases; - return; + assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(self->tp_bases == NULL); + if (PyTuple_GET_SIZE(bases) == 0) { + assert(self->tp_base == NULL); + } + else { + assert(PyTuple_GET_SIZE(bases) == 1); + assert(PyTuple_GET_ITEM(bases, 0) == (PyObject *)self->tp_base); + assert(self->tp_base->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(_Py_IsImmortal(self->tp_base)); + } + _Py_SetImmortal(bases); } self->tp_bases = bases; } @@ -301,10 +302,12 @@ static inline void clear_tp_bases(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - Py_CLEAR(state->tp_bases); + if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (self->tp_bases != NULL) { + assert(_Py_IsImmortal(self->tp_bases)); + // XXX Delete the tuple? + } + } return; } Py_CLEAR(self->tp_bases); From ff411b235033d927f98a8596dab990b4d875fd29 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 13:14:59 -0600 Subject: [PATCH 2/8] Add _Py_ClearImmortal(). --- Include/internal/pycore_object.h | 15 +++++++++++++++ Objects/typeobject.c | 6 ++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 500b3eece68055..0981d1122fec54 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -76,6 +76,21 @@ static inline void _Py_SetImmortal(PyObject *op) } #define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) +/* _Py_ClearImmortal() should only be used during runtime finalization. */ +static inline void _Py_ClearImmortal(PyObject *op) +{ + if (op) { + assert(op->ob_refcnt == _Py_IMMORTAL_REFCNT); + op->ob_refcnt = 1; + Py_DECREF(op); + } +} +#define _Py_ClearImmortal(op) \ + do { \ + _Py_ClearImmortal(_PyObject_CAST(op)); \ + op = NULL; \ + } while (0) + static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 45c13ab0a32d92..b4ee0c81d98b9d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -303,9 +303,11 @@ clear_tp_bases(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { - if (self->tp_bases != NULL) { + if (self->tp_bases != NULL + && PyTuple_GET_SIZE(self->tp_bases) > 0) + { assert(_Py_IsImmortal(self->tp_bases)); - // XXX Delete the tuple? + _Py_ClearImmortal(self->tp_bases); } } return; From 354ff5f6d9de6f01a5d79ec0e4db3f323a162059 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 13:20:47 -0600 Subject: [PATCH 3/8] Drop PyInterpreterState.types.builtins[i].tp_bases. --- Include/internal/pycore_typeobject.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c7d00002fbc63c..eee18fd0aa41f5 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -49,7 +49,6 @@ typedef struct { // XXX tp_dict, tp_bases, and tp_mro can probably be statically // allocated, instead of dynamically and stored on the interpreter. PyObject *tp_dict; - PyObject *tp_bases; PyObject *tp_mro; PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since From 5be53f472e9550bd5876e216aae20474b1b8b185 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 13:27:53 -0600 Subject: [PATCH 4/8] Make sure tp_bases is a tuple. --- Objects/typeobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b4ee0c81d98b9d..77582c2901f8fc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -281,6 +281,7 @@ _PyType_GetBases(PyTypeObject *self) static inline void set_tp_bases(PyTypeObject *self, PyObject *bases) { + assert(PyTuple_CheckExact(bases)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); assert(self->tp_bases == NULL); From bf1578e6c2a3105690e65fc07912b3143432fb07 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 13:35:12 -0600 Subject: [PATCH 5/8] Make tp_mro immortal for static builtin types. --- Include/internal/pycore_typeobject.h | 1 - Objects/typeobject.c | 36 ++++++++++++++++------------ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index eee18fd0aa41f5..51963c865f7e45 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -49,7 +49,6 @@ typedef struct { // XXX tp_dict, tp_bases, and tp_mro can probably be statically // allocated, instead of dynamically and stored on the interpreter. PyObject *tp_dict; - PyObject *tp_mro; PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since they will effectively never get triggered. However, there diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 77582c2901f8fc..ae2eec7f97020e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -320,12 +320,6 @@ clear_tp_bases(PyTypeObject *self) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - return state->tp_mro; - } return self->tp_mro; } @@ -339,12 +333,12 @@ _PyType_GetMRO(PyTypeObject *self) static inline void set_tp_mro(PyTypeObject *self, PyObject *mro) { + assert(PyTuple_CheckExact(mro)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - state->tp_mro = mro; - return; + assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); + assert(self->tp_mro == NULL); + /* Other checks are done via set_tp_bases. */ + _Py_SetImmortal(mro); } self->tp_mro = mro; } @@ -353,10 +347,14 @@ static inline void clear_tp_mro(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - Py_CLEAR(state->tp_mro); + if (_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + if (self->tp_mro != NULL + && PyTuple_GET_SIZE(self->tp_mro) > 0) + { + assert(_Py_IsImmortal(self->tp_mro)); + _Py_ClearImmortal(self->tp_mro); + } + } return; } Py_CLEAR(self->tp_mro); @@ -7159,6 +7157,14 @@ type_ready_preheader(PyTypeObject *type) static int type_ready_mro(PyTypeObject *type) { + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + if (!_Py_IsMainInterpreter(_PyInterpreterState_GET())) { + assert(lookup_tp_mro(type) != NULL); + return 0; + } + assert(lookup_tp_mro(type) == NULL); + } + /* Calculate method resolution order */ if (mro_internal(type, NULL) < 0) { return -1; From f4a5da1c3725c449a6223922a92e8d17bc8a8425 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 15:40:34 -0600 Subject: [PATCH 6/8] Update a TODO comment. --- Include/internal/pycore_typeobject.h | 4 ++-- Objects/typeobject.c | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 51963c865f7e45..8f3fbbcdb5ffcd 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -46,8 +46,8 @@ typedef struct { PyTypeObject *type; int readying; int ready; - // XXX tp_dict, tp_bases, and tp_mro can probably be statically - // allocated, instead of dynamically and stored on the interpreter. + // XXX tp_dict can probably be statically allocated, + // instead of dynamically and stored on the interpreter. PyObject *tp_dict; PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ae2eec7f97020e..671162411a99be 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -283,6 +283,8 @@ set_tp_bases(PyTypeObject *self, PyObject *bases) { assert(PyTuple_CheckExact(bases)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + // XXX tp_bases can probably be statically allocated for each + // static builtin type. assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); assert(self->tp_bases == NULL); if (PyTuple_GET_SIZE(bases) == 0) { @@ -335,6 +337,8 @@ set_tp_mro(PyTypeObject *self, PyObject *mro) { assert(PyTuple_CheckExact(mro)); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + // XXX tp_mro can probably be statically allocated for each + // static builtin type. assert(_Py_IsMainInterpreter(_PyInterpreterState_GET())); assert(self->tp_mro == NULL); /* Other checks are done via set_tp_bases. */ From e054bbe9b0472047a6c02403d51fe01c97169ca4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 16:00:35 -0600 Subject: [PATCH 7/8] Add a test to avoid more regressions. --- Lib/test/test_capi/test_misc.py | 33 +++++++++++++++++++++++++++++++++ Modules/_testcapimodule.c | 23 +++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index dc3441e4496a5d..037c8112d53e7a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1593,6 +1593,39 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +class BuiltinStaticTypesTests(unittest.TestCase): + + TYPES = [ + object, + type, + int, + str, + dict, + type(None), + bool, + BaseException, + Exception, + Warning, + DeprecationWarning, # Warning subclass + ] + + def test_tp_bases_is_set(self): + # PyTypeObject.tp_bases is documented as public API. + # See https://github.com/python/cpython/issues/105020. + for typeobj in self.TYPES: + with self.subTest(typeobj): + bases = _testcapi.type_get_tp_bases(typeobj) + self.assertIsNot(bases, None) + + def test_tp_mro_is_set(self): + # PyTypeObject.tp_bases is documented as public API. + # See https://github.com/python/cpython/issues/105020. + for typeobj in self.TYPES: + with self.subTest(typeobj): + mro = _testcapi.type_get_tp_mro(typeobj) + self.assertIsNot(mro, None) + + class TestThreadState(unittest.TestCase): @threading_helper.reap_threads diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index aff09aac80c610..66c1cbabe0f8c4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2606,6 +2606,27 @@ type_assign_version(PyObject *self, PyObject *type) } +static PyObject * +type_get_tp_bases(PyObject *self, PyObject *type) +{ + PyObject *bases = ((PyTypeObject *)type)->tp_bases; + if (bases == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(bases); +} + +static PyObject * +type_get_tp_mro(PyObject *self, PyObject *type) +{ + PyObject *mro = ((PyTypeObject *)type)->tp_mro; + if (mro == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(mro); +} + + // Test PyThreadState C API static PyObject * test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) @@ -3361,6 +3382,8 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, + {"type_get_tp_bases", type_get_tp_bases, METH_O}, + {"type_get_tp_mro", type_get_tp_mro, METH_O}, {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"frame_getlocals", frame_getlocals, METH_O, NULL}, {"frame_getglobals", frame_getglobals, METH_O, NULL}, From 78c7d0ec4fcb732efb7cf7dc5aac08d7c6a8be7e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 May 2023 17:45:39 -0600 Subject: [PATCH 8/8] Add a NEWS entry. --- .../next/C API/2023-05-30-17-45-32.gh-issue-105115.iRho1K.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/C API/2023-05-30-17-45-32.gh-issue-105115.iRho1K.rst diff --git a/Misc/NEWS.d/next/C API/2023-05-30-17-45-32.gh-issue-105115.iRho1K.rst b/Misc/NEWS.d/next/C API/2023-05-30-17-45-32.gh-issue-105115.iRho1K.rst new file mode 100644 index 00000000000000..595cc0e2013d96 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-05-30-17-45-32.gh-issue-105115.iRho1K.rst @@ -0,0 +1,3 @@ +``PyTypeObject.tp_bases`` (and ``tp_mro``) for builtin static types are now +shared by all interpreters, whereas in 3.12-beta1 they were stored on +``PyInterpreterState``. Also note that now the tuples are immortal objects.