From 1b39c8844187c979f33ddf1a663d8cac1f144fbc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 15:33:50 +0100 Subject: [PATCH 1/7] Treat tp_weakref and tp_dictoffset like other opaque slots for multiple inheritance. --- Lib/test/test_capi.py | 44 +++++++++++++++++++++++++++++++----------- Objects/typeobject.c | 45 +++++++++++++------------------------------ 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index e6516b33aec018..b698e34dec4e22 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -677,21 +677,43 @@ def test_heaptype_with_custom_metaclass(self): def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): - class Both1(_testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithDict): + with self.assertRaises(TypeError): + class Both1(_testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithDict): + pass + with self.assertRaises(TypeError): + class Both2(_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): + pass + + def test_multiple_inheritance_ctypes_with_weakref_or_dict_and_other_builtin(self): + + with self.assertRaises(TypeError): + class C1(_testcapi.HeapCTypeWithDict, list): + pass + + with self.assertRaises(TypeError): + class C2(_testcapi.HeapCTypeWithWeakref, list): + pass + + class C3(_testcapi.HeapCTypeWithManagedDict, list): pass - class Both2(_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): + class C4(_testcapi.HeapCTypeWithManagedWeakref, list): pass - for cls in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithDict2, - _testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithWeakref2): - for cls2 in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithDict2, - _testcapi.HeapCTypeWithWeakref, _testcapi.HeapCTypeWithWeakref2): - if cls is not cls2: - class S(cls, cls2): - pass - class B1(Both1, cls): + inst = C3() + inst.append(0) + str(inst.__dict__) + + inst = C4() + inst.append(0) + str(inst.__weakref__) + + for cls in (_testcapi.HeapCTypeWithManagedDict, _testcapi.HeapCTypeWithManagedWeakref): + for cls2 in (_testcapi.HeapCTypeWithDict, _testcapi.HeapCTypeWithWeakref): + class S(cls, cls2): + pass + class B1(C3, cls): pass - class B2(Both1, cls): + class B2(C4, cls): pass def test_pytype_fromspec_with_repeated_slots(self): diff --git a/Objects/typeobject.c b/Objects/typeobject.c index c881aeb63d7565..9381b916fe98f0 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2330,36 +2330,13 @@ best_base(PyObject *bases) return base; } -#define ADDED_FIELD_AT_OFFSET(name, offset) \ - (type->tp_ ## name && (base->tp_ ##name == 0) && \ - type->tp_ ## name + sizeof(PyObject *) == (offset) && \ - type->tp_flags & Py_TPFLAGS_HEAPTYPE) - static int -extra_ivars(PyTypeObject *type, PyTypeObject *base) +shape_differs(PyTypeObject *t1, PyTypeObject *t2) { - size_t t_size = type->tp_basicsize; - size_t b_size = base->tp_basicsize; - - assert(t_size >= b_size); /* Else type smaller than base! */ - if (type->tp_itemsize || base->tp_itemsize) { - /* If itemsize is involved, stricter rules */ - return t_size != b_size || - type->tp_itemsize != base->tp_itemsize; - } - /* Check for __dict__ and __weakrefs__ slots in either order */ - if (ADDED_FIELD_AT_OFFSET(weaklistoffset, t_size)) { - t_size -= sizeof(PyObject *); - } - if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0 && - ADDED_FIELD_AT_OFFSET(dictoffset, t_size)) { - t_size -= sizeof(PyObject *); - } - /* Check __weakrefs__ again, in case it precedes __dict__ */ - if (ADDED_FIELD_AT_OFFSET(weaklistoffset, t_size)) { - t_size -= sizeof(PyObject *); - } - return t_size != b_size; + return ( + t1->tp_basicsize != t2->tp_basicsize || + t1->tp_itemsize != t2->tp_itemsize + ); } static PyTypeObject * @@ -2367,14 +2344,18 @@ solid_base(PyTypeObject *type) { PyTypeObject *base; - if (type->tp_base) + if (type->tp_base) { base = solid_base(type->tp_base); - else + } + else { base = &PyBaseObject_Type; - if (extra_ivars(type, base)) + } + if (shape_differs(type, base)) { return type; - else + } + else { return base; + } } static void object_dealloc(PyObject *); From b994446d02bec790be550ef3c3f41a060eafeaa1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 16:47:58 +0100 Subject: [PATCH 2/7] Document Py_TPFLAGS_MANAGED_DICT and Py_TPFLAGS_MANAGED_WEAKREF in what's new. --- Doc/whatsnew/3.12.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 5926205ce5c07d..90367b17fc6db7 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -450,6 +450,10 @@ New Features inherit the ``Py_TPFLAGS_HAVE_VECTORCALL`` flag. (Contributed by Petr Viktorin in :gh:`93274`.) + The :const:`Py_TPFLAGS_MANAGED_DICT` and `Py_TPFLAGS_MANAGED_WEAKREF` flags + have been added. This allows extensions classes to support object ``__dict__`` + and weakrefs with less bookkeeping, using less memory and with faster access. + Porting to Python 3.12 ---------------------- @@ -486,6 +490,18 @@ Porting to Python 3.12 :c:func:`PyUnicode_FromFormatV`. (Contributed by Philip Georgi in :gh:`95504`.) +* Extension classes wanting to a ``__dict__`` or weak reference slot + should use :const:`Py_TPFLAGS_MANAGED_DICT` and + `Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and + ``tp_weaklistoffset``, respectively. + The use of ``tp_dictoffset`` and ``tp_weaklistoffset`` is still + supported, but will not fully support multiple inheritance + (:gh: `95589`), and performance may be worse. + Classes declaring :const:`Py_TPFLAGS_MANAGED_DICT` should call + :c:func:`_PyObject_VisitManagedDict` and :c:func:`_PyObject_ClearManagedDict` + to traverse and clear their instance's dictionaries. + To clear weakrefs, call :c:func:`PyObject_ClearWeakRefs`, as before. + Deprecated ---------- From bca93afb46fe62cff6df66fd8cede692539186a1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 16:55:19 +0100 Subject: [PATCH 3/7] Add news. --- Doc/whatsnew/3.12.rst | 2 +- .../next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 90367b17fc6db7..402a6099126873 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -492,7 +492,7 @@ Porting to Python 3.12 * Extension classes wanting to a ``__dict__`` or weak reference slot should use :const:`Py_TPFLAGS_MANAGED_DICT` and - `Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and + :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. The use of ``tp_dictoffset`` and ``tp_weaklistoffset`` is still supported, but will not fully support multiple inheritance diff --git a/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst b/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst new file mode 100644 index 00000000000000..9fc756a994261a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst @@ -0,0 +1,4 @@ +Extensions classes that set ``tp_dictoffset`` and ``tp_weaklistoffset`` +loose the support for multiple inheritance, but are now safe. Extension +classes should use :const:`Py_TPFLAGS_MANAGED_DICT` and +:const:`Py_TPFLAGS_MANAGED_WEAKREF` instead. From 2ce1e9f644d6301ffca4c96d4fad23ca9015b32e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 16:59:12 +0100 Subject: [PATCH 4/7] Fix typo --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 402a6099126873..7f329feb8a6166 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -490,7 +490,7 @@ Porting to Python 3.12 :c:func:`PyUnicode_FromFormatV`. (Contributed by Philip Georgi in :gh:`95504`.) -* Extension classes wanting to a ``__dict__`` or weak reference slot +* Extension classes wanting to add a ``__dict__`` or weak reference slot should use :const:`Py_TPFLAGS_MANAGED_DICT` and :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. From 21b50cf0cea85c75dba84c1dba22af5485edc35f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 17:01:40 +0100 Subject: [PATCH 5/7] Fix typo --- .../next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst b/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst index 9fc756a994261a..696e3c80db62cf 100644 --- a/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst +++ b/Misc/NEWS.d/next/C API/2022-08-16-16-54-42.gh-issue-95589.6xE1ar.rst @@ -1,4 +1,4 @@ Extensions classes that set ``tp_dictoffset`` and ``tp_weaklistoffset`` -loose the support for multiple inheritance, but are now safe. Extension +lose the support for multiple inheritance, but are now safe. Extension classes should use :const:`Py_TPFLAGS_MANAGED_DICT` and :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead. From 0be2b9120046af9585e71258011eeb94af31dcdb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Aug 2022 17:31:00 +0100 Subject: [PATCH 6/7] Fix another typo. --- Doc/whatsnew/3.12.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 7f329feb8a6166..c908ad392d8ab0 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -450,9 +450,10 @@ New Features inherit the ``Py_TPFLAGS_HAVE_VECTORCALL`` flag. (Contributed by Petr Viktorin in :gh:`93274`.) - The :const:`Py_TPFLAGS_MANAGED_DICT` and `Py_TPFLAGS_MANAGED_WEAKREF` flags - have been added. This allows extensions classes to support object ``__dict__`` - and weakrefs with less bookkeeping, using less memory and with faster access. + The :const:`Py_TPFLAGS_MANAGED_DICT` and :const:`Py_TPFLAGS_MANAGED_WEAKREF` + flags have been added. This allows extensions classes to support object + ``__dict__`` and weakrefs with less bookkeeping, + using less memory and with faster access. Porting to Python 3.12 ---------------------- From c930792556b23738b53a2590b0eaaeaf88864785 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Aug 2022 11:56:43 +0100 Subject: [PATCH 7/7] Fix tense. --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c908ad392d8ab0..9689d9df9dfc45 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -496,7 +496,7 @@ Porting to Python 3.12 :const:`Py_TPFLAGS_MANAGED_WEAKREF` instead of ``tp_dictoffset`` and ``tp_weaklistoffset``, respectively. The use of ``tp_dictoffset`` and ``tp_weaklistoffset`` is still - supported, but will not fully support multiple inheritance + supported, but does not fully support multiple inheritance (:gh: `95589`), and performance may be worse. Classes declaring :const:`Py_TPFLAGS_MANAGED_DICT` should call :c:func:`_PyObject_VisitManagedDict` and :c:func:`_PyObject_ClearManagedDict`