From 8a29421ba5000f508775a6836dcb6016d13bbc55 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 19 Feb 2025 21:05:01 -0500 Subject: [PATCH 01/13] gh-130327: Only set the managed dictionary if there aren't inline values --- Objects/dictobject.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 900d001d4dd56a..b25897428539f9 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7170,15 +7170,16 @@ set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) PyDictValues *values = _PyObject_InlineValues(obj); - Py_XINCREF(new_dict); - FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); - if (values->valid) { FT_ATOMIC_STORE_UINT8(values->valid, 0); for (Py_ssize_t i = 0; i < values->capacity; i++) { Py_CLEAR(values->values[i]); } } + else { + Py_XINCREF(new_dict); + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); + } } #ifdef Py_GIL_DISABLED From e3cafb732f4a86933dd5e59cc8498bf839cd1276 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 19 Feb 2025 21:06:35 -0500 Subject: [PATCH 02/13] Add blurb. --- .../2025-02-19-21-06-30.gh-issue-130327.z3TaR8.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-02-19-21-06-30.gh-issue-130327.z3TaR8.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-19-21-06-30.gh-issue-130327.z3TaR8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-19-21-06-30.gh-issue-130327.z3TaR8.rst new file mode 100644 index 00000000000000..9b9a282b5ab414 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-19-21-06-30.gh-issue-130327.z3TaR8.rst @@ -0,0 +1,2 @@ +Fix erroneous clearing of an object's :attr:`~object.__dict__` if +overwritten at runtime. From 671d3f70fdfb3f484fb8368a0397e455a431cc9a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 19 Feb 2025 21:15:43 -0500 Subject: [PATCH 03/13] Add a test case. --- Lib/test/test_dict.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 86b2f22dee5347..cef3d00d96563e 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1486,6 +1486,26 @@ def make_pairs(): self.assertEqual(d.get(key3_3), 44) self.assertGreaterEqual(eq_count, 1) + def test_overwrite_managed_dict(self): + # GH-130327: Overwriting an object's managed dict incorrectly stored + # the new dictionary in the managed dict pointer for objects supporting + # inline values, leading to early clearing of the dictionary + import gc + + class Shenanigans: + pass + + to_be_deleted = Shenanigans() + to_be_deleted.attr = "whatever" + holds_reference = Shenanigans() + holds_reference.__dict__ = to_be_deleted.__dict__ + holds_reference.ref = {"circular": to_be_deleted, "data": 42} + + del to_be_deleted + gc.collect() + self.assertEqual(holds_reference.ref['data'], 42) + self.assertEqual(holds_reference.attr, "whatever") + class CAPITest(unittest.TestCase): From 2fa09a9ad927f4ac540d0d9e66cdcf6cb5a5af0f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 22 Feb 2025 13:47:13 -0500 Subject: [PATCH 04/13] Revert "gh-130327: Only set the managed dictionary if there aren't inline values" This reverts commit 8a29421ba5000f508775a6836dcb6016d13bbc55. --- Objects/dictobject.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b25897428539f9..900d001d4dd56a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7170,16 +7170,15 @@ set_dict_inline_values(PyObject *obj, PyDictObject *new_dict) PyDictValues *values = _PyObject_InlineValues(obj); + Py_XINCREF(new_dict); + FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); + if (values->valid) { FT_ATOMIC_STORE_UINT8(values->valid, 0); for (Py_ssize_t i = 0; i < values->capacity; i++) { Py_CLEAR(values->values[i]); } } - else { - Py_XINCREF(new_dict); - FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict); - } } #ifdef Py_GIL_DISABLED From 07c493572720f208f53b7e2af9bd14c2658efd78 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 22 Feb 2025 13:55:11 -0500 Subject: [PATCH 05/13] Treat the dictionary as owning values, not the object. --- Lib/test/test_dict.py | 6 +++--- Objects/dictobject.c | 14 +++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index cef3d00d96563e..dac53b11aa9915 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1487,9 +1487,9 @@ def make_pairs(): self.assertGreaterEqual(eq_count, 1) def test_overwrite_managed_dict(self): - # GH-130327: Overwriting an object's managed dict incorrectly stored - # the new dictionary in the managed dict pointer for objects supporting - # inline values, leading to early clearing of the dictionary + # GH-130327: Overwriting an object's managed dictionary with another object's + # skipped traversal in favor of inline values, causing the GC to believe that + # the __dict__ wasn't reachable. import gc class Shenanigans: diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 900d001d4dd56a..227f22c4d190a8 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4577,10 +4577,8 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->embedded) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); - } + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); } } else { @@ -7150,6 +7148,13 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; + if (dict != NULL) { + // GH-130327: If there's a managed dictionary available, we should + // *always* traverse it, including when inline values are available. + Py_VISIT(dict); + return 0; + } if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictValues *values = _PyObject_InlineValues(obj); if (values->valid) { @@ -7159,7 +7164,6 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) return 0; } } - Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); return 0; } From 39e4a812cd85810148c785fdc8a27435b912d740 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 23 Feb 2025 08:06:32 -0500 Subject: [PATCH 06/13] Check for validity of the values. --- Objects/dictobject.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 227f22c4d190a8..ed728f1c5e0237 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4577,8 +4577,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (!mp->ma_values->valid) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { From 33808bc890599d6f0160b83950baf8b3f6efaa0b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 23 Feb 2025 08:15:00 -0500 Subject: [PATCH 07/13] Also check if they're embedded. --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ed728f1c5e0237..135893f85b594a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4577,7 +4577,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->valid) { + if (!mp->ma_values->valid || !mp->ma_values->embedded) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } From 67bc92fd707bf3c9bceba28dfb8e6c25905ddbc0 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 23 Feb 2025 08:19:39 -0500 Subject: [PATCH 08/13] Swap the case to (hopefully) make CIFuzz happy. --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 135893f85b594a..65c3f3070c0982 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4577,7 +4577,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->valid || !mp->ma_values->embedded) { + if (!mp->ma_values->embedded || !mp->ma_values->valid) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } From 5025d610a29786ca6da46a85c439c73dbac68f13 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 23 Feb 2025 08:41:15 -0500 Subject: [PATCH 09/13] Fix negation of case. --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 65c3f3070c0982..876fe2c85b7c4c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4577,7 +4577,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->embedded || !mp->ma_values->valid) { + if (mp->ma_values->valid) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } From ff426797300eadf9fa606825327380acfc0b2ae8 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 25 Mar 2025 13:20:21 +0000 Subject: [PATCH 10/13] Use an atomic load. --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 876fe2c85b7c4c..f0d9ab60456dcb 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4577,7 +4577,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (mp->ma_values->valid) { + if (!FT_ATOMIC_LOAD_UINT8(mp->ma_values->valid)) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } From 719fbd195491af865ce2e1ea460790cc24b42e2a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 19 Jun 2025 12:46:42 -0400 Subject: [PATCH 11/13] Not sure what I was doing earlier but I think this works. --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d7eb209ce1cba4..e60043320de555 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4579,7 +4579,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!FT_ATOMIC_LOAD_UINT8(mp->ma_values->valid)) { + if (!FT_ATOMIC_LOAD_UINT8(mp->ma_values->embedded)) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } From c1d0cd2155acf16f05d0aa04238a4e0f447bf7d5 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 19 Jun 2025 12:55:18 -0400 Subject: [PATCH 12/13] Remove atomic load. I turned this into a mess when trying to debug the other crash. --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a437ffda774122..0f0d9097b34041 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4619,7 +4619,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!FT_ATOMIC_LOAD_UINT8(mp->ma_values->embedded)) { + if (!mp->ma_values->embedded) { for (i = 0; i < n; i++) { Py_VISIT(mp->ma_values->values[i]); } From 4ecf9676cd2e718d711228c89e4f5f43d633d947 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 19 Jun 2025 13:28:52 -0400 Subject: [PATCH 13/13] Back to square one. --- Objects/dictobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 0f0d9097b34041..72fe4c5db9ea61 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4619,10 +4619,8 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->embedded) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); - } + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); } } else {