From 71ae2125197bcc791c7d6e7b45c097dd4bd81618 Mon Sep 17 00:00:00 2001 From: Bast0006 Date: Sat, 29 Mar 2025 07:22:50 -0700 Subject: [PATCH 1/7] GH-91153: Handle _getbytevalue potentially invalidating bytearray allocation during item assignment --- Objects/bytearrayobject.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 8f2d2dd02151c1..cd0c66ce036392 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,7 +709,8 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // GH-91153: we cannot store a reference to the internal buffer here, as _getbytevalue might call into python code + // that could then invalidate it. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +745,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +806,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char* buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +847,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char* buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, From b2f7df50850f78e3f6bdbe6dbdb7f93f71661acf Mon Sep 17 00:00:00 2001 From: Bast Date: Thu, 10 Apr 2025 13:14:34 -0700 Subject: [PATCH 2/7] GH-91153: Add additional cases to `test_mutating_index` --- Lib/test/test_bytes.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 44486760c08349..8d2543df2883d4 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1889,6 +1889,41 @@ def __index__(self): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + class BoomContinued: + def __index__(self): + nonlocal new_ba + # Clear the original bytearray, mutating it during index assignment. + # If the internal buffers are held over this operation, they become dangling + # However, this will fail a bounds check as above (as the clear sets bounds to zero) + ba.clear() + # At this moment, the bytearray potentially has a dangling pointer + # Create a new bytearray to catch any writes + new_ba = bytearray(0x180) + # Ensure bounds check passes + ba.extend([0] * 0x180) + return 0 + + with self.subTest("skip_bounds_safety"): + new_ba: bytearray + ba = bytearray(0x180) + ba[BoomContinued()] = ord("?") + self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_capi"): + new_ba: bytearray + ba = bytearray(0x180) + self._testlimitedcapi.sequence_setitem(ba, BoomContinued(), ord("?")) + self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_slice"): + new_ba: bytearray + ba = bytearray(0x180) + ba[BoomContinued():1] = [ord("?")] + self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + class AssortedBytesTest(unittest.TestCase): # From bb96c2a57ea7132c170b306c84bc9bbbb0307610 Mon Sep 17 00:00:00 2001 From: Bast Date: Sat, 17 May 2025 17:50:06 -0500 Subject: [PATCH 3/7] GH-91153: Move mutating index bytearray test to it's own separate test --- Lib/test/test_bytes.py | 43 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 8d2543df2883d4..cfa21bf8783b2d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1889,40 +1889,43 @@ def __index__(self): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) - class BoomContinued: + def test_mutating_index_inbounds(self): + class MutatesOnIndex: + new_ba: bytearray + + def __init__(self): + self.ba = bytearray(0x180) + def __index__(self): - nonlocal new_ba # Clear the original bytearray, mutating it during index assignment. # If the internal buffers are held over this operation, they become dangling # However, this will fail a bounds check as above (as the clear sets bounds to zero) - ba.clear() + self.ba.clear() # At this moment, the bytearray potentially has a dangling pointer # Create a new bytearray to catch any writes - new_ba = bytearray(0x180) + self.new_ba = bytearray(0x180) # Ensure bounds check passes - ba.extend([0] * 0x180) + self.ba.extend([0] * 0x180) return 0 with self.subTest("skip_bounds_safety"): - new_ba: bytearray - ba = bytearray(0x180) - ba[BoomContinued()] = ord("?") - self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") - self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") with self.subTest("skip_bounds_safety_capi"): - new_ba: bytearray - ba = bytearray(0x180) - self._testlimitedcapi.sequence_setitem(ba, BoomContinued(), ord("?")) - self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") - self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self._testlimitedcapi.sequence_setitem(instance.ba, instance, ord("?")) + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") with self.subTest("skip_bounds_safety_slice"): - new_ba: bytearray - ba = bytearray(0x180) - ba[BoomContinued():1] = [ord("?")] - self.assertEqual(ba[0], ord("?"), "Assigned bytearray not altered") - self.assertEqual(new_ba, bytearray(0x180), "Wrong object altered") + instance = MutatesOnIndex() + instance.ba[instance:1] = [ord("?")] + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") class AssortedBytesTest(unittest.TestCase): From d29422b98873355cbf601e772a95f3ff516c062c Mon Sep 17 00:00:00 2001 From: Bast Date: Sat, 17 May 2025 18:22:52 -0500 Subject: [PATCH 4/7] GH-91153: add news entry --- .../2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst new file mode 100644 index 00000000000000..b337935fd3bb75 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst @@ -0,0 +1,3 @@ +Fix an issue where a :class:`bytearray` item assignment could crash or write +to the wrong bytearray when resized by the new value's :meth:`__index__` +method. From d3d19747e5257f4d8406270361951ec57a89fa48 Mon Sep 17 00:00:00 2001 From: Bast Date: Sat, 17 May 2025 20:56:56 -0500 Subject: [PATCH 5/7] Nit cleanup --- Lib/test/test_bytes.py | 2 ++ .../2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst | 3 --- .../2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index cfa21bf8783b2d..c2843e726dfdcc 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1890,6 +1890,8 @@ def __index__(self): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) def test_mutating_index_inbounds(self): + # See gh-91153 + class MutatesOnIndex: new_ba: bytearray diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst deleted file mode 100644 index b337935fd3bb75..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-18-22-12.gh-issue-91153.ioA_83.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where a :class:`bytearray` item assignment could crash or write -to the wrong bytearray when resized by the new value's :meth:`__index__` -method. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst new file mode 100644 index 00000000000000..d90f7231355500 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -0,0 +1,3 @@ +Fix an issue where a :class:`bytearray` item assignment could crash or write +to the wrong bytearray when resized by the new value's +:meth:`~object.__index__` method. From 18e6597b9eac8d8f0e9ef1bbd617a68e0bd777b3 Mon Sep 17 00:00:00 2001 From: Bast <52266665+bast0006@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:44:52 -0700 Subject: [PATCH 6/7] GH-91153: Fixup comments and NEWS entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_bytes.py | 11 ++--------- .../2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst | 4 +--- Objects/bytearrayobject.c | 5 +++-- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index c2843e726dfdcc..f356cea2b1e659 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1893,21 +1893,14 @@ def test_mutating_index_inbounds(self): # See gh-91153 class MutatesOnIndex: - new_ba: bytearray def __init__(self): self.ba = bytearray(0x180) def __index__(self): - # Clear the original bytearray, mutating it during index assignment. - # If the internal buffers are held over this operation, they become dangling - # However, this will fail a bounds check as above (as the clear sets bounds to zero) self.ba.clear() - # At this moment, the bytearray potentially has a dangling pointer - # Create a new bytearray to catch any writes - self.new_ba = bytearray(0x180) - # Ensure bounds check passes - self.ba.extend([0] * 0x180) + self.new_ba = bytearray(0x180) # to catch out-of-bounds writes + self.ba.extend([0] * 0x180) # to check bounds checks return 0 with self.subTest("skip_bounds_safety"): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst index d90f7231355500..dc2f1e22ba5b05 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -1,3 +1 @@ -Fix an issue where a :class:`bytearray` item assignment could crash or write -to the wrong bytearray when resized by the new value's -:meth:`~object.__index__` method. +Fix a crash when a :class:`bytearray` is concurrently mutated during item assignment. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index cd0c66ce036392..38ffcb2ccb79a5 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,8 +709,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - // GH-91153: we cannot store a reference to the internal buffer here, as _getbytevalue might call into python code - // that could then invalidate it. + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); From 4b7ec9cb909770251e28ecf851dbad5a6b2fa351 Mon Sep 17 00:00:00 2001 From: Bast <52266665+bast0006@users.noreply.github.com> Date: Tue, 3 Jun 2025 13:50:26 -0700 Subject: [PATCH 7/7] GH-91153: Add short clarifying comments to test --- Lib/test/test_bytes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f356cea2b1e659..fcc88682a3cbce 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1872,6 +1872,8 @@ def test_repeat_after_setslice(self): self.assertEqual(b3, b'xcxcxc') def test_mutating_index(self): + # bytearray slice assignment can call into python code + # that reallocates the internal buffer # See gh-91153 class Boom: @@ -1890,10 +1892,10 @@ def __index__(self): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) def test_mutating_index_inbounds(self): - # See gh-91153 + # gh-91153 continued + # Ensure buffer is not broken even if length is correct class MutatesOnIndex: - def __init__(self): self.ba = bytearray(0x180)