From c7872cdb3691d2e69ddadfad208193c7e574b963 Mon Sep 17 00:00:00 2001 From: edward_xu Date: Fri, 29 Nov 2024 20:56:11 +0800 Subject: [PATCH 1/5] gh-127085: fix `memoryview->exports` race condition. --- .../test_free_threading/test_memoryview.py | 36 +++++++++++++++++++ Objects/memoryobject.c | 4 +++ 2 files changed, 40 insertions(+) create mode 100644 Lib/test/test_free_threading/test_memoryview.py diff --git a/Lib/test/test_free_threading/test_memoryview.py b/Lib/test/test_free_threading/test_memoryview.py new file mode 100644 index 00000000000000..6a99958a83fd30 --- /dev/null +++ b/Lib/test/test_free_threading/test_memoryview.py @@ -0,0 +1,36 @@ +import multiprocessing.shared_memory +from threading import Thread +import unittest + +from test.support import threading_helper + + +class TestBase(unittest.TestCase): + pass + + +def do_race(): + """Repeatly access the memoryview for racing.""" + n = 100 + + obj = multiprocessing.shared_memory.ShareableList("Uq..SeDAmB+EBrkLl.SG.Z+Z.ZdsV..wT+zLxKwdN\b") + threads = [] + for _ in range(n): + threads.append(Thread(target=obj.count, args=(1,))) + + for t in threads: + t.start() + + for t in threads: + t.join() + + del obj + + +@threading_helper.requires_working_threading() +class TestMemoryView(TestBase): + def test_racing_getbuf_and_releasebuf(self): + do_race() + +if __name__ == "__main__": + unittest.main() diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 25634f997ac66b..eb90c12515614d 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1589,7 +1589,9 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); + Py_BEGIN_CRITICAL_SECTION(self); self->exports++; + Py_END_CRITICAL_SECTION(); return 0; } @@ -1598,7 +1600,9 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; + Py_BEGIN_CRITICAL_SECTION(self); self->exports--; + Py_END_CRITICAL_SECTION(); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } From bb9d67e3504c77372c14c108503d2ce5344a7cbe Mon Sep 17 00:00:00 2001 From: edward_xu Date: Sat, 30 Nov 2024 22:55:11 +0800 Subject: [PATCH 2/5] gh-127085: use atomic updating for `exports` and add protection for reading. --- .../test_free_threading/test_memoryview.py | 36 ------------------- Lib/test/test_memoryview.py | 23 +++++++++++- Objects/memoryobject.c | 33 ++++++++++------- 3 files changed, 42 insertions(+), 50 deletions(-) delete mode 100644 Lib/test/test_free_threading/test_memoryview.py diff --git a/Lib/test/test_free_threading/test_memoryview.py b/Lib/test/test_free_threading/test_memoryview.py deleted file mode 100644 index 6a99958a83fd30..00000000000000 --- a/Lib/test/test_free_threading/test_memoryview.py +++ /dev/null @@ -1,36 +0,0 @@ -import multiprocessing.shared_memory -from threading import Thread -import unittest - -from test.support import threading_helper - - -class TestBase(unittest.TestCase): - pass - - -def do_race(): - """Repeatly access the memoryview for racing.""" - n = 100 - - obj = multiprocessing.shared_memory.ShareableList("Uq..SeDAmB+EBrkLl.SG.Z+Z.ZdsV..wT+zLxKwdN\b") - threads = [] - for _ in range(n): - threads.append(Thread(target=obj.count, args=(1,))) - - for t in threads: - t.start() - - for t in threads: - t.join() - - del obj - - -@threading_helper.requires_working_threading() -class TestMemoryView(TestBase): - def test_racing_getbuf_and_releasebuf(self): - do_race() - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 2d4bf5f1408df8..2ad3152877b92a 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -14,8 +14,10 @@ import copy import pickle import struct +import multiprocessing.shared_memory +from threading import Thread -from test.support import import_helper +from test.support import import_helper, threading_helper class MyObject: @@ -679,5 +681,24 @@ def test_picklebuffer_reference_loop(self): self.assertIsNone(wr()) +@threading_helper.requires_working_threading() +class RacingTest(unittest.TestCase): + def test_racing_getbuf_and_releasebuf(self): + """Repeatly access the memoryview for racing.""" + n = 100 + + obj = multiprocessing.shared_memory.ShareableList("Uq..SeDAmB+EBrkLl.SG.Z+Z.ZdsV..wT+zLxKwdN\b") + threads = [] + for _ in range(n): + # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` + # counter of `memoryview`, this issue has no direct relation with `ShareableList`. + threads.append(Thread(target=obj.count, args=(1,))) + + with threading_helper.start_threads(threads): + pass + + del obj + + if __name__ == "__main__": unittest.main() diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index eb90c12515614d..1e64a63092c4e3 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1086,6 +1086,16 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde return ret; } +static inline Py_ssize_t +get_exports(PyMemoryViewObject *buf) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&buf->exports); +#else + return buf->exports; +#endif +} + /****************************************************************************/ /* Release/GC management */ @@ -1098,7 +1108,7 @@ PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char orde static void _memory_release(PyMemoryViewObject *self) { - assert(self->exports == 0); + assert(get_exports(self) == 0); if (self->flags & _Py_MEMORYVIEW_RELEASED) return; @@ -1119,15 +1129,16 @@ static PyObject * memoryview_release_impl(PyMemoryViewObject *self) /*[clinic end generated code: output=d0b7e3ba95b7fcb9 input=bc71d1d51f4a52f0]*/ { - if (self->exports == 0) { + Py_ssize_t exports = get_exports(self); + if (exports == 0) { _memory_release(self); Py_RETURN_NONE; } - if (self->exports > 0) { + if (exports > 0) { PyErr_Format(PyExc_BufferError, - "memoryview has %zd exported buffer%s", self->exports, - self->exports==1 ? "" : "s"); + "memoryview has %zd exported buffer%s", exports, + exports==1 ? "" : "s"); return NULL; } @@ -1140,7 +1151,7 @@ static void memory_dealloc(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - assert(self->exports == 0); + assert(get_exports(self) == 0); _PyObject_GC_UNTRACK(self); _memory_release(self); Py_CLEAR(self->mbuf); @@ -1161,7 +1172,7 @@ static int memory_clear(PyObject *_self) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - if (self->exports == 0) { + if (get_exports(self) == 0) { _memory_release(self); Py_CLEAR(self->mbuf); } @@ -1589,9 +1600,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); - Py_BEGIN_CRITICAL_SECTION(self); - self->exports++; - Py_END_CRITICAL_SECTION(); + _Py_atomic_add_ssize(&self->exports, 1); return 0; } @@ -1600,9 +1609,7 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - Py_BEGIN_CRITICAL_SECTION(self); - self->exports--; - Py_END_CRITICAL_SECTION(); + _Py_atomic_add_ssize(&self->exports, -1); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } From 49957d11132c35ef546fcdc78b2270fc9068d530 Mon Sep 17 00:00:00 2001 From: edward_xu Date: Sun, 1 Dec 2024 00:00:08 +0800 Subject: [PATCH 3/5] gh-127085: refactor test case and add macro for exports atomic actions. --- Lib/test/test_memoryview.py | 29 ++++++++++--------- ...-11-30-23-35-45.gh-issue-127085.KLKylb.rst | 1 + Objects/memoryobject.c | 8 +++++ 3 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 2ad3152877b92a..a2af0a0343ca2c 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -14,9 +14,8 @@ import copy import pickle import struct -import multiprocessing.shared_memory -from threading import Thread +from test import support from test.support import import_helper, threading_helper @@ -682,22 +681,26 @@ def test_picklebuffer_reference_loop(self): @threading_helper.requires_working_threading() +@support.requires_resource("cpu") class RacingTest(unittest.TestCase): def test_racing_getbuf_and_releasebuf(self): """Repeatly access the memoryview for racing.""" - n = 100 - - obj = multiprocessing.shared_memory.ShareableList("Uq..SeDAmB+EBrkLl.SG.Z+Z.ZdsV..wT+zLxKwdN\b") - threads = [] - for _ in range(n): - # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` - # counter of `memoryview`, this issue has no direct relation with `ShareableList`. - threads.append(Thread(target=obj.count, args=(1,))) + from multiprocessing.managers import SharedMemoryManager + from threading import Thread - with threading_helper.start_threads(threads): - pass + n = 100 + with SharedMemoryManager() as smm: + obj = smm.ShareableList(range(100)) + threads = [] + for _ in range(n): + # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` + # counter of `memoryview`, this issue has no direct relation with `ShareableList`. + threads.append(Thread(target=obj.count, args=(1,))) + + with threading_helper.start_threads(threads): + pass - del obj + del obj if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst new file mode 100644 index 00000000000000..18931b45674ff6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst @@ -0,0 +1 @@ +Fix the race condition of ``PyMemoryViewObject->exports`` in free thread diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 1e64a63092c4e3..9be08bca1b51f6 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1600,7 +1600,11 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); + #ifdef Py_GIL_DISABLED _Py_atomic_add_ssize(&self->exports, 1); + #else + self->exports++; + #endif return 0; } @@ -1609,7 +1613,11 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; + #ifdef Py_GIL_DISABLED _Py_atomic_add_ssize(&self->exports, -1); + #else + self->exports--; + #endif return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } From 666fde35308f18880e719e48f6cb4b44b845883b Mon Sep 17 00:00:00 2001 From: edward_xu Date: Sun, 1 Dec 2024 22:31:08 +0800 Subject: [PATCH 4/5] gh-127085: fix formatting issue. --- Objects/memoryobject.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 9be08bca1b51f6..0dba19bc0fec9a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1600,11 +1600,11 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); - #ifdef Py_GIL_DISABLED +#ifdef Py_GIL_DISABLED _Py_atomic_add_ssize(&self->exports, 1); - #else +#else self->exports++; - #endif +#endif return 0; } @@ -1613,11 +1613,11 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; - #ifdef Py_GIL_DISABLED +#ifdef Py_GIL_DISABLED _Py_atomic_add_ssize(&self->exports, -1); - #else +#else self->exports--; - #endif +#endif return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } From 102d30ab010550ae49f919d7b2f99399bb6f7c6a Mon Sep 17 00:00:00 2001 From: edward_xu Date: Sun, 8 Dec 2024 23:11:58 +0800 Subject: [PATCH 5/5] rewrite the blurb changelog. --- .../2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst index 18931b45674ff6..a59b9502eaa33e 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-30-23-35-45.gh-issue-127085.KLKylb.rst @@ -1 +1 @@ -Fix the race condition of ``PyMemoryViewObject->exports`` in free thread +Fix race when exporting a buffer from a :class:`memoryview` object on the :term:`free-threaded ` build.