From 86d3fbbe8ce547d3c99f6f22a607626e49a0f028 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 17 Jun 2025 13:21:54 +0800 Subject: [PATCH 01/13] fix lock-free list access on heapq --- Modules/_heapqmodule.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 7784cdcd9ffa24..d2aee1ffb6be26 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -59,8 +59,8 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - arr[parentpos] = newitem; - arr[pos] = parent; + _Py_atomic_store_ptr(&arr[parentpos], newitem); + _Py_atomic_store_ptr(&arr[pos], parent); pos = parentpos; } return 0; @@ -108,8 +108,8 @@ siftup(PyListObject *heap, Py_ssize_t pos) /* Move the smaller child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - arr[childpos] = tmp2; - arr[pos] = tmp1; + _Py_atomic_store_ptr(&arr[childpos], tmp2); + _Py_atomic_store_ptr(&arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -172,8 +172,10 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) if (!n) return lastelt; returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, lastelt); - if (siftup_func((PyListObject *)heap, 0)) { + // We're in the critical section now + PyListObject *list = _PyList_CAST(heap); + _Py_atomic_store_ptr(&list->ob_item[0], lastelt); + if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; } From 5f24742ff913fac98a8857fda0a15dacc5fd1a58 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 17 Jun 2025 23:10:13 +0800 Subject: [PATCH 02/13] use `FT_ATOMIC` macro to store ptr --- Modules/_heapqmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index d2aee1ffb6be26..8251336a59938b 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -59,8 +59,8 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - _Py_atomic_store_ptr(&arr[parentpos], newitem); - _Py_atomic_store_ptr(&arr[pos], parent); + FT_ATOMIC_STORE_PTR_RELEASE(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELEASE(arr[pos], parent); pos = parentpos; } return 0; @@ -108,8 +108,8 @@ siftup(PyListObject *heap, Py_ssize_t pos) /* Move the smaller child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - _Py_atomic_store_ptr(&arr[childpos], tmp2); - _Py_atomic_store_ptr(&arr[pos], tmp1); + FT_ATOMIC_STORE_PTR_RELEASE(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELEASE(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -174,7 +174,7 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) returnitem = PyList_GET_ITEM(heap, 0); // We're in the critical section now PyListObject *list = _PyList_CAST(heap); - _Py_atomic_store_ptr(&list->ob_item[0], lastelt); + FT_ATOMIC_STORE_PTR_RELEASE(list->ob_item[0], lastelt); if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; From 06328a1e7ba845d0800c942b0f1c1adf2038f300 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 17 Jun 2025 23:14:04 +0800 Subject: [PATCH 03/13] add news entry --- .../next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst diff --git a/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst b/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst new file mode 100644 index 00000000000000..56d32d913ccb57 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst @@ -0,0 +1,2 @@ +Fix races on ``heapq`` updates and lock-free list reads in free-threaded +build. From 3b8ea80db1ef7dea890dc01b12eea13cbfb5cc7b Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 17 Jun 2025 23:52:59 +0800 Subject: [PATCH 04/13] add test --- Lib/test/test_heapq.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index d6623fee9bb2b4..dffcb0f60df836 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -5,9 +5,10 @@ import doctest from test.support import import_helper -from unittest import TestCase, skipUnless +from unittest import TestCase, skipUnless, skipIf from operator import itemgetter + py_heapq = import_helper.import_fresh_module('heapq', blocked=['_heapq']) c_heapq = import_helper.import_fresh_module('heapq', fresh=['_heapq']) @@ -402,6 +403,36 @@ def __le__(self, other): self.assertEqual(hsort(data, LT), target) self.assertRaises(TypeError, data, LE) + @skipIf(py_heapq, 'only used to test c_heapq') + def test_lock_free_list_read(self): + n = 1_000_000 + l = [] + def writer(): + for i in range(n): + self.module.heappush(l, 1) + self.module.heappop(l) + + def reader(): + for i in range(n): + try: + l[0] + except IndexError: + pass + + import threading + threads = [] + for _ in range(10): + t1 = threading.Thread(target=writer) + t2 = threading.Thread(target=reader) + threads.append(t1) + threads.append(t2) + t1.start() + t2.start() + + for t in threads: + t.join() + + class TestHeapPython(TestHeap, TestCase): module = py_heapq From 481ef22265369aa2c3f94a64bd38160080526028 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Wed, 18 Jun 2025 00:07:17 +0800 Subject: [PATCH 05/13] add missing header --- Modules/_heapqmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 8251336a59938b..e430397ac265b9 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -12,6 +12,7 @@ annotated by François Pinard, and converted to C by Raymond Hettinger. #include "Python.h" #include "pycore_list.h" // _PyList_ITEMS(), _PyList_AppendTakeRef() +#include "pycore_pyatomic_ft_wrappers.h" #include "clinic/_heapqmodule.c.h" From d00d6986f86f80af87f3797d3fe9d5bd6b5bc6c7 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang <44627253+xuantengh@users.noreply.github.com> Date: Wed, 18 Jun 2025 11:18:56 +0800 Subject: [PATCH 06/13] Update Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst Co-authored-by: Peter Bierma --- .../next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst b/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst index 56d32d913ccb57..eabf5ea4aaa2ac 100644 --- a/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst +++ b/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst @@ -1,2 +1,2 @@ -Fix races on ``heapq`` updates and lock-free list reads in free-threaded +Fix races on :mod:`heapq` updates and :class:`list` reads on the :term:`free threaded ` build. From dd8f5ceaeac3b0a6a48f98bc15117a864a6f6758 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Wed, 18 Jun 2025 11:23:34 +0800 Subject: [PATCH 07/13] use relaxed atomic store --- Modules/_heapqmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index e430397ac265b9..cc1d741c8546fe 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -60,8 +60,8 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - FT_ATOMIC_STORE_PTR_RELEASE(arr[parentpos], newitem); - FT_ATOMIC_STORE_PTR_RELEASE(arr[pos], parent); + FT_ATOMIC_STORE_PTR_RELAXED(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], parent); pos = parentpos; } return 0; @@ -109,8 +109,8 @@ siftup(PyListObject *heap, Py_ssize_t pos) /* Move the smaller child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - FT_ATOMIC_STORE_PTR_RELEASE(arr[childpos], tmp2); - FT_ATOMIC_STORE_PTR_RELEASE(arr[pos], tmp1); + FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -175,7 +175,7 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) returnitem = PyList_GET_ITEM(heap, 0); // We're in the critical section now PyListObject *list = _PyList_CAST(heap); - FT_ATOMIC_STORE_PTR_RELEASE(list->ob_item[0], lastelt); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], lastelt); if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; From b7528bd7e513b8ae44e5f37960d1878e4afd7ba7 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Wed, 18 Jun 2025 14:38:29 +0800 Subject: [PATCH 08/13] update tests --- Lib/test/test_free_threading/test_heapq.py | 36 +++++++++++++++++++++- Lib/test/test_heapq.py | 33 +------------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index f75fb264c8ac0f..4d4795a2a4ec4e 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -6,7 +6,7 @@ from threading import Thread, Barrier from random import shuffle, randint -from test.support import threading_helper +from test.support import threading_helper, Py_GIL_DISABLED from test import test_heapq @@ -178,6 +178,40 @@ def heapreplace_max_func(max_heap, replace_items): self.assertEqual(len(max_heap), OBJECT_COUNT) self.test_heapq.check_max_invariant(max_heap) + @unittest.skipUnless(Py_GIL_DISABLED, 'only used to test under free-threaded build') + def test_lock_free_list_read(self): + n, n_threads = 1_000_000, 10 + l = [] + barrier = Barrier(n_threads * 2) + + def writer(): + barrier.wait() + for i in range(n): + heapq.heappush(l, 1) + heapq.heappop(l) + + def reader(): + barrier.wait() + for i in range(n): + try: + l[0] + except IndexError: + pass + + import threading + threads = [] + with threading_helper.catch_threading_exception() as cm: + for _ in range(n_threads): + t1 = threading.Thread(target=writer) + t2 = threading.Thread(target=reader) + threads.append(t1) + threads.append(t2) + t1.start() + t2.start() + + for t in threads: + t.join() + @staticmethod def is_sorted_ascending(lst): """ diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index dffcb0f60df836..d6623fee9bb2b4 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -5,10 +5,9 @@ import doctest from test.support import import_helper -from unittest import TestCase, skipUnless, skipIf +from unittest import TestCase, skipUnless from operator import itemgetter - py_heapq = import_helper.import_fresh_module('heapq', blocked=['_heapq']) c_heapq = import_helper.import_fresh_module('heapq', fresh=['_heapq']) @@ -403,36 +402,6 @@ def __le__(self, other): self.assertEqual(hsort(data, LT), target) self.assertRaises(TypeError, data, LE) - @skipIf(py_heapq, 'only used to test c_heapq') - def test_lock_free_list_read(self): - n = 1_000_000 - l = [] - def writer(): - for i in range(n): - self.module.heappush(l, 1) - self.module.heappop(l) - - def reader(): - for i in range(n): - try: - l[0] - except IndexError: - pass - - import threading - threads = [] - for _ in range(10): - t1 = threading.Thread(target=writer) - t2 = threading.Thread(target=reader) - threads.append(t1) - threads.append(t2) - t1.start() - t2.start() - - for t in threads: - t.join() - - class TestHeapPython(TestHeap, TestCase): module = py_heapq From 925a093d6a30fb3f22148d6c5352e415144d3b8b Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 19 Jun 2025 08:48:17 +0800 Subject: [PATCH 09/13] reduce test num elements --- Lib/test/test_free_threading/test_heapq.py | 2 +- Modules/_heapqmodule.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index 4d4795a2a4ec4e..10f145760c4dc7 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -180,7 +180,7 @@ def heapreplace_max_func(max_heap, replace_items): @unittest.skipUnless(Py_GIL_DISABLED, 'only used to test under free-threaded build') def test_lock_free_list_read(self): - n, n_threads = 1_000_000, 10 + n, n_threads = 1_000, 10 l = [] barrier = Barrier(n_threads * 2) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index cc1d741c8546fe..6c59602d34bc09 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -173,7 +173,6 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) if (!n) return lastelt; returnitem = PyList_GET_ITEM(heap, 0); - // We're in the critical section now PyListObject *list = _PyList_CAST(heap); FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], lastelt); if (siftup_func(list, 0)) { From 27d6cb62cc97a601df994081a78a4d1da422f62b Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 19 Jun 2025 18:29:52 +0800 Subject: [PATCH 10/13] use ft store in heappushpop and heapreplace as well --- Modules/_heapqmodule.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 6c59602d34bc09..05677f1ce7a54a 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -210,8 +210,9 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup_func((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -286,8 +287,9 @@ _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup(list, 0)) { Py_DECREF(returnitem); return NULL; } From 32d767fdf0ad3fa486ae15283c79d94c44da8cf5 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 19 Jun 2025 21:14:18 +0800 Subject: [PATCH 11/13] use `run_concurrent` to spawn threads --- Lib/test/test_free_threading/test_heapq.py | 47 +++++++++------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index 10f145760c4dc7..ee7adfb2b78d83 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -3,10 +3,10 @@ import heapq from enum import Enum -from threading import Thread, Barrier +from threading import Thread, Barrier, Lock from random import shuffle, randint -from test.support import threading_helper, Py_GIL_DISABLED +from test.support import threading_helper from test import test_heapq @@ -178,39 +178,32 @@ def heapreplace_max_func(max_heap, replace_items): self.assertEqual(len(max_heap), OBJECT_COUNT) self.test_heapq.check_max_invariant(max_heap) - @unittest.skipUnless(Py_GIL_DISABLED, 'only used to test under free-threaded build') def test_lock_free_list_read(self): n, n_threads = 1_000, 10 l = [] barrier = Barrier(n_threads * 2) - def writer(): - barrier.wait() - for i in range(n): - heapq.heappush(l, 1) - heapq.heappop(l) + count = 0 + lock = Lock() + + def worker(): + with lock: + nonlocal count + x = count + count += 1 - def reader(): barrier.wait() for i in range(n): - try: - l[0] - except IndexError: - pass - - import threading - threads = [] - with threading_helper.catch_threading_exception() as cm: - for _ in range(n_threads): - t1 = threading.Thread(target=writer) - t2 = threading.Thread(target=reader) - threads.append(t1) - threads.append(t2) - t1.start() - t2.start() - - for t in threads: - t.join() + if x % 2: + heapq.heappush(l, 1) + heapq.heappop(l) + else: + try: + l[0] + except IndexError: + pass + + self.run_concurrently(worker, (), n_threads * 2) @staticmethod def is_sorted_ascending(lst): From 4fb93896bc9ecede32cf30720b42582e4fb7bfda Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 19 Jun 2025 21:58:27 +0800 Subject: [PATCH 12/13] fix --- Modules/_heapqmodule.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 05677f1ce7a54a..d889766af5f210 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -441,8 +441,8 @@ siftdown_max(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - arr[parentpos] = newitem; - arr[pos] = parent; + FT_ATOMIC_STORE_PTR_RELAXED(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], parent); pos = parentpos; } return 0; @@ -490,8 +490,8 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) /* Move the smaller child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - arr[childpos] = tmp2; - arr[pos] = tmp1; + FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -625,8 +625,9 @@ _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup_max((PyListObject *)heap, 0) < 0) { + PyListObject * list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup_max(list, 0) < 0) { Py_DECREF(returnitem); return NULL; } From a10560eeb0bab74216b7d75d0e3d75f64e40b3e9 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Thu, 19 Jun 2025 22:40:54 +0800 Subject: [PATCH 13/13] fix style --- Modules/_heapqmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index d889766af5f210..560fe431fcac99 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -625,7 +625,7 @@ _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyListObject * list = _PyList_CAST(heap); + PyListObject *list = _PyList_CAST(heap); FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); if (siftup_max(list, 0) < 0) { Py_DECREF(returnitem);