From ec3cade6608c3cdec39a0afa2961ac19fe41c78e Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 27 May 2025 07:44:08 -0500 Subject: [PATCH 01/10] gh-116738: Make _heapq module thread-safe --- Lib/test/test_free_threading/test_heapq.py | 279 +++++++++++++++++++++ Modules/_heapqmodule.c | 30 ++- Modules/clinic/_heapqmodule.c.h | 23 +- 3 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 Lib/test/test_free_threading/test_heapq.py diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py new file mode 100644 index 00000000000000..e83e1298209bf8 --- /dev/null +++ b/Lib/test/test_free_threading/test_heapq.py @@ -0,0 +1,279 @@ +import unittest + +import heapq +import operator + +from enum import Enum +from threading import Thread, Barrier +from random import shuffle, randint + +from test.support import threading_helper + + +NTHREADS: int = 10 +OBJECT_COUNT: int = 5_000 + + +class HeapKind(Enum): + MIN = 1 + MAX = 2 + + +@threading_helper.requires_working_threading() +class TestHeapq(unittest.TestCase): + def test_racing_heapify(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + + def heapify_func(heap: list[int]): + heapq.heapify(heap) + + self.run_concurrently( + worker_func=heapify_func, args=(heap,), nthreads=NTHREADS + ) + self.assertTrue(self.is_min_heap_property_satisfied(heap)) + + def test_racing_heappush(self): + heap = [] + + def heappush_func(heap: list[int]): + for item in reversed(range(OBJECT_COUNT)): + heapq.heappush(heap, item) + + self.run_concurrently( + worker_func=heappush_func, args=(heap,), nthreads=NTHREADS + ) + self.assertTrue(self.is_min_heap_property_satisfied(heap)) + + def test_racing_heappop(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + heapq.heapify(heap) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(0, OBJECT_COUNT % NTHREADS) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_func(heap: list[int], pop_count: int): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop(heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_ascending(local_list)) + + self.run_concurrently( + worker_func=heappop_func, + args=(heap, per_thread_pop_count), + nthreads=NTHREADS, + ) + self.assertEqual(0, len(heap)) + + def test_racing_heappushpop(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + heapq.heapify(heap) + + pushpop_items = [ + randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) + ] + + def heappushpop_func(heap: list[int], pushpop_items: list[int]): + for item in pushpop_items: + popped_item = heapq.heappushpop(heap, item) + self.assertTrue(popped_item <= item) + + self.run_concurrently( + worker_func=heappushpop_func, + args=(heap, pushpop_items), + nthreads=NTHREADS, + ) + self.assertEqual(OBJECT_COUNT, len(heap)) + self.assertTrue(self.is_min_heap_property_satisfied(heap)) + + def test_racing_heapreplace(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + heapq.heapify(heap) + + replace_items = [ + randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) + ] + + def heapreplace_func(heap: list[int], replace_items: list[int]): + for item in replace_items: + popped_item = heapq.heapreplace(heap, item) + + self.run_concurrently( + worker_func=heapreplace_func, + args=(heap, replace_items), + nthreads=NTHREADS, + ) + self.assertEqual(OBJECT_COUNT, len(heap)) + self.assertTrue(self.is_min_heap_property_satisfied(heap)) + + def test_racing_heapify_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + + def heapify_max_func(max_heap: list[int]): + heapq.heapify_max(max_heap) + + self.run_concurrently( + worker_func=heapify_max_func, args=(max_heap,), nthreads=NTHREADS + ) + self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + + def test_racing_heappush_max(self): + max_heap = [] + + def heappush_max_func(max_heap: list[int]): + for item in range(OBJECT_COUNT): + heapq.heappush_max(max_heap, item) + + self.run_concurrently( + worker_func=heappush_max_func, args=(max_heap,), nthreads=NTHREADS + ) + self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + + def test_racing_heappop_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + heapq.heapify_max(max_heap) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(0, OBJECT_COUNT % NTHREADS) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_max_func(max_heap: list[int], pop_count: int): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop_max(max_heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_descending(local_list)) + + self.run_concurrently( + worker_func=heappop_max_func, + args=(max_heap, per_thread_pop_count), + nthreads=NTHREADS, + ) + self.assertEqual(0, len(max_heap)) + + def test_racing_heappushpop_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + heapq.heapify_max(max_heap) + + pushpop_items = [ + randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) + ] + + def heappushpop_max_func( + max_heap: list[int], pushpop_items: list[int] + ): + for item in pushpop_items: + popped_item = heapq.heappushpop_max(max_heap, item) + self.assertTrue(popped_item >= item) + + self.run_concurrently( + worker_func=heappushpop_max_func, + args=(max_heap, pushpop_items), + nthreads=NTHREADS, + ) + self.assertEqual(OBJECT_COUNT, len(max_heap)) + self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + + def test_racing_heapreplace_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + heapq.heapify_max(max_heap) + + replace_items = [ + randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) + ] + + def heapreplace_max_func( + max_heap: list[int], replace_items: list[int] + ): + for item in replace_items: + popped_item = heapq.heapreplace_max(max_heap, item) + + self.run_concurrently( + worker_func=heapreplace_max_func, + args=(max_heap, replace_items), + nthreads=NTHREADS, + ) + self.assertEqual(OBJECT_COUNT, len(max_heap)) + self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + + def is_min_heap_property_satisfied(self, heap: list[object]) -> bool: + """ + The value of a parent node should be less than or equal to the + values of its children. + """ + return self.is_heap_property_satisfied(heap, HeapKind.MIN) + + def is_max_heap_property_satisfied(self, heap: list[object]) -> bool: + """ + The value of a parent node should be greater than or equal to the + values of its children. + """ + return self.is_heap_property_satisfied(heap, HeapKind.MAX) + + @staticmethod + def is_heap_property_satisfied( + heap: list[object], heap_kind: HeapKind + ) -> bool: + """ + Check if the heap property is satisfied. + """ + op = operator.le if heap_kind == HeapKind.MIN else operator.ge + # position 0 has no parent + for pos in range(1, len(heap)): + parent_pos = (pos - 1) >> 1 + if not op(heap[parent_pos], heap[pos]): + return False + + return True + + @staticmethod + def is_sorted_ascending(lst: list[object]) -> bool: + """ + Check if the list is sorted in ascending order (non-decreasing). + """ + return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) + + @staticmethod + def is_sorted_descending(lst: list[object]) -> bool: + """ + Check if the list is sorted in descending order (non-increasing). + """ + return all(lst[i - 1] >= lst[i] for i in range(1, len(lst))) + + @staticmethod + def run_concurrently(worker_func, args, nthreads) -> None: + """ + Run the worker function concurrently in multiple threads. + """ + barrier = Barrier(NTHREADS) + + def wrapper_func(*args): + # Wait for all threadss to reach this point before proceeding. + barrier.wait() + worker_func(*args) + + workers = [] + for _ in range(nthreads): + worker = Thread(target=wrapper_func, args=args) + workers.append(worker) + worker.start() + + for worker in workers: + worker.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 095866eec7d75a..92900073b85583 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -117,6 +117,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush heap: object(subclass_of='&PyList_Type') @@ -128,7 +129,7 @@ Push item onto heap, maintaining the heap invariant. static PyObject * _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=912c094f47663935 input=7c69611f3698aceb]*/ +/*[clinic end generated code: output=912c094f47663935 input=f7a4f03ef8d52e67]*/ { if (PyList_Append(heap, item)) return NULL; @@ -171,6 +172,7 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heappop heap: object(subclass_of='&PyList_Type') @@ -181,7 +183,7 @@ Pop the smallest item off the heap, maintaining the heap invariant. static PyObject * _heapq_heappop_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=96dfe82d37d9af76 input=91487987a583c856]*/ +/*[clinic end generated code: output=96dfe82d37d9af76 input=ed396461b153dd51]*/ { return heappop_internal(heap, siftup); } @@ -207,6 +209,7 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec /*[clinic input] +@critical_section heap _heapq.heapreplace heap: object(subclass_of='&PyList_Type') @@ -226,12 +229,13 @@ this routine unless written as part of a conditional replacement: static PyObject * _heapq_heapreplace_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=82ea55be8fbe24b4 input=719202ac02ba10c8]*/ +/*[clinic end generated code: output=82ea55be8fbe24b4 input=9be1678b817ef1a9]*/ { return heapreplace_internal(heap, item, siftup); } /*[clinic input] +@critical_section heap _heapq.heappushpop heap: object(subclass_of='&PyList_Type') @@ -246,7 +250,7 @@ a separate call to heappop(). static PyObject * _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=67231dc98ed5774f input=5dc701f1eb4a4aa7]*/ +/*[clinic end generated code: output=67231dc98ed5774f input=db05c81b1dd92c44]*/ { PyObject *returnitem; int cmp; @@ -371,6 +375,7 @@ heapify_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heapify heap: object(subclass_of='&PyList_Type') @@ -381,7 +386,7 @@ Transform list into a heap, in-place, in O(len(heap)) time. static PyObject * _heapq_heapify_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=e63a636fcf83d6d0 input=53bb7a2166febb73]*/ +/*[clinic end generated code: output=e63a636fcf83d6d0 input=aaaaa028b9b6af08]*/ { return heapify_internal(heap, siftup); } @@ -481,6 +486,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush_max heap: object(subclass_of='&PyList_Type') @@ -492,7 +498,7 @@ Push item onto max heap, maintaining the heap invariant. static PyObject * _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=c869d5f9deb08277 input=4743d7db137b6e2b]*/ +/*[clinic end generated code: output=c869d5f9deb08277 input=c437e3d1ff8dcb70]*/ { if (PyList_Append(heap, item)) { return NULL; @@ -506,6 +512,7 @@ _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) } /*[clinic input] +@critical_section heap _heapq.heappop_max heap: object(subclass_of='&PyList_Type') @@ -516,12 +523,13 @@ Maxheap variant of heappop. static PyObject * _heapq_heappop_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=2f051195ab404b77 input=e62b14016a5a26de]*/ +/*[clinic end generated code: output=2f051195ab404b77 input=5d70c997798aec64]*/ { return heappop_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapreplace_max heap: object(subclass_of='&PyList_Type') @@ -533,12 +541,13 @@ Maxheap variant of heapreplace. static PyObject * _heapq_heapreplace_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=8770778b5a9cbe9b input=21a3d28d757c881c]*/ +/*[clinic end generated code: output=8770778b5a9cbe9b input=fe70175356e4a649]*/ { return heapreplace_internal(heap, item, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapify_max heap: object(subclass_of='&PyList_Type') @@ -549,12 +558,13 @@ Maxheap variant of heapify. static PyObject * _heapq_heapify_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=8401af3856529807 input=edda4255728c431e]*/ +/*[clinic end generated code: output=8401af3856529807 input=4eee63231e7d1573]*/ { return heapify_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heappushpop_max heap: object(subclass_of='&PyList_Type') @@ -569,7 +579,7 @@ a separate call to heappop_max(). static PyObject * _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=ff0019f0941aca0d input=525a843013cbd6c0]*/ +/*[clinic end generated code: output=ff0019f0941aca0d input=24d0defa6fd6df4a]*/ { PyObject *returnitem; int cmp; diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index 81d108627265ab..b43155b6c24e3c 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_heapq_heappush__doc__, @@ -32,7 +33,9 @@ _heapq_heappush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -61,7 +64,9 @@ _heapq_heappop(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -103,7 +108,9 @@ _heapq_heapreplace(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -140,7 +147,9 @@ _heapq_heappushpop(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -169,7 +178,9 @@ _heapq_heapify(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -203,7 +214,9 @@ _heapq_heappush_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -232,7 +245,9 @@ _heapq_heappop_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -266,7 +281,9 @@ _heapq_heapreplace_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -295,7 +312,9 @@ _heapq_heapify_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,9 +351,11 @@ _heapq_heappushpop_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=f55d8595ce150c76 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e83d50002c29a96d input=a9049054013a1b77]*/ From 01b2be4c2d6577415a1e9f1ed0b10d568e76b532 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Mon, 2 Jun 2025 14:03:16 -0700 Subject: [PATCH 02/10] gh-116738: Add news entry in Misc/NEWS/next --- .../2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst new file mode 100644 index 00000000000000..0a77a75c2b1e5a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst @@ -0,0 +1 @@ +Make methods on :class:`heapq` thread-safe when the GIL is disabled. From 68f3a266cdbb5239d8d9ad2ef12112998b301459 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Tue, 3 Jun 2025 08:57:01 -0700 Subject: [PATCH 03/10] gh-116738: Address the review comments --- Lib/test/test_free_threading/test_heapq.py | 110 ++++++++---------- ...-06-02-13-57-40.gh-issue-116738.ycJsL8.rst | 2 +- 2 files changed, 52 insertions(+), 60 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index e83e1298209bf8..3bb3ca65d84254 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -25,11 +25,8 @@ def test_racing_heapify(self): heap = list(range(OBJECT_COUNT)) shuffle(heap) - def heapify_func(heap: list[int]): - heapq.heapify(heap) - self.run_concurrently( - worker_func=heapify_func, args=(heap,), nthreads=NTHREADS + worker_func=heapq.heapify, args=(heap,), nthreads=NTHREADS ) self.assertTrue(self.is_min_heap_property_satisfied(heap)) @@ -46,12 +43,10 @@ def heappush_func(heap: list[int]): self.assertTrue(self.is_min_heap_property_satisfied(heap)) def test_racing_heappop(self): - heap = list(range(OBJECT_COUNT)) - shuffle(heap) - heapq.heapify(heap) + heap = self.create_heap(OBJECT_COUNT, HeapKind.MIN) # Each thread pops (OBJECT_COUNT / NTHREADS) items - self.assertEqual(0, OBJECT_COUNT % NTHREADS) + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) per_thread_pop_count = OBJECT_COUNT // NTHREADS def heappop_func(heap: list[int], pop_count: int): @@ -68,16 +63,11 @@ def heappop_func(heap: list[int], pop_count: int): args=(heap, per_thread_pop_count), nthreads=NTHREADS, ) - self.assertEqual(0, len(heap)) + self.assertEqual(len(heap), 0) def test_racing_heappushpop(self): - heap = list(range(OBJECT_COUNT)) - shuffle(heap) - heapq.heapify(heap) - - pushpop_items = [ - randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) - ] + heap = self.create_heap(OBJECT_COUNT, HeapKind.MIN) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) def heappushpop_func(heap: list[int], pushpop_items: list[int]): for item in pushpop_items: @@ -89,17 +79,12 @@ def heappushpop_func(heap: list[int], pushpop_items: list[int]): args=(heap, pushpop_items), nthreads=NTHREADS, ) - self.assertEqual(OBJECT_COUNT, len(heap)) + self.assertEqual(len(heap), OBJECT_COUNT) self.assertTrue(self.is_min_heap_property_satisfied(heap)) def test_racing_heapreplace(self): - heap = list(range(OBJECT_COUNT)) - shuffle(heap) - heapq.heapify(heap) - - replace_items = [ - randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) - ] + heap = self.create_heap(OBJECT_COUNT, HeapKind.MIN) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) def heapreplace_func(heap: list[int], replace_items: list[int]): for item in replace_items: @@ -110,18 +95,15 @@ def heapreplace_func(heap: list[int], replace_items: list[int]): args=(heap, replace_items), nthreads=NTHREADS, ) - self.assertEqual(OBJECT_COUNT, len(heap)) + self.assertEqual(len(heap), OBJECT_COUNT) self.assertTrue(self.is_min_heap_property_satisfied(heap)) def test_racing_heapify_max(self): max_heap = list(range(OBJECT_COUNT)) shuffle(max_heap) - def heapify_max_func(max_heap: list[int]): - heapq.heapify_max(max_heap) - self.run_concurrently( - worker_func=heapify_max_func, args=(max_heap,), nthreads=NTHREADS + worker_func=heapq.heapify_max, args=(max_heap,), nthreads=NTHREADS ) self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) @@ -138,12 +120,10 @@ def heappush_max_func(max_heap: list[int]): self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) def test_racing_heappop_max(self): - max_heap = list(range(OBJECT_COUNT)) - shuffle(max_heap) - heapq.heapify_max(max_heap) + max_heap = self.create_heap(OBJECT_COUNT, HeapKind.MAX) # Each thread pops (OBJECT_COUNT / NTHREADS) items - self.assertEqual(0, OBJECT_COUNT % NTHREADS) + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) per_thread_pop_count = OBJECT_COUNT // NTHREADS def heappop_max_func(max_heap: list[int], pop_count: int): @@ -160,16 +140,11 @@ def heappop_max_func(max_heap: list[int], pop_count: int): args=(max_heap, per_thread_pop_count), nthreads=NTHREADS, ) - self.assertEqual(0, len(max_heap)) + self.assertEqual(len(max_heap), 0) def test_racing_heappushpop_max(self): - max_heap = list(range(OBJECT_COUNT)) - shuffle(max_heap) - heapq.heapify_max(max_heap) - - pushpop_items = [ - randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) - ] + max_heap = self.create_heap(OBJECT_COUNT, HeapKind.MAX) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) def heappushpop_max_func( max_heap: list[int], pushpop_items: list[int] @@ -183,17 +158,12 @@ def heappushpop_max_func( args=(max_heap, pushpop_items), nthreads=NTHREADS, ) - self.assertEqual(OBJECT_COUNT, len(max_heap)) + self.assertEqual(len(max_heap), OBJECT_COUNT) self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) def test_racing_heapreplace_max(self): - max_heap = list(range(OBJECT_COUNT)) - shuffle(max_heap) - heapq.heapify_max(max_heap) - - replace_items = [ - randint(-OBJECT_COUNT, OBJECT_COUNT) for _ in range(OBJECT_COUNT) - ] + max_heap = self.create_heap(OBJECT_COUNT, HeapKind.MAX) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) def heapreplace_max_func( max_heap: list[int], replace_items: list[int] @@ -206,7 +176,7 @@ def heapreplace_max_func( args=(max_heap, replace_items), nthreads=NTHREADS, ) - self.assertEqual(OBJECT_COUNT, len(max_heap)) + self.assertEqual(len(max_heap), OBJECT_COUNT) self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) def is_min_heap_property_satisfied(self, heap: list[object]) -> bool: @@ -254,25 +224,47 @@ def is_sorted_descending(lst: list[object]) -> bool: return all(lst[i - 1] >= lst[i] for i in range(1, len(lst))) @staticmethod - def run_concurrently(worker_func, args, nthreads) -> None: + def create_heap(size: int, heap_kind: HeapKind) -> list[int]: + """ + Create a min/max heap where elements are in the range (0, size - 1) and + shuffled before heapify. + """ + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + if heap_kind == HeapKind.MIN: + heapq.heapify(heap) + else: + heapq.heapify_max(heap) + + return heap + + @staticmethod + def create_random_list(a: int, b: int, size: int) -> list[int]: + """ + Create a random list where elements are in the range a <= elem <= b + """ + return [randint(-a, b) for _ in range(size)] + + def run_concurrently(self, worker_func, args, nthreads) -> None: """ Run the worker function concurrently in multiple threads. """ - barrier = Barrier(NTHREADS) + barrier = Barrier(nthreads) def wrapper_func(*args): # Wait for all threadss to reach this point before proceeding. barrier.wait() worker_func(*args) - workers = [] - for _ in range(nthreads): - worker = Thread(target=wrapper_func, args=args) - workers.append(worker) - worker.start() + with threading_helper.catch_threading_exception() as cm: + workers = ( + Thread(target=wrapper_func, args=args) for _ in range(nthreads) + ) + with threading_helper.start_threads(workers): + pass - for worker in workers: - worker.join() + # Worker threads should not raise any exceptions + self.assertIsNone(cm.exc_value) if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst index 0a77a75c2b1e5a..506eefdb21aa9a 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-02-13-57-40.gh-issue-116738.ycJsL8.rst @@ -1 +1 @@ -Make methods on :class:`heapq` thread-safe when the GIL is disabled. +Make methods in :mod:`heapq` thread-safe on the :term:`free threaded ` build. From 41d145ad5c8bd756783b350bd0ff2ef10858cd87 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Tue, 3 Jun 2025 13:44:44 -0700 Subject: [PATCH 04/10] gh-116738: Address the review comments --- Lib/test/test_free_threading/test_heapq.py | 2 +- Modules/_heapqmodule.c | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index 3bb3ca65d84254..5cf93664274238 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -241,7 +241,7 @@ def create_heap(size: int, heap_kind: HeapKind) -> list[int]: @staticmethod def create_random_list(a: int, b: int, size: int) -> list[int]: """ - Create a random list where elements are in the range a <= elem <= b + Create a list of random numbers between a and b (inclusive). """ return [randint(-a, b) for _ in range(size)] diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 92900073b85583..f0755cb60cb1b1 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -11,7 +11,7 @@ annotated by François Pinard, and converted to C by Raymond Hettinger. #endif #include "Python.h" -#include "pycore_list.h" // _PyList_ITEMS() +#include "pycore_list.h" // _PyList_ITEMS(), _PyList_AppendTakeRef() #include "clinic/_heapqmodule.c.h" @@ -131,7 +131,9 @@ static PyObject * _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) /*[clinic end generated code: output=912c094f47663935 input=f7a4f03ef8d52e67]*/ { - if (PyList_Append(heap, item)) + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) return NULL; if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) @@ -500,7 +502,9 @@ static PyObject * _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) /*[clinic end generated code: output=c869d5f9deb08277 input=c437e3d1ff8dcb70]*/ { - if (PyList_Append(heap, item)) { + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; } From a22da8f9af597d36b5e320f7dccd9d3973fc9f1e Mon Sep 17 00:00:00 2001 From: alperyoney Date: Wed, 4 Jun 2025 16:09:01 -0700 Subject: [PATCH 05/10] gh-116738: Remove type hints --- Lib/test/test_free_threading/test_heapq.py | 88 +++++++++------------- 1 file changed, 36 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index 5cf93664274238..d0f9bbbdb862a8 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -10,11 +10,11 @@ from test.support import threading_helper -NTHREADS: int = 10 -OBJECT_COUNT: int = 5_000 +NTHREADS = 10 +OBJECT_COUNT = 5_000 -class HeapKind(Enum): +class Heap(Enum): MIN = 1 MAX = 2 @@ -28,28 +28,28 @@ def test_racing_heapify(self): self.run_concurrently( worker_func=heapq.heapify, args=(heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_min_heap_property_satisfied(heap)) + self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) def test_racing_heappush(self): heap = [] - def heappush_func(heap: list[int]): + def heappush_func(heap): for item in reversed(range(OBJECT_COUNT)): heapq.heappush(heap, item) self.run_concurrently( worker_func=heappush_func, args=(heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_min_heap_property_satisfied(heap)) + self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) def test_racing_heappop(self): - heap = self.create_heap(OBJECT_COUNT, HeapKind.MIN) + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) # Each thread pops (OBJECT_COUNT / NTHREADS) items self.assertEqual(OBJECT_COUNT % NTHREADS, 0) per_thread_pop_count = OBJECT_COUNT // NTHREADS - def heappop_func(heap: list[int], pop_count: int): + def heappop_func(heap, pop_count): local_list = [] for _ in range(pop_count): item = heapq.heappop(heap) @@ -66,10 +66,10 @@ def heappop_func(heap: list[int], pop_count: int): self.assertEqual(len(heap), 0) def test_racing_heappushpop(self): - heap = self.create_heap(OBJECT_COUNT, HeapKind.MIN) + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) - def heappushpop_func(heap: list[int], pushpop_items: list[int]): + def heappushpop_func(heap, pushpop_items): for item in pushpop_items: popped_item = heapq.heappushpop(heap, item) self.assertTrue(popped_item <= item) @@ -80,13 +80,13 @@ def heappushpop_func(heap: list[int], pushpop_items: list[int]): nthreads=NTHREADS, ) self.assertEqual(len(heap), OBJECT_COUNT) - self.assertTrue(self.is_min_heap_property_satisfied(heap)) + self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) def test_racing_heapreplace(self): - heap = self.create_heap(OBJECT_COUNT, HeapKind.MIN) + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) - def heapreplace_func(heap: list[int], replace_items: list[int]): + def heapreplace_func(heap, replace_items): for item in replace_items: popped_item = heapq.heapreplace(heap, item) @@ -96,7 +96,7 @@ def heapreplace_func(heap: list[int], replace_items: list[int]): nthreads=NTHREADS, ) self.assertEqual(len(heap), OBJECT_COUNT) - self.assertTrue(self.is_min_heap_property_satisfied(heap)) + self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) def test_racing_heapify_max(self): max_heap = list(range(OBJECT_COUNT)) @@ -105,28 +105,28 @@ def test_racing_heapify_max(self): self.run_concurrently( worker_func=heapq.heapify_max, args=(max_heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) def test_racing_heappush_max(self): max_heap = [] - def heappush_max_func(max_heap: list[int]): + def heappush_max_func(max_heap): for item in range(OBJECT_COUNT): heapq.heappush_max(max_heap, item) self.run_concurrently( worker_func=heappush_max_func, args=(max_heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) def test_racing_heappop_max(self): - max_heap = self.create_heap(OBJECT_COUNT, HeapKind.MAX) + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) # Each thread pops (OBJECT_COUNT / NTHREADS) items self.assertEqual(OBJECT_COUNT % NTHREADS, 0) per_thread_pop_count = OBJECT_COUNT // NTHREADS - def heappop_max_func(max_heap: list[int], pop_count: int): + def heappop_max_func(max_heap, pop_count): local_list = [] for _ in range(pop_count): item = heapq.heappop_max(max_heap) @@ -143,12 +143,10 @@ def heappop_max_func(max_heap: list[int], pop_count: int): self.assertEqual(len(max_heap), 0) def test_racing_heappushpop_max(self): - max_heap = self.create_heap(OBJECT_COUNT, HeapKind.MAX) + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) - def heappushpop_max_func( - max_heap: list[int], pushpop_items: list[int] - ): + def heappushpop_max_func(max_heap, pushpop_items): for item in pushpop_items: popped_item = heapq.heappushpop_max(max_heap, item) self.assertTrue(popped_item >= item) @@ -159,15 +157,13 @@ def heappushpop_max_func( nthreads=NTHREADS, ) self.assertEqual(len(max_heap), OBJECT_COUNT) - self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) + self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) def test_racing_heapreplace_max(self): - max_heap = self.create_heap(OBJECT_COUNT, HeapKind.MAX) + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) - def heapreplace_max_func( - max_heap: list[int], replace_items: list[int] - ): + def heapreplace_max_func(max_heap, replace_items): for item in replace_items: popped_item = heapq.heapreplace_max(max_heap, item) @@ -177,30 +173,18 @@ def heapreplace_max_func( nthreads=NTHREADS, ) self.assertEqual(len(max_heap), OBJECT_COUNT) - self.assertTrue(self.is_max_heap_property_satisfied(max_heap)) - - def is_min_heap_property_satisfied(self, heap: list[object]) -> bool: - """ - The value of a parent node should be less than or equal to the - values of its children. - """ - return self.is_heap_property_satisfied(heap, HeapKind.MIN) - - def is_max_heap_property_satisfied(self, heap: list[object]) -> bool: - """ - The value of a parent node should be greater than or equal to the - values of its children. - """ - return self.is_heap_property_satisfied(heap, HeapKind.MAX) + self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) @staticmethod - def is_heap_property_satisfied( - heap: list[object], heap_kind: HeapKind - ) -> bool: + def is_heap_property_satisfied(heap, heap_kind): """ Check if the heap property is satisfied. + MIN-Heap: The value of a parent node should be less than or equal to the + values of its children. + MAX-Heap: The value of a parent node should be greater than or equal to the + values of its children. """ - op = operator.le if heap_kind == HeapKind.MIN else operator.ge + op = operator.le if heap_kind == Heap.MIN else operator.ge # position 0 has no parent for pos in range(1, len(heap)): parent_pos = (pos - 1) >> 1 @@ -210,28 +194,28 @@ def is_heap_property_satisfied( return True @staticmethod - def is_sorted_ascending(lst: list[object]) -> bool: + def is_sorted_ascending(lst): """ Check if the list is sorted in ascending order (non-decreasing). """ return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) @staticmethod - def is_sorted_descending(lst: list[object]) -> bool: + def is_sorted_descending(lst): """ Check if the list is sorted in descending order (non-increasing). """ return all(lst[i - 1] >= lst[i] for i in range(1, len(lst))) @staticmethod - def create_heap(size: int, heap_kind: HeapKind) -> list[int]: + def create_heap(size, heap_kind): """ Create a min/max heap where elements are in the range (0, size - 1) and shuffled before heapify. """ heap = list(range(OBJECT_COUNT)) shuffle(heap) - if heap_kind == HeapKind.MIN: + if heap_kind == Heap.MIN: heapq.heapify(heap) else: heapq.heapify_max(heap) @@ -239,7 +223,7 @@ def create_heap(size: int, heap_kind: HeapKind) -> list[int]: return heap @staticmethod - def create_random_list(a: int, b: int, size: int) -> list[int]: + def create_random_list(a, b, size): """ Create a list of random numbers between a and b (inclusive). """ From 75a1d3a5d429d34c578660cbf354a7a470728562 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Thu, 5 Jun 2025 09:44:06 -0700 Subject: [PATCH 06/10] gh-116738: Address the review comments --- Lib/test/test_free_threading/test_heapq.py | 4 ++-- Modules/_heapqmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index d0f9bbbdb862a8..1f765ac004bf76 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -88,7 +88,7 @@ def test_racing_heapreplace(self): def heapreplace_func(heap, replace_items): for item in replace_items: - popped_item = heapq.heapreplace(heap, item) + heapq.heapreplace(heap, item) self.run_concurrently( worker_func=heapreplace_func, @@ -165,7 +165,7 @@ def test_racing_heapreplace_max(self): def heapreplace_max_func(max_heap, replace_items): for item in replace_items: - popped_item = heapq.heapreplace_max(max_heap, item) + heapq.heapreplace_max(max_heap, item) self.run_concurrently( worker_func=heapreplace_max_func, diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index f0755cb60cb1b1..ddf1c7cd5e3231 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -133,7 +133,7 @@ _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) { // In a free-threaded build, the heap is locked at this point. // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. - if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_XNewRef(item))) return NULL; if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) @@ -504,7 +504,7 @@ _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) { // In a free-threaded build, the heap is locked at this point. // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. - if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_XNewRef(item))) { return NULL; } From e8138be42a97cd63511773e42c76d390b4d84bc4 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Thu, 5 Jun 2025 14:29:07 -0700 Subject: [PATCH 07/10] gh-116738: Add NULL check for the item arg in heappush() --- Modules/_heapqmodule.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index ddf1c7cd5e3231..7784cdcd9ffa24 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -131,13 +131,20 @@ static PyObject * _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) /*[clinic end generated code: output=912c094f47663935 input=f7a4f03ef8d52e67]*/ { + if (item == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + // In a free-threaded build, the heap is locked at this point. // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. - if (_PyList_AppendTakeRef((PyListObject *)heap, Py_XNewRef(item))) + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; + } - if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) + if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) { return NULL; + } Py_RETURN_NONE; } @@ -502,9 +509,14 @@ static PyObject * _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) /*[clinic end generated code: output=c869d5f9deb08277 input=c437e3d1ff8dcb70]*/ { + if (item == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + // In a free-threaded build, the heap is locked at this point. // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. - if (_PyList_AppendTakeRef((PyListObject *)heap, Py_XNewRef(item))) { + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; } From 3f0968925d303caa7dacde714883b65bb486d479 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Thu, 5 Jun 2025 14:53:47 -0700 Subject: [PATCH 08/10] gh-116738: Fix typo --- Lib/test/test_free_threading/test_heapq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index 1f765ac004bf76..fa1838147dfe9c 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -236,7 +236,7 @@ def run_concurrently(self, worker_func, args, nthreads) -> None: barrier = Barrier(nthreads) def wrapper_func(*args): - # Wait for all threadss to reach this point before proceeding. + # Wait for all threads to reach this point before proceeding. barrier.wait() worker_func(*args) From 7ffdb6d091bf3dd3c0c0baeed32b209839224bc0 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Thu, 5 Jun 2025 14:53:47 -0700 Subject: [PATCH 09/10] gh-116738: Use invariant checks from heapq test --- Lib/test/test_free_threading/test_heapq.py | 39 +++++++--------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index fa1838147dfe9c..8c456a9edb3a0a 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -1,13 +1,13 @@ import unittest import heapq -import operator from enum import Enum from threading import Thread, Barrier from random import shuffle, randint from test.support import threading_helper +from test import test_heapq NTHREADS = 10 @@ -21,6 +21,9 @@ class Heap(Enum): @threading_helper.requires_working_threading() class TestHeapq(unittest.TestCase): + def setUp(self): + self.test_heapq = test_heapq.TestHeapPython() + def test_racing_heapify(self): heap = list(range(OBJECT_COUNT)) shuffle(heap) @@ -28,7 +31,7 @@ def test_racing_heapify(self): self.run_concurrently( worker_func=heapq.heapify, args=(heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) + self.test_heapq.check_invariant(heap) def test_racing_heappush(self): heap = [] @@ -40,7 +43,7 @@ def heappush_func(heap): self.run_concurrently( worker_func=heappush_func, args=(heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) + self.test_heapq.check_invariant(heap) def test_racing_heappop(self): heap = self.create_heap(OBJECT_COUNT, Heap.MIN) @@ -80,7 +83,7 @@ def heappushpop_func(heap, pushpop_items): nthreads=NTHREADS, ) self.assertEqual(len(heap), OBJECT_COUNT) - self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) + self.test_heapq.check_invariant(heap) def test_racing_heapreplace(self): heap = self.create_heap(OBJECT_COUNT, Heap.MIN) @@ -96,7 +99,7 @@ def heapreplace_func(heap, replace_items): nthreads=NTHREADS, ) self.assertEqual(len(heap), OBJECT_COUNT) - self.assertTrue(self.is_heap_property_satisfied(heap, Heap.MIN)) + self.test_heapq.check_invariant(heap) def test_racing_heapify_max(self): max_heap = list(range(OBJECT_COUNT)) @@ -105,7 +108,7 @@ def test_racing_heapify_max(self): self.run_concurrently( worker_func=heapq.heapify_max, args=(max_heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) + self.test_heapq.check_max_invariant(max_heap) def test_racing_heappush_max(self): max_heap = [] @@ -117,7 +120,7 @@ def heappush_max_func(max_heap): self.run_concurrently( worker_func=heappush_max_func, args=(max_heap,), nthreads=NTHREADS ) - self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) + self.test_heapq.check_max_invariant(max_heap) def test_racing_heappop_max(self): max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) @@ -157,7 +160,7 @@ def heappushpop_max_func(max_heap, pushpop_items): nthreads=NTHREADS, ) self.assertEqual(len(max_heap), OBJECT_COUNT) - self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) + self.test_heapq.check_max_invariant(max_heap) def test_racing_heapreplace_max(self): max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) @@ -173,25 +176,7 @@ def heapreplace_max_func(max_heap, replace_items): nthreads=NTHREADS, ) self.assertEqual(len(max_heap), OBJECT_COUNT) - self.assertTrue(self.is_heap_property_satisfied(max_heap, Heap.MAX)) - - @staticmethod - def is_heap_property_satisfied(heap, heap_kind): - """ - Check if the heap property is satisfied. - MIN-Heap: The value of a parent node should be less than or equal to the - values of its children. - MAX-Heap: The value of a parent node should be greater than or equal to the - values of its children. - """ - op = operator.le if heap_kind == Heap.MIN else operator.ge - # position 0 has no parent - for pos in range(1, len(heap)): - parent_pos = (pos - 1) >> 1 - if not op(heap[parent_pos], heap[pos]): - return False - - return True + self.test_heapq.check_max_invariant(max_heap) @staticmethod def is_sorted_ascending(lst): From c2225a5599f1b588011704ee896b361f20fc1341 Mon Sep 17 00:00:00 2001 From: alperyoney Date: Mon, 9 Jun 2025 08:28:56 -0700 Subject: [PATCH 10/10] gh-116738: Remove forgotten type hint --- Lib/test/test_free_threading/test_heapq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index 8c456a9edb3a0a..f75fb264c8ac0f 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -214,7 +214,7 @@ def create_random_list(a, b, size): """ return [randint(-a, b) for _ in range(size)] - def run_concurrently(self, worker_func, args, nthreads) -> None: + def run_concurrently(self, worker_func, args, nthreads): """ Run the worker function concurrently in multiple threads. """