From f20855cb4ad9968b21ff31d45e7a66f2ee7b0cfa Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 22 Apr 2025 22:47:56 +0200 Subject: [PATCH 1/3] Make itertools.product and itertools.combinations thread-safe --- .../test_itertools_combinatoric.py | 52 +++++++++++++++++++ Modules/itertoolsmodule.c | 24 ++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_free_threading/test_itertools_combinatoric.py diff --git a/Lib/test/test_free_threading/test_itertools_combinatoric.py b/Lib/test/test_free_threading/test_itertools_combinatoric.py new file mode 100644 index 00000000000000..33abb134b5dd33 --- /dev/null +++ b/Lib/test/test_free_threading/test_itertools_combinatoric.py @@ -0,0 +1,52 @@ +import unittest +from threading import Thread, Barrier +from itertools import combinations, product +from test.support import threading_helper + + +threading_helper.requires_working_threading(module=True) + +def test_concurrent_iteration(iterator, number_of_threads): + barrier = Barrier(number_of_threads) + def iterator_worker(it): + barrier.wait() + while True: + try: + _ = next(it) + except StopIteration: + return + + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=iterator_worker, args=[iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + +class ItertoolsThreading(unittest.TestCase): + + @threading_helper.reap_threads + def test_combinations(self): + number_of_threads = 10 + number_of_iterations = 24 + + for it in range(number_of_iterations): + iterator = combinations((1, 2, 3, 4, 5), 2) + test_concurrent_iteration(iterator, number_of_threads) + + @threading_helper.reap_threads + def test_product(self): + number_of_threads = 10 + number_of_iterations = 24 + + for it in range(number_of_iterations): + iterator = product((1, 2, 3, 4, 5), (10, 20, 30)) + test_concurrent_iteration(iterator, number_of_threads) + + +if __name__ == "__main__": + unittest.main() + \ No newline at end of file diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 943c1e8607b38f..1b0de428253d53 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2081,7 +2081,7 @@ product_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -product_next(PyObject *op) +product_next_lock_held(PyObject *op) { productobject *lz = productobject_CAST(op); PyObject *pool; @@ -2167,6 +2167,16 @@ product_next(PyObject *op) return NULL; } +static PyObject * +product_next(PyObject *op) +{ + PyObject * result; + Py_BEGIN_CRITICAL_SECTION(op); + result = product_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef product_methods[] = { {"__sizeof__", product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2314,7 +2324,7 @@ combinations_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -combinations_next(PyObject *op) +combinations_next_lock_held(PyObject *op) { combinationsobject *co = combinationsobject_CAST(op); PyObject *elem; @@ -2399,6 +2409,16 @@ combinations_next(PyObject *op) return NULL; } +static PyObject * +combinations_next(PyObject *op) +{ + PyObject * result; + Py_BEGIN_CRITICAL_SECTION(op); + result = combinations_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef combinations_methods[] = { {"__sizeof__", combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ From 6f14bd0c15cee9eb5a1a0a8763dfdf27ad01cc37 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:00:30 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst b/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst new file mode 100644 index 00000000000000..a4b4b6d2c23d49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.combinations` and :class:`itertools.product` safe under free-threading. From a204f6db8d4b90f6e131396a0bdc94659d69b025 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 22 Apr 2025 23:02:37 +0200 Subject: [PATCH 3/3] whitespace --- .../test_free_threading/test_itertools_combinatoric.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_free_threading/test_itertools_combinatoric.py b/Lib/test/test_free_threading/test_itertools_combinatoric.py index 33abb134b5dd33..5b3b88deedd121 100644 --- a/Lib/test/test_free_threading/test_itertools_combinatoric.py +++ b/Lib/test/test_free_threading/test_itertools_combinatoric.py @@ -15,7 +15,7 @@ def iterator_worker(it): _ = next(it) except StopIteration: return - + worker_threads = [] for ii in range(number_of_threads): worker_threads.append( @@ -25,9 +25,9 @@ def iterator_worker(it): pass barrier.reset() - + class ItertoolsThreading(unittest.TestCase): - + @threading_helper.reap_threads def test_combinations(self): number_of_threads = 10 @@ -45,8 +45,7 @@ def test_product(self): for it in range(number_of_iterations): iterator = product((1, 2, 3, 4, 5), (10, 20, 30)) test_concurrent_iteration(iterator, number_of_threads) - + if __name__ == "__main__": unittest.main() - \ No newline at end of file