From 678f29e5219f28c54567a5d45e13ccd053b55946 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 18 Jun 2025 21:17:36 +0200 Subject: [PATCH 1/6] Make itertools.chain thread-safe --- .../test_free_threading/test_itertools.py | 31 ++++++++++++++++++- Modules/itertoolsmodule.c | 14 +++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py index b8663ade1d4aca..a170241cc0269f 100644 --- a/Lib/test/test_free_threading/test_itertools.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -1,6 +1,6 @@ import unittest from threading import Thread, Barrier -from itertools import batched, cycle +from itertools import batched, chain, cycle from test.support import threading_helper @@ -62,6 +62,35 @@ def work(it): barrier.reset() + @threading_helper.reap_threads + def test_chain(self): + number_of_threads = 6 + number_of_iterations = 20 + + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + while True: + try: + _ = next(it) + except StopIteration: + break + + + data = [(1, )] * 200 + for it in range(number_of_iterations): + chain_iterator = chain(*data) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[chain_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + if __name__ == "__main__": unittest.main() diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 2003546ce84cef..6943d4d87f8ed4 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1880,8 +1880,8 @@ chain_traverse(PyObject *op, visitproc visit, void *arg) return 0; } -static PyObject * -chain_next(PyObject *op) +static inline PyObject * +chain_next_lock_held(PyObject *op) { chainobject *lz = chainobject_CAST(op); PyObject *item; @@ -1919,6 +1919,16 @@ chain_next(PyObject *op) return NULL; } +static PyObject * +chain_next(PyObject *op) +{ + PyObject * result; + Py_BEGIN_CRITICAL_SECTION(op); + result = chain_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + PyDoc_STRVAR(chain_doc, "chain(*iterables)\n\ --\n\ From 6d7c934125831d49777ddab9f2d13e548f8c2dde Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 19:25:39 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst diff --git a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst new file mode 100644 index 00000000000000..75bba0240cd020 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.chain` safe under free-threading. From 07881aca9f463e0cb83122d5b946c866a0577685 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 19 Jun 2025 20:57:21 +0200 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Peter Bierma --- .../next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst | 2 +- Modules/itertoolsmodule.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst index 75bba0240cd020..6f395024a9e179 100644 --- a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst +++ b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst @@ -1 +1 @@ -Make concurrent iterations over :class:`itertools.chain` safe under free-threading. +Make concurrent iterations over :class:`itertools.chain` safe under :term:`free threading`. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6943d4d87f8ed4..e6536c250109b1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1922,7 +1922,7 @@ chain_next_lock_held(PyObject *op) static PyObject * chain_next(PyObject *op) { - PyObject * result; + PyObject *result; Py_BEGIN_CRITICAL_SECTION(op); result = chain_next_lock_held(op); Py_END_CRITICAL_SECTION() From 79699bb6b2643e7b2d35c04bb914b730ecc380ce Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 19 Jun 2025 21:35:19 +0200 Subject: [PATCH 4/6] refactor --- Modules/itertoolsmodule.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index e6536c250109b1..d047d5f1f983ee 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1881,11 +1881,12 @@ chain_traverse(PyObject *op, visitproc visit, void *arg) } static inline PyObject * -chain_next_lock_held(PyObject *op) +chain_next(PyObject *op) { chainobject *lz = chainobject_CAST(op); - PyObject *item; + PyObject *item = NULL; + Py_BEGIN_CRITICAL_SECTION(op); /* lz->source is the iterator of iterables. If it's NULL, we've already * consumed them all. lz->active is the current iterator. If it's NULL, * we should grab a new one from lz->source. */ @@ -1894,41 +1895,38 @@ chain_next_lock_held(PyObject *op) PyObject *iterable = PyIter_Next(lz->source); if (iterable == NULL) { Py_CLEAR(lz->source); - return NULL; /* no more input sources */ + goto exit; /* no more input sources */ } lz->active = PyObject_GetIter(iterable); Py_DECREF(iterable); if (lz->active == NULL) { Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + goto exit; /* input not iterable */ } } item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; + if (item != NULL) { + goto exit; + } if (PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); - else - return NULL; /* input raised an exception */ + else { + goto exit; /* input raised an exception */ + } } /* lz->active is consumed, try with the next iterable. */ Py_CLEAR(lz->active); } /* Everything had been consumed already. */ - return NULL; -} -static PyObject * -chain_next(PyObject *op) -{ - PyObject *result; - Py_BEGIN_CRITICAL_SECTION(op); - result = chain_next_lock_held(op); +exit: Py_END_CRITICAL_SECTION() - return result; + + return item; } + PyDoc_STRVAR(chain_doc, "chain(*iterables)\n\ --\n\ From 0ad0b1531cdbefa2ad73d482454ebda10f2e6542 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 19 Jun 2025 21:37:23 +0200 Subject: [PATCH 5/6] reduce diff --- .../test_free_threading/test_itertools.py | 30 +------------------ Modules/itertoolsmodule.c | 2 +- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py index a170241cc0269f..984e9488a3eff9 100644 --- a/Lib/test/test_free_threading/test_itertools.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -1,6 +1,6 @@ import unittest from threading import Thread, Barrier -from itertools import batched, chain, cycle +from itertools import batched, cycle from test.support import threading_helper @@ -62,34 +62,6 @@ def work(it): barrier.reset() - @threading_helper.reap_threads - def test_chain(self): - number_of_threads = 6 - number_of_iterations = 20 - - barrier = Barrier(number_of_threads) - def work(it): - barrier.wait() - while True: - try: - _ = next(it) - except StopIteration: - break - - - data = [(1, )] * 200 - for it in range(number_of_iterations): - chain_iterator = chain(*data) - worker_threads = [] - for ii in range(number_of_threads): - worker_threads.append( - Thread(target=work, args=[chain_iterator])) - - with threading_helper.start_threads(worker_threads): - pass - - barrier.reset() - if __name__ == "__main__": diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d047d5f1f983ee..28eb1628384ac1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1880,7 +1880,7 @@ chain_traverse(PyObject *op, visitproc visit, void *arg) return 0; } -static inline PyObject * +static PyObject * chain_next(PyObject *op) { chainobject *lz = chainobject_CAST(op); From e804d2fe2750bf1fc5862ab30d8a6f5168e8feb1 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 19 Jun 2025 21:37:58 +0200 Subject: [PATCH 6/6] reduce diff --- Lib/test/test_free_threading/test_itertools.py | 1 - .../next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst | 1 - 2 files changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py index 984e9488a3eff9..b8663ade1d4aca 100644 --- a/Lib/test/test_free_threading/test_itertools.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -63,6 +63,5 @@ def work(it): barrier.reset() - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst deleted file mode 100644 index 6f395024a9e179..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst +++ /dev/null @@ -1 +0,0 @@ -Make concurrent iterations over :class:`itertools.chain` safe under :term:`free threading`.