From 6ab2257ec5d5fc5e7b129489f1430b6b973bf0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:13:37 +0200 Subject: [PATCH 01/15] Relax constraints on queue objects for queue handlers. Any object implementing the Queue public API can be used for configuring a QueueHandler. Only the presence of the methods is checked, but not their signature. --- Lib/logging/config.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 95e129ae988c24..299623d84a6e6a 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -791,31 +791,30 @@ def configure_handler(self, config): if '()' not in qspec: raise TypeError('Invalid queue specifier %r' % qspec) config['queue'] = self.configure_custom(dict(qspec)) + elif isinstance(qspec, queue.Queue): + pass else: + # defer importing multiprocessing as much as possible from multiprocessing.queues import Queue as MPQueue - if not isinstance(qspec, (queue.Queue, MPQueue)): - # Safely check if 'qspec' is an instance of Manager.Queue - # / Manager.JoinableQueue - - from multiprocessing import Manager as MM - from multiprocessing.managers import BaseProxy - - # if it's not an instance of BaseProxy, it also can't be - # an instance of Manager.Queue / Manager.JoinableQueue - if isinstance(qspec, BaseProxy): - # Sometimes manager or queue creation might fail - # (e.g. see issue gh-120868). In that case, any - # exception during the creation of these queues will - # propagate up to the caller and be wrapped in a - # `ValueError`, whose cause will indicate the details of - # the failure. - mm = MM() - proxy_queue = mm.Queue() - proxy_joinable_queue = mm.JoinableQueue() - if not isinstance(qspec, (type(proxy_queue), type(proxy_joinable_queue))): - raise TypeError('Invalid queue specifier %r' % qspec) - else: + if not isinstance(qspec, MPQueue): + # Depending on the multiprocessing context, + # we cannot create a Manager instance here + # to get the runtime type of Manager.Queue() + # or Manager.JoinableQueue() (see gh-121723). + # + # Since we only need to support an object + # implementing the Queue API (see gh-120868), + # we only do a protocol check but do not rely + # on typing.runtime_checkable and typing.Protocol + # to reduce import time. + queue_interface = [ + 'empty', 'full', 'get', 'get_nowait', + 'put', 'put_nowait', 'join', 'qsize', + 'task_done', + ] + if not all(callable(getattr(qspec, method, None)) + for method in queue_interface): raise TypeError('Invalid queue specifier %r' % qspec) if 'listener' in config: From 9af323bbcdcb7c9f6153f5dff1cee36596c56047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:13:42 +0200 Subject: [PATCH 02/15] add tests --- Lib/test/test_logging.py | 69 +++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 6d688d4b81bbf4..5cc40266f98576 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -2368,6 +2368,26 @@ class CustomListener(logging.handlers.QueueListener): class CustomQueue(queue.Queue): pass +class CustomQueueProtocol: + def __init__(self, maxsize=0): + self.queue = queue.Queue(maxsize) + + def __getattr__(self, attribute): + queue = object.__getattribute__(self, 'queue') + return getattr(queue, attribute) + +class FuzzyCustomQueueProtocol(CustomQueueProtocol): + # An object implementing the Queue API (incorrect signatures). + # The object will be considered a valid queue class since we + # do not check the signatures (only callability of methods) + # but will NOT be usable in production since a ValueError will + # be raised due to a missing argument. + def empty(self, x): + pass + +class BadCustomQueueProtocol(CustomQueueProtocol): + empty = None + def queueMaker(): return queue.Queue() @@ -3901,18 +3921,16 @@ def do_queuehandler_configuration(self, qspec, lspec): @threading_helper.requires_working_threading() @support.requires_subprocess() def test_config_queue_handler(self): - q = CustomQueue() - dq = { - '()': __name__ + '.CustomQueue', - 'maxsize': 10 - } + qs = [CustomQueue(), CustomQueueProtocol()] + dqs = [{'()': f'{__name__}.{cls}', 'maxsize': 10} + for cls in ['CustomQueue', 'CustomQueueProtocol']] dl = { '()': __name__ + '.listenerMaker', 'arg1': None, 'arg2': None, 'respect_handler_level': True } - qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', dq, q) + qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', *dqs, *qs) lvalues = (None, __name__ + '.CustomListener', dl, CustomListener) for qspec, lspec in itertools.product(qvalues, lvalues): self.do_queuehandler_configuration(qspec, lspec) @@ -3932,15 +3950,21 @@ def test_config_queue_handler(self): @support.requires_subprocess() @patch("multiprocessing.Manager") def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): - # gh-120868 + # gh-120868, gh-121723 from multiprocessing import Queue as MQ q1 = {"()": "queue.Queue", "maxsize": -1} q2 = MQ() q3 = queue.Queue() - - for qspec in (q1, q2, q3): + # FuzzyCustomQueueProtocol pass the checks but will not be usable + # since the signatures are incompatible. Checking the Queue API + # without testing the type of the actual queue is a trade-off + # between usability and the work we need to do in order to safely + # check that the queue object is a valid queue object. + q4 = FuzzyCustomQueueProtocol() + + for qspec in (q1, q2, q3, q4): self.apply_config( { "version": 1, @@ -3956,21 +3980,22 @@ def test_config_queue_handler_does_not_create_multiprocessing_manager(self, mana @patch("multiprocessing.Manager") def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): - # gh-120868 + # gh-120868, gh-121723 - with self.assertRaises(ValueError): - self.apply_config( - { - "version": 1, - "handlers": { - "queue_listener": { - "class": "logging.handlers.QueueHandler", - "queue": object(), + for qspec in [object(), BadCustomQueueProtocol()]: + with self.assertRaises(ValueError): + self.apply_config( + { + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, }, - }, - } - ) - manager.assert_not_called() + } + ) + manager.assert_not_called() @skip_if_tsan_fork @support.requires_subprocess() From 78936a1de3f699e72a3f68b29941f09ebc7b4c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:13:45 +0200 Subject: [PATCH 03/15] update docs --- Doc/library/logging.config.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index dfbf0b1cf2f9ff..6346d8b7d3da00 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -753,9 +753,12 @@ The ``queue`` and ``listener`` keys are optional. If the ``queue`` key is present, the corresponding value can be one of the following: -* An actual instance of :class:`queue.Queue` or a subclass thereof. This is of course - only possible if you are constructing or modifying the configuration dictionary in - code. +* An object implementing the :class:`queue.Queue` public API. For instance, + this may be an actual instance of :class:`queue.Queue` or a subclass thereof, + or a proxy obtained by :meth:`multiprocessing.SyncManager.Queue`. + + This is of course only possible if you are constructing or modifying + the configuration dictionary in code. * A string that resolves to a callable which, when called with no arguments, returns the :class:`queue.Queue` instance to use. That callable could be a From 5fc3d5ac8ab014a8759246e5bd6ef1999a0207e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:13:49 +0200 Subject: [PATCH 04/15] blurb --- .../Library/2024-07-23-10-59-38.gh-issue-121723.iJEf7e.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-07-23-10-59-38.gh-issue-121723.iJEf7e.rst diff --git a/Misc/NEWS.d/next/Library/2024-07-23-10-59-38.gh-issue-121723.iJEf7e.rst b/Misc/NEWS.d/next/Library/2024-07-23-10-59-38.gh-issue-121723.iJEf7e.rst new file mode 100644 index 00000000000000..cabb4024fb10f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-10-59-38.gh-issue-121723.iJEf7e.rst @@ -0,0 +1,3 @@ +Make :func:`logging.config.dictConfig` accept any object implementing the +Queue public API. See the :ref:`queue configuration ` +section for details. Patch by Bénédikt Tran. From 04c78135bbe71340e743226d979c4eb13f9700f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:19:44 +0200 Subject: [PATCH 05/15] refactor into a helper --- Lib/logging/config.py | 54 +++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 299623d84a6e6a..84af6c9565906c 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -497,6 +497,33 @@ def as_tuple(self, value): value = tuple(value) return value +def _is_queue_like_object(obj): + """Check that *obj* implements the Queue API.""" + if isinstance(obj, queue.Queue): + return True + # defer importing multiprocessing as much as possible + from multiprocessing.queues import Queue as MPQueue + if isinstance(obj, MPQueue): + return True + # Depending on the multiprocessing start context, we cannot create + # a multiprocessing.managers.BaseManager instance 'mm' to get the + # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-121723). + # + # Since we only need an object implementing the Queue API, we only + # do a protocol check without relying on typing.runtime_checkable + # and typing.Protocol to reduce import time (see gh-121723). + # + # Ideally, we would have wanted to simply use strict type checking + # instead of protocol-based type checking since this approach does + # not consider incompatible signatures. + queue_interface = [ + 'empty', 'full', 'get', 'get_nowait', + 'put', 'put_nowait', 'join', 'qsize', + 'task_done', + ] + return all(callable(getattr(obj, method, None)) + for method in queue_interface) + class DictConfigurator(BaseConfigurator): """ Configure logging using a dictionary-like object to describe the @@ -791,31 +818,8 @@ def configure_handler(self, config): if '()' not in qspec: raise TypeError('Invalid queue specifier %r' % qspec) config['queue'] = self.configure_custom(dict(qspec)) - elif isinstance(qspec, queue.Queue): - pass - else: - # defer importing multiprocessing as much as possible - from multiprocessing.queues import Queue as MPQueue - - if not isinstance(qspec, MPQueue): - # Depending on the multiprocessing context, - # we cannot create a Manager instance here - # to get the runtime type of Manager.Queue() - # or Manager.JoinableQueue() (see gh-121723). - # - # Since we only need to support an object - # implementing the Queue API (see gh-120868), - # we only do a protocol check but do not rely - # on typing.runtime_checkable and typing.Protocol - # to reduce import time. - queue_interface = [ - 'empty', 'full', 'get', 'get_nowait', - 'put', 'put_nowait', 'join', 'qsize', - 'task_done', - ] - if not all(callable(getattr(qspec, method, None)) - for method in queue_interface): - raise TypeError('Invalid queue specifier %r' % qspec) + elif not _is_queue_like_object(qspec): + raise TypeError('Invalid queue specifier %r' % qspec) if 'listener' in config: lspec = config['listener'] From e5cdb1c7a307869b6df6f4412a011ee7102e2e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:50:40 +0200 Subject: [PATCH 06/15] add regression test --- Lib/test/test_logging.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 5cc40266f98576..a1c890fa39c126 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3997,6 +3997,40 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man ) manager.assert_not_called() + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build") + def test_config_queue_handler_multiprocessing_context(self): + # regression test for gh-121723 + for startmethod in ['fork', 'spawn', 'forkserver']: + with self.subTest(startmethod=startmethod): + ctx = multiprocessing.get_context(startmethod) + with ctx.Manager() as manager: + q = manager.Queue() + records = [] + # use 1 process and 1 task per child to put 1 record + with ctx.Pool(1, initializer=self._mpinit_issue121723, + initargs=(q, "text"), maxtasksperchild=1): + records.append(q.get(timeout=0.5)) + self.assertTrue(q.empty()) + self.assertEqual(len(records), 1) + + @staticmethod + def _mpinit_issue121723(q, msg): + # static method for pickling support + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': True, + 'handlers': { + 'log_to_parent': { + 'class': 'logging.handlers.QueueHandler', + 'queue': q + } + }, + 'root': {'handlers': ['log_to_parent'], 'level': 'DEBUG'} + }) + logging.getLogger().info(msg) + @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): From b770787e1f634b4fff39f77954e3708aa2c7e7df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 24 Jul 2024 13:27:28 +0200 Subject: [PATCH 07/15] fix start methods selection on MS-Windows --- Lib/test/test_logging.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index a1c890fa39c126..3cf4fc90b189ef 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4002,9 +4002,13 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man @unittest.skipUnless(support.Py_DEBUG, "requires a debug build") def test_config_queue_handler_multiprocessing_context(self): # regression test for gh-121723 - for startmethod in ['fork', 'spawn', 'forkserver']: - with self.subTest(startmethod=startmethod): - ctx = multiprocessing.get_context(startmethod) + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) with ctx.Manager() as manager: q = manager.Queue() records = [] From 65a89877d676b2dfb984d47e2a0624ee1a13b0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 24 Jul 2024 13:52:10 +0200 Subject: [PATCH 08/15] fixup refs --- Doc/library/logging.config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 6346d8b7d3da00..0ddbc1a5f88048 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -755,7 +755,7 @@ If the ``queue`` key is present, the corresponding value can be one of the follo * An object implementing the :class:`queue.Queue` public API. For instance, this may be an actual instance of :class:`queue.Queue` or a subclass thereof, - or a proxy obtained by :meth:`multiprocessing.SyncManager.Queue`. + or a proxy obtained by :meth:`multiprocessing.managers.SyncManager.Queue`. This is of course only possible if you are constructing or modifying the configuration dictionary in code. From 7729fb6d4aa8f735932df47b29400802d4a65443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:43:11 +0200 Subject: [PATCH 09/15] increase the timeout (maybe it works like this?) --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 3cf4fc90b189ef..4cd6bf36e7e793 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4015,7 +4015,7 @@ def test_config_queue_handler_multiprocessing_context(self): # use 1 process and 1 task per child to put 1 record with ctx.Pool(1, initializer=self._mpinit_issue121723, initargs=(q, "text"), maxtasksperchild=1): - records.append(q.get(timeout=0.5)) + records.append(q.get(timeout=2)) self.assertTrue(q.empty()) self.assertEqual(len(records), 1) From 57d7190837d3f7e538671cd3cd2b0477f7e53e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 28 Jul 2024 10:44:34 +0200 Subject: [PATCH 10/15] fixup! typos --- Lib/logging/config.py | 9 +++++---- Lib/test/test_logging.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 84af6c9565906c..f7f103d64a8350 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -501,21 +501,22 @@ def _is_queue_like_object(obj): """Check that *obj* implements the Queue API.""" if isinstance(obj, queue.Queue): return True + # defer importing multiprocessing as much as possible from multiprocessing.queues import Queue as MPQueue if isinstance(obj, MPQueue): return True # Depending on the multiprocessing start context, we cannot create # a multiprocessing.managers.BaseManager instance 'mm' to get the - # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-121723). + # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819). # # Since we only need an object implementing the Queue API, we only - # do a protocol check without relying on typing.runtime_checkable + # do a protocol check, but we do not use typing.runtime_checkable() # and typing.Protocol to reduce import time (see gh-121723). # # Ideally, we would have wanted to simply use strict type checking - # instead of protocol-based type checking since this approach does - # not consider incompatible signatures. + # instead of a protocol-based type checking since the latetr does + # not check the method signatures. queue_interface = [ 'empty', 'full', 'get', 'get_nowait', 'put', 'put_nowait', 'join', 'qsize', diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 4cd6bf36e7e793..1993e0ddbd739b 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -2380,7 +2380,7 @@ class FuzzyCustomQueueProtocol(CustomQueueProtocol): # An object implementing the Queue API (incorrect signatures). # The object will be considered a valid queue class since we # do not check the signatures (only callability of methods) - # but will NOT be usable in production since a ValueError will + # but will NOT be usable in production since a TypeError will # be raised due to a missing argument. def empty(self, x): pass @@ -3957,11 +3957,11 @@ def test_config_queue_handler_does_not_create_multiprocessing_manager(self, mana q1 = {"()": "queue.Queue", "maxsize": -1} q2 = MQ() q3 = queue.Queue() - # FuzzyCustomQueueProtocol pass the checks but will not be usable + # FuzzyCustomQueueProtocol passes the checks but will not be usable # since the signatures are incompatible. Checking the Queue API # without testing the type of the actual queue is a trade-off # between usability and the work we need to do in order to safely - # check that the queue object is a valid queue object. + # check that the queue object correctly implements the API. q4 = FuzzyCustomQueueProtocol() for qspec in (q1, q2, q3, q4): From e4800c4ae204a0a46ffafc107babdd012e70f4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 28 Jul 2024 10:45:56 +0200 Subject: [PATCH 11/15] fixup! typos --- Lib/logging/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/logging/config.py b/Lib/logging/config.py index f7f103d64a8350..611d026e33d4f1 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -515,7 +515,7 @@ def _is_queue_like_object(obj): # and typing.Protocol to reduce import time (see gh-121723). # # Ideally, we would have wanted to simply use strict type checking - # instead of a protocol-based type checking since the latetr does + # instead of a protocol-based type checking since the latter does # not check the method signatures. queue_interface = [ 'empty', 'full', 'get', 'get_nowait', From bd128936e383dddd864dc7eacb08b95dec3eb8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 28 Jul 2024 10:46:31 +0200 Subject: [PATCH 12/15] reduce line numbers --- Lib/logging/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 611d026e33d4f1..3781cb1aeb9ae2 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -501,7 +501,6 @@ def _is_queue_like_object(obj): """Check that *obj* implements the Queue API.""" if isinstance(obj, queue.Queue): return True - # defer importing multiprocessing as much as possible from multiprocessing.queues import Queue as MPQueue if isinstance(obj, MPQueue): From 70d69e5b721c9d1255002b8dddf1d6c4bc0a2717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 28 Jul 2024 10:49:46 +0200 Subject: [PATCH 13/15] renaming --- Lib/test/test_logging.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1993e0ddbd739b..d3fde52291b0d7 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -2376,7 +2376,7 @@ def __getattr__(self, attribute): queue = object.__getattribute__(self, 'queue') return getattr(queue, attribute) -class FuzzyCustomQueueProtocol(CustomQueueProtocol): +class CustomQueueFakeProtocol(CustomQueueProtocol): # An object implementing the Queue API (incorrect signatures). # The object will be considered a valid queue class since we # do not check the signatures (only callability of methods) @@ -2385,7 +2385,7 @@ class FuzzyCustomQueueProtocol(CustomQueueProtocol): def empty(self, x): pass -class BadCustomQueueProtocol(CustomQueueProtocol): +class CustomQueueWrongProtocol(CustomQueueProtocol): empty = None def queueMaker(): @@ -3962,7 +3962,7 @@ def test_config_queue_handler_does_not_create_multiprocessing_manager(self, mana # without testing the type of the actual queue is a trade-off # between usability and the work we need to do in order to safely # check that the queue object correctly implements the API. - q4 = FuzzyCustomQueueProtocol() + q4 = CustomQueueFakeProtocol() for qspec in (q1, q2, q3, q4): self.apply_config( @@ -3982,7 +3982,7 @@ def test_config_queue_handler_does_not_create_multiprocessing_manager(self, mana def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): # gh-120868, gh-121723 - for qspec in [object(), BadCustomQueueProtocol()]: + for qspec in [object(), CustomQueueWrongProtocol()]: with self.assertRaises(ValueError): self.apply_config( { @@ -3999,7 +3999,8 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man @skip_if_tsan_fork @support.requires_subprocess() - @unittest.skipUnless(support.Py_DEBUG, "requires a debug build") + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + "assertions in multiprocessing") def test_config_queue_handler_multiprocessing_context(self): # regression test for gh-121723 if support.MS_WINDOWS: @@ -4020,7 +4021,7 @@ def test_config_queue_handler_multiprocessing_context(self): self.assertEqual(len(records), 1) @staticmethod - def _mpinit_issue121723(q, msg): + def _mpinit_issue121723(qspec, message_to_log): # static method for pickling support logging.config.dictConfig({ 'version': 1, @@ -4028,12 +4029,13 @@ def _mpinit_issue121723(q, msg): 'handlers': { 'log_to_parent': { 'class': 'logging.handlers.QueueHandler', - 'queue': q + 'queue': qspec } }, 'root': {'handlers': ['log_to_parent'], 'level': 'DEBUG'} }) - logging.getLogger().info(msg) + # log a message (this creates a record put in the queue) + logging.getLogger().info(message_to_log) @skip_if_tsan_fork @support.requires_subprocess() From d6b6ef33a627e2fb29c2ff9be627dc09be5bb249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:25:25 +0200 Subject: [PATCH 14/15] fixup! refs --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d3fde52291b0d7..4fa46ff7ad304a 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3957,7 +3957,7 @@ def test_config_queue_handler_does_not_create_multiprocessing_manager(self, mana q1 = {"()": "queue.Queue", "maxsize": -1} q2 = MQ() q3 = queue.Queue() - # FuzzyCustomQueueProtocol passes the checks but will not be usable + # CustomQueueFakeProtocol passes the checks but will not be usable # since the signatures are incompatible. Checking the Queue API # without testing the type of the actual queue is a trade-off # between usability and the work we need to do in order to safely From d2bfdc25b02f19b5179d9195b9266d243e4439dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 1 Aug 2024 09:54:41 +0200 Subject: [PATCH 15/15] increase timeout (again) --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 4fa46ff7ad304a..49523756e115c6 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4016,7 +4016,7 @@ def test_config_queue_handler_multiprocessing_context(self): # use 1 process and 1 task per child to put 1 record with ctx.Pool(1, initializer=self._mpinit_issue121723, initargs=(q, "text"), maxtasksperchild=1): - records.append(q.get(timeout=2)) + records.append(q.get(timeout=60)) self.assertTrue(q.empty()) self.assertEqual(len(records), 1)