From f40ba3fedcc6682344fcedd517e3ddfcfd9b2d86 Mon Sep 17 00:00:00 2001 From: Varun Nuthalapati Date: Mon, 1 Jun 2026 21:37:22 -0700 Subject: [PATCH 1/4] fix(logging): prevent log file race condition under parallel attempts When multiprocessing.Pool forks worker processes they inherit the parent's open FileHandler file descriptors. Concurrent writes (or reentrant flushes triggered by third-party library destructors such as openai/httpcore closing connections in __del__) on those shared handles cause a RuntimeError in Python 3.13+. Add a _worker_logging_init pool initializer that closes all inherited handlers and re-opens a fresh FileHandler in each worker, giving every worker its own private file descriptor. The log file path is read from the GARAK_LOG_FILE env var that garak/__init__.py already sets before spawning any children. Fixes #1355 Signed-off-by: Varun Nuthalapati --- garak/probes/base.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/garak/probes/base.py b/garak/probes/base.py index 4acb42724..b1f1a6b86 100644 --- a/garak/probes/base.py +++ b/garak/probes/base.py @@ -27,6 +27,37 @@ import garak.resources.theme +def _worker_logging_init(): + """Pool initializer: close inherited log file handles and re-configure logging. + + When :mod:`multiprocessing` spawns or forks worker processes the parent's + open :class:`logging.FileHandler` file descriptors are inherited. Concurrent + writes through those shared handles can trigger a reentrant-flush + ``RuntimeError`` (see GitHub issue #1355). This initializer runs once inside + each worker, closes every inherited handler, and re-applies the same + ``basicConfig`` the main process used, so each worker gets its own private + file descriptor pointing at the same log file. + """ + import os + import logging + + root = logging.getLogger() + for handler in root.handlers[:]: + try: + handler.close() + except Exception: + pass + root.removeHandler(handler) + + log_filename = os.environ.get("GARAK_LOG_FILE") + if log_filename: + logging.basicConfig( + filename=log_filename, + level=logging.DEBUG, + format="%(asctime)s %(levelname)s %(message)s", + ) + + class Probe(Configurable): """Base class for objects that define and execute LLM evaluations""" @@ -332,7 +363,7 @@ def _execute_all(self, attempts) -> Iterable[garak.attempt.Attempt]: attempt_pool = None try: - attempt_pool = Pool(pool_size) + attempt_pool = Pool(pool_size, initializer=_worker_logging_init) for result in attempt_pool.imap_unordered( self._execute_attempt, attempts ): From 727728ba53a25573e94ae36e851c56b54f3cccfa Mon Sep 17 00:00:00 2001 From: Varun Nuthalapati Date: Thu, 4 Jun 2026 21:06:08 -0700 Subject: [PATCH 2/4] test(logging): add unit and integration tests for _worker_logging_init Adds tests/probes/test_probes_base_parallel_logging.py to reproduce and verify the fix for #1355. Four unit tests cover handler teardown, file handler creation from GARAK_LOG_FILE, absence of handler when env is unset, and idempotency. Two integration tests spawn a real Pool to confirm workers log concurrently without RuntimeError and that each worker opens its own private file descriptor rather than sharing the parent's inherited fd. Signed-off-by: Varun Nuthalapati Co-Authored-By: Claude Sonnet 4.6 --- .../test_probes_base_parallel_logging.py | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 tests/probes/test_probes_base_parallel_logging.py diff --git a/tests/probes/test_probes_base_parallel_logging.py b/tests/probes/test_probes_base_parallel_logging.py new file mode 100644 index 000000000..c7ba1a59c --- /dev/null +++ b/tests/probes/test_probes_base_parallel_logging.py @@ -0,0 +1,247 @@ +# SPDX-FileCopyrightText: Portions Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the _worker_logging_init pool initializer that prevents log-file +race conditions under parallel probe execution (issue #1355).""" + +import logging +import os +import tempfile +from multiprocessing import Pool +from unittest.mock import MagicMock + +import pytest + +from garak.probes.base import _worker_logging_init + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _log_from_worker(log_file: str) -> int: + """Target function run inside a Pool worker. + + Returns the file-descriptor number of the root logger's FileHandler so the + parent can verify each worker got its own private fd. + """ + for i in range(50): + logging.debug("worker log line %d", i) + root = logging.getLogger() + for h in root.handlers: + if isinstance(h, logging.FileHandler): + return h.stream.fileno() + return -1 + + +# --------------------------------------------------------------------------- +# Unit tests for _worker_logging_init +# --------------------------------------------------------------------------- + + +def test_worker_init_clears_inherited_handlers(): + """_worker_logging_init must remove all handlers the parent process had.""" + root = logging.getLogger() + original_handlers = root.handlers[:] + + mock_handler = MagicMock(spec=logging.Handler) + root.addHandler(mock_handler) + assert mock_handler in root.handlers + + try: + _worker_logging_init() + assert mock_handler not in root.handlers, ( + "_worker_logging_init should remove all inherited handlers" + ) + finally: + # Restore whatever was there before (mock already removed by init) + for h in root.handlers[:]: + root.removeHandler(h) + for h in original_handlers: + root.addHandler(h) + + +def test_worker_init_opens_file_handler_when_env_set(): + """_worker_logging_init must install a FileHandler pointing at GARAK_LOG_FILE.""" + root = logging.getLogger() + original_handlers = root.handlers[:] + + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as f: + log_path = f.name + + old_env = os.environ.get("GARAK_LOG_FILE") + os.environ["GARAK_LOG_FILE"] = log_path + + try: + # Clear handlers so basicConfig will take effect + for h in root.handlers[:]: + root.removeHandler(h) + + _worker_logging_init() + + file_handlers = [ + h for h in root.handlers if isinstance(h, logging.FileHandler) + ] + assert len(file_handlers) >= 1, ( + "_worker_logging_init should add a FileHandler when GARAK_LOG_FILE is set" + ) + assert any(h.baseFilename == log_path for h in file_handlers), ( + "FileHandler should point at GARAK_LOG_FILE" + ) + finally: + for h in root.handlers[:]: + if isinstance(h, logging.FileHandler): + h.close() + root.removeHandler(h) + for h in original_handlers: + root.addHandler(h) + if old_env is None: + os.environ.pop("GARAK_LOG_FILE", None) + else: + os.environ["GARAK_LOG_FILE"] = old_env + os.unlink(log_path) + + +def test_worker_init_no_file_handler_when_env_unset(): + """_worker_logging_init must not add a FileHandler when GARAK_LOG_FILE is absent.""" + root = logging.getLogger() + original_handlers = root.handlers[:] + + old_env = os.environ.pop("GARAK_LOG_FILE", None) + + try: + for h in root.handlers[:]: + root.removeHandler(h) + + _worker_logging_init() + + file_handlers = [ + h for h in root.handlers if isinstance(h, logging.FileHandler) + ] + assert file_handlers == [], ( + "_worker_logging_init should not add a FileHandler when GARAK_LOG_FILE is not set" + ) + finally: + for h in root.handlers[:]: + if isinstance(h, logging.FileHandler): + h.close() + root.removeHandler(h) + for h in original_handlers: + root.addHandler(h) + if old_env is not None: + os.environ["GARAK_LOG_FILE"] = old_env + + +def test_worker_init_is_idempotent(): + """Calling _worker_logging_init twice must not accumulate extra handlers.""" + root = logging.getLogger() + original_handlers = root.handlers[:] + + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as f: + log_path = f.name + + old_env = os.environ.get("GARAK_LOG_FILE") + os.environ["GARAK_LOG_FILE"] = log_path + + try: + for h in root.handlers[:]: + root.removeHandler(h) + + _worker_logging_init() + count_after_first = len(root.handlers) + _worker_logging_init() + count_after_second = len(root.handlers) + + assert count_after_second <= count_after_first, ( + "Calling _worker_logging_init twice must not add duplicate handlers" + ) + finally: + for h in root.handlers[:]: + if isinstance(h, logging.FileHandler): + h.close() + root.removeHandler(h) + for h in original_handlers: + root.addHandler(h) + if old_env is None: + os.environ.pop("GARAK_LOG_FILE", None) + else: + os.environ["GARAK_LOG_FILE"] = old_env + os.unlink(log_path) + + +# --------------------------------------------------------------------------- +# Integration test: parallel workers log without RuntimeError +# --------------------------------------------------------------------------- + + +def test_parallel_workers_log_without_error(): + """Pool workers initialised with _worker_logging_init must not raise + RuntimeError due to shared-fd log writes (regression for issue #1355).""" + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as f: + log_path = f.name + + root = logging.getLogger() + parent_handler = logging.FileHandler(log_path) + root.addHandler(parent_handler) + + old_env = os.environ.get("GARAK_LOG_FILE") + os.environ["GARAK_LOG_FILE"] = log_path + + errors = [] + try: + with Pool(processes=4, initializer=_worker_logging_init) as pool: + results = pool.map(_log_from_worker, [log_path] * 8) + # All workers should have returned a valid fd number + assert all(isinstance(r, int) for r in results), ( + "All worker tasks should complete and return an integer fd" + ) + except RuntimeError as exc: + errors.append(str(exc)) + finally: + parent_handler.close() + root.removeHandler(parent_handler) + if old_env is None: + os.environ.pop("GARAK_LOG_FILE", None) + else: + os.environ["GARAK_LOG_FILE"] = old_env + os.unlink(log_path) + + assert not errors, ( + f"Parallel logging raised RuntimeError (race condition): {errors}" + ) + + +def test_parallel_worker_fds_are_independent(): + """Each Pool worker should open its own file descriptor for the log file, + not share the parent's inherited fd (regression for issue #1355).""" + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as f: + log_path = f.name + + root = logging.getLogger() + parent_handler = logging.FileHandler(log_path) + root.addHandler(parent_handler) + parent_fd = parent_handler.stream.fileno() + + old_env = os.environ.get("GARAK_LOG_FILE") + os.environ["GARAK_LOG_FILE"] = log_path + + try: + with Pool(processes=2, initializer=_worker_logging_init) as pool: + worker_fds = pool.map(_log_from_worker, [log_path] * 2) + + # Workers that got a FileHandler should report a valid fd + valid_fds = [fd for fd in worker_fds if fd != -1] + if valid_fds: + assert all(fd != parent_fd for fd in valid_fds), ( + "Worker file descriptors must differ from the parent's fd — " + "shared fds cause the reentrant-flush race condition" + ) + finally: + parent_handler.close() + root.removeHandler(parent_handler) + if old_env is None: + os.environ.pop("GARAK_LOG_FILE", None) + else: + os.environ["GARAK_LOG_FILE"] = old_env + os.unlink(log_path) From 7aa5f8611fcd1a4e2cef0586e6e61cfbafa793e7 Mon Sep 17 00:00:00 2001 From: Varun Nuthalapati Date: Thu, 11 Jun 2026 09:08:45 -0700 Subject: [PATCH 3/4] test(probes): add regression test demonstrating parallel logging initializer fix Signed-off-by: Varun Nuthalapati --- .../test_probes_base_parallel_logging.py | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/probes/test_probes_base_parallel_logging.py b/tests/probes/test_probes_base_parallel_logging.py index c7ba1a59c..92ad18dce 100644 --- a/tests/probes/test_probes_base_parallel_logging.py +++ b/tests/probes/test_probes_base_parallel_logging.py @@ -8,10 +8,13 @@ import os import tempfile from multiprocessing import Pool -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest +import garak._config +import garak._plugins +import garak.attempt from garak.probes.base import _worker_logging_init @@ -245,3 +248,47 @@ def test_parallel_worker_fds_are_independent(): else: os.environ["GARAK_LOG_FILE"] = old_env os.unlink(log_path) + + +# --------------------------------------------------------------------------- +# Regression test: Probe._execute_all must wire up the initializer +# --------------------------------------------------------------------------- + + +def test_execute_all_passes_logging_initializer_to_pool(): + """Probe._execute_all must construct its worker Pool with + initializer=_worker_logging_init. Without this wiring, forked workers + inherit the parent's log FileHandler and share its file descriptor, + which can raise a reentrant-flush RuntimeError under concurrent writes + (issue #1355). This test fails if the initializer argument is removed.""" + with open(os.devnull, "w+", encoding="utf-8") as fh: + garak._config.load_base_config() + garak._config.transient.reportfile = fh + + p = garak._plugins.load_plugin( + "probes.test.Test", config_root=garak._config + ) + g = garak._plugins.load_plugin( + "generators.test.Repeat", config_root=garak._config + ) + p.generator = g + p.parallel_attempts = 2 + + attempts = [ + garak.attempt.Attempt(prompt=garak.attempt.Message("test one")), + garak.attempt.Attempt(prompt=garak.attempt.Message("test two")), + ] + + real_pool = Pool + + with patch("multiprocessing.Pool", wraps=real_pool) as mock_pool: + p._execute_all(attempts) + + garak._config.transient.reportfile = None + + assert mock_pool.called, "Probe._execute_all should use a worker Pool" + _, kwargs = mock_pool.call_args + assert kwargs.get("initializer") is _worker_logging_init, ( + "Probe._execute_all must pass _worker_logging_init as the Pool " + "initializer to avoid shared log file descriptors in workers" + ) From 76d00035230fc102964fdb7d5d73bd50994b76e0 Mon Sep 17 00:00:00 2001 From: Varun Nuthalapati Date: Mon, 22 Jun 2026 20:07:04 -0700 Subject: [PATCH 4/4] test: add regression test demonstrating worker fd isolation for log race fix Address review feedback from jmartin-tech on PR #1827: the existing tests only asserted that _worker_logging_init was wired into Pool(...), without exercising the actual fd-sharing bug from issue #1355. Add test_worker_inherits_parent_fd_without_initializer_but_not_with_it, which runs the same worker function through a fork-context Pool twice: once with no initializer (the pre-fix behavior, where a forked worker inherits the parent's exact FileHandler fd) and once with _worker_logging_init (the fix). The test asserts the worker's reported fd equals the parent's fd in the first case and differs in the second, so it fails if the initializer is removed or not passed to Pool. The test is skipped on platforms without a fork-capable multiprocessing context (e.g. Windows, which only supports spawn) since fd inheritance cannot be reproduced there; it runs for real on fork-based CI. Signed-off-by: Varun Nuthalapati --- .../test_probes_base_parallel_logging.py | 106 +++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/tests/probes/test_probes_base_parallel_logging.py b/tests/probes/test_probes_base_parallel_logging.py index 92ad18dce..0da9a3f72 100644 --- a/tests/probes/test_probes_base_parallel_logging.py +++ b/tests/probes/test_probes_base_parallel_logging.py @@ -5,6 +5,7 @@ race conditions under parallel probe execution (issue #1355).""" import logging +import multiprocessing import os import tempfile from multiprocessing import Pool @@ -237,7 +238,7 @@ def test_parallel_worker_fds_are_independent(): valid_fds = [fd for fd in worker_fds if fd != -1] if valid_fds: assert all(fd != parent_fd for fd in valid_fds), ( - "Worker file descriptors must differ from the parent's fd — " + "Worker file descriptors must differ from the parent's fd - " "shared fds cause the reentrant-flush race condition" ) finally: @@ -250,6 +251,107 @@ def test_parallel_worker_fds_are_independent(): os.unlink(log_path) +# --------------------------------------------------------------------------- +# Deterministic fork-based regression test: this is the test that actually +# exercises the bug fixed by this PR. It must FAIL if _worker_logging_init is +# bypassed (simulating the pre-fix code) and PASS with it wired up. +# --------------------------------------------------------------------------- + + +_FORK_AVAILABLE = "fork" in multiprocessing.get_all_start_methods() + + +@pytest.mark.skipif( + not _FORK_AVAILABLE, + reason=( + "This test requires the 'fork' multiprocessing start method to " + "reproduce fd inheritance; platforms that only support 'spawn' " + "(e.g. Windows) never inherit the parent's open file handles, so " + "the underlying bug cannot manifest there." + ), +) +def test_worker_inherits_parent_fd_without_initializer_but_not_with_it(): + """Demonstrates the actual bug fixed by this PR (issue #1355). + + Root cause: when :mod:`multiprocessing` *forks* worker processes, each + worker inherits the parent's exact, shared file descriptor for any open + ``logging.FileHandler``. Concurrent writes -- or reentrant flushes + triggered by third-party destructors -- on that shared fd can raise + ``RuntimeError: reentrant call inside <_io.BufferedWriter>``. + + This test runs the *same* worker function through a fork-context Pool + twice: + + 1. WITHOUT any initializer (simulating the code before this PR) -- the + worker must report the identical fd number as the parent's open + FileHandler, proving the dangerous fd-sharing actually happens. + 2. WITH ``_worker_logging_init`` as the Pool initializer (the fix in + this PR) -- the worker must report a *different* fd, proving each + worker now owns a private file descriptor. + + If ``_worker_logging_init`` is removed or not passed to ``Pool(...)``, + step 2 will report the same (shared) fd as step 1 and this test fails. + """ + ctx = multiprocessing.get_context("fork") + + with tempfile.NamedTemporaryFile(suffix=".log", delete=False) as f: + log_path = f.name + + root = logging.getLogger() + original_handlers = root.handlers[:] + for h in original_handlers: + root.removeHandler(h) + + old_env = os.environ.get("GARAK_LOG_FILE") + os.environ["GARAK_LOG_FILE"] = log_path + + parent_handler = logging.FileHandler(log_path) + root.addHandler(parent_handler) + parent_fd = parent_handler.stream.fileno() + + try: + # Step 1: no initializer -- forked worker inherits the parent's fd + # table, so it must report the SAME fd as the parent. This is the + # exact pre-fix condition that allows concurrent/reentrant writes + # on a shared underlying file description to race. + with ctx.Pool(processes=1) as pool: + (fd_without_initializer,) = pool.map(_log_from_worker, [log_path]) + + assert fd_without_initializer == parent_fd, ( + "Sanity check failed: a forked worker without any logging " + "initializer was expected to inherit the parent's exact file " + f"descriptor ({parent_fd}), but got {fd_without_initializer}. " + "If this fails, the fd-inheritance precondition for issue " + "#1355 is not being reproduced on this platform/Python version." + ) + + # Step 2: with _worker_logging_init -- the worker must close the + # inherited handler and open its own, so its fd must differ from + # the parent's. + with ctx.Pool(processes=1, initializer=_worker_logging_init) as pool: + (fd_with_initializer,) = pool.map(_log_from_worker, [log_path]) + + assert fd_with_initializer != parent_fd, ( + "Worker process shared the parent's file descriptor " + f"({parent_fd}) even though _worker_logging_init was used as " + "the Pool initializer. This is the exact race condition from " + "issue #1355: without a fresh, private fd per worker, " + "concurrent or reentrant flushes on the shared handle can " + "raise 'RuntimeError: reentrant call inside " + "<_io.BufferedWriter>'." + ) + finally: + parent_handler.close() + root.removeHandler(parent_handler) + for h in original_handlers: + root.addHandler(h) + if old_env is None: + os.environ.pop("GARAK_LOG_FILE", None) + else: + os.environ["GARAK_LOG_FILE"] = old_env + os.unlink(log_path) + + # --------------------------------------------------------------------------- # Regression test: Probe._execute_all must wire up the initializer # --------------------------------------------------------------------------- @@ -291,4 +393,4 @@ def test_execute_all_passes_logging_initializer_to_pool(): assert kwargs.get("initializer") is _worker_logging_init, ( "Probe._execute_all must pass _worker_logging_init as the Pool " "initializer to avoid shared log file descriptors in workers" - ) + ) \ No newline at end of file