From e9ec2fa33c7d2d303f76719d49c8d4d595fc746a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 27 Jun 2025 00:23:53 +0100 Subject: [PATCH 1/9] gh-91048: Fix external inspection multi-threaded performance --- Lib/test/test_external_inspection.py | 117 ++++++++++++++++++ Modules/_remote_debugging_module.c | 69 +++++++++-- Modules/clinic/_remote_debugging_module.c.h | 40 ++++-- .../benchmark_external_inspection.py | 16 ++- 4 files changed, 222 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 90214e814f2b35..cf57cea631f978 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -5,6 +5,7 @@ import sys import socket import threading +import time from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry @@ -876,6 +877,122 @@ def test_self_trace(self): ], ) + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_only_active_thread(self): + # Test that only_active_thread parameter works correctly + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import time, sys, socket, threading + + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def worker_thread(name, barrier, ready_event): + barrier.wait() # Synchronize thread start + ready_event.wait() # Wait for main thread signal + # Sleep to keep thread alive + time.sleep(10_000) + + def main_work(): + # Do busy work to hold the GIL + count = 0 + while count < 100000000: + count += 1 + if count % 10000000 == 0: + pass # Keep main thread busy + + # Create synchronization primitives + num_threads = 3 + barrier = threading.Barrier(num_threads + 1) # +1 for main thread + ready_event = threading.Event() + + # Start worker threads + threads = [] + for i in range(num_threads): + t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) + t.start() + threads.append(t) + + # Wait for all threads to be ready + barrier.wait() + + # Signal ready to parent process + sock.sendall(b"ready\\n") + + # Signal threads to start waiting + ready_event.set() + + # Give threads time to start sleeping + time.sleep(0.1) + + # Now do busy work to hold the GIL + main_work() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + + # Wait for ready signal + response = b"" + while b"ready" not in response: + response += client_socket.recv(1024) + + # Give threads a moment to start their busy work + time.sleep(0.1) + + # Get stack trace with all threads + unwinder_all = RemoteUnwinder(p.pid, all_threads=True) + all_traces = unwinder_all.get_stack_trace() + + # Get stack trace with only GIL holder + unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) + gil_traces = unwinder_gil.get_stack_trace() + + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # Verify we got multiple threads in all_traces + self.assertGreater(len(all_traces), 1, "Should have multiple threads") + + # Verify we got exactly one thread in gil_traces + self.assertEqual(len(gil_traces), 1, "Should have exactly one GIL holder") + + # The GIL holder should be in the all_traces list + gil_thread_id = gil_traces[0][0] + all_thread_ids = [trace[0] for trace in all_traces] + self.assertIn(gil_thread_id, all_thread_ids, + "GIL holder should be among all threads") + if __name__ == "__main__": unittest.main() diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index c2421cac6bdb17..7a457c3da39973 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -64,12 +64,14 @@ #endif #ifdef Py_GIL_DISABLED -#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) #else -#define INTERP_STATE_MIN_SIZE MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) #endif #define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) @@ -206,6 +208,7 @@ typedef struct { uint64_t code_object_generation; _Py_hashtable_t *code_object_cache; int debug; + int only_active_thread; RemoteDebuggingState *cached_state; // Cached module state #ifdef Py_GIL_DISABLED // TLBC cache invalidation tracking @@ -2496,6 +2499,7 @@ _remote_debugging.RemoteUnwinder.__init__ pid: int * all_threads: bool = False + only_active_thread: bool = False debug: bool = False Initialize a new RemoteUnwinder object for debugging a remote Python process. @@ -2504,6 +2508,8 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. pid: Process ID of the target Python process to debug all_threads: If True, initialize state for all threads in the process. If False, only initialize for the main thread. + only_active_thread: If True, only sample the thread holding the GIL. + Cannot be used together with all_threads=True. debug: If True, chain exceptions to explain the sequence of events that lead to the exception. @@ -2514,15 +2520,25 @@ process, including examining thread states, stack frames and other runtime data. PermissionError: If access to the target process is denied OSError: If unable to attach to the target process or access its memory RuntimeError: If unable to read debug information from the target process + ValueError: If both all_threads and only_active_thread are True [clinic start generated code]*/ static int _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, + int only_active_thread, int debug) -/*[clinic end generated code: output=3982f2a7eba49334 input=48a762566b828e91]*/ +/*[clinic end generated code: output=13ba77598ecdcbe1 input=8f8f12504e17da04]*/ { + // Validate that all_threads and only_active_thread are not both True + if (all_threads && only_active_thread) { + PyErr_SetString(PyExc_ValueError, + "all_threads and only_active_thread cannot both be True"); + return -1; + } + self->debug = debug; + self->only_active_thread = only_active_thread; self->cached_state = NULL; if (_Py_RemoteDebug_InitProcHandle(&self->handle, pid) < 0) { set_exception_cause(self, PyExc_RuntimeError, "Failed to initialize process handle"); @@ -2602,13 +2618,18 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, @critical_section _remote_debugging.RemoteUnwinder.get_stack_trace -Returns a list of stack traces for all threads in the target process. +Returns a list of stack traces for threads in the target process. Each element in the returned list is a tuple of (thread_id, frame_list), where: - thread_id is the OS thread identifier - frame_list is a list of tuples (function_name, filename, line_number) representing the Python stack frames for that thread, ordered from most recent to oldest +The threads returned depend on the initialization parameters: +- If only_active_thread was True: returns only the thread holding the GIL +- If all_threads was True: returns all threads +- Otherwise: returns only the main thread + Example: [ (1234, [ @@ -2632,7 +2653,7 @@ Each element in the returned list is a tuple of (thread_id, frame_list), where: static PyObject * _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=666192b90c69d567 input=331dbe370578badf]*/ +/*[clinic end generated code: output=666192b90c69d567 input=f756f341206f9116]*/ { PyObject* result = NULL; // Read interpreter state into opaque buffer @@ -2655,6 +2676,28 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self _Py_hashtable_clear(self->code_object_cache); } + // If only_active_thread is true, we need to determine which thread holds the GIL + PyThreadState* gil_holder = NULL; + if (self->only_active_thread) { + // The GIL state is already in interp_state_buffer, just read from there + // Check if GIL is locked + int gil_locked = GET_MEMBER(int, interp_state_buffer, + self->debug_offsets.interpreter_state.gil_runtime_state_locked); + + if (gil_locked) { + // Get the last holder (current holder when GIL is locked) + gil_holder = GET_MEMBER(PyThreadState*, interp_state_buffer, + self->debug_offsets.interpreter_state.gil_runtime_state_holder); + } else { + // GIL is not locked, return empty list + result = PyList_New(0); + if (!result) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create empty result list"); + } + goto exit; + } + } + #ifdef Py_GIL_DISABLED // Check TLBC generation and invalidate cache if needed uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, @@ -2666,7 +2709,10 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self #endif uintptr_t current_tstate; - if (self->tstate_addr == 0) { + if (self->only_active_thread && gil_holder != NULL) { + // We have the GIL holder, process only that thread + current_tstate = (uintptr_t)gil_holder; + } else if (self->tstate_addr == 0) { // Get threads head from buffer current_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, self->debug_offsets.interpreter_state.threads_head); @@ -2700,6 +2746,11 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self if (self->tstate_addr) { break; } + + // If we're only processing the GIL holder, we're done after one iteration + if (self->only_active_thread && gil_holder != NULL) { + break; + } } exit: diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h index 5c313a2d66404a..e80b24b54c0ffa 100644 --- a/Modules/clinic/_remote_debugging_module.c.h +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -10,7 +10,8 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, -"RemoteUnwinder(pid, *, all_threads=False, debug=False)\n" +"RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" +" debug=False)\n" "--\n" "\n" "Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" @@ -19,6 +20,8 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, " pid: Process ID of the target Python process to debug\n" " all_threads: If True, initialize state for all threads in the process.\n" " If False, only initialize for the main thread.\n" +" only_active_thread: If True, only sample the thread holding the GIL.\n" +" Cannot be used together with all_threads=True.\n" " debug: If True, chain exceptions to explain the sequence of events that\n" " lead to the exception.\n" "\n" @@ -28,11 +31,13 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "Raises:\n" " PermissionError: If access to the target process is denied\n" " OSError: If unable to attach to the target process or access its memory\n" -" RuntimeError: If unable to read debug information from the target process"); +" RuntimeError: If unable to read debug information from the target process\n" +" ValueError: If both all_threads and only_active_thread are True"); static int _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, + int only_active_thread, int debug); static int @@ -41,7 +46,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -50,7 +55,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(debug), }, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(debug), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -59,19 +64,20 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pid", "all_threads", "debug", NULL}; + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "debug", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "RemoteUnwinder", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; int pid; int all_threads = 0; + int only_active_thread = 0; int debug = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, @@ -95,12 +101,21 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje goto skip_optional_kwonly; } } - debug = PyObject_IsTrue(fastargs[2]); + if (fastargs[2]) { + only_active_thread = PyObject_IsTrue(fastargs[2]); + if (only_active_thread < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + debug = PyObject_IsTrue(fastargs[3]); if (debug < 0) { goto exit; } skip_optional_kwonly: - return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, debug); + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, debug); exit: return return_value; @@ -110,13 +125,18 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, "get_stack_trace($self, /)\n" "--\n" "\n" -"Returns a list of stack traces for all threads in the target process.\n" +"Returns a list of stack traces for threads in the target process.\n" "\n" "Each element in the returned list is a tuple of (thread_id, frame_list), where:\n" "- thread_id is the OS thread identifier\n" "- frame_list is a list of tuples (function_name, filename, line_number) representing\n" " the Python stack frames for that thread, ordered from most recent to oldest\n" "\n" +"The threads returned depend on the initialization parameters:\n" +"- If only_active_thread was True: returns only the thread holding the GIL\n" +"- If all_threads was True: returns all threads\n" +"- Otherwise: returns only the main thread\n" +"\n" "Example:\n" " [\n" " (1234, [\n" @@ -253,4 +273,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=774ec34aa653402d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a37ab223d5081b16 input=a9049054013a1b77]*/ diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index 62182194c1ab2a..5fae83806dbb7b 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -346,6 +346,13 @@ def parse_arguments(): help="Code example to benchmark (default: basic)", ) + parser.add_argument( + "--threads", + choices=["all", "main", "only_active"], + default="all", + help="Which threads to include in the benchmark (default: all)", + ) + return parser.parse_args() @@ -419,8 +426,15 @@ def main(): # Create unwinder and run benchmark print(f"{colors.BLUE}Initializing unwinder...{colors.RESET}") try: + kwargs = {} + if args.threads == "all": + kwargs["all_threads"] = True + elif args.threads == "main": + kwargs["all_threads"] = False + elif args.threads == "only_active": + kwargs["only_active_thread"] = True unwinder = _remote_debugging.RemoteUnwinder( - process.pid, all_threads=True + process.pid, **kwargs ) results = benchmark(unwinder, duration_seconds=args.duration) finally: From 0663518cc1d082b7ec3855de509601bc2bfa385d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 27 Jun 2025 02:18:18 +0100 Subject: [PATCH 2/9] Remove the cache as it was making things worse --- Modules/_remote_debugging_module.c | 6 --- Python/remote_debug.h | 78 ------------------------------ 2 files changed, 84 deletions(-) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 7a457c3da39973..9984c316940db8 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -2754,7 +2754,6 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } exit: - _Py_RemoteDebug_ClearCache(&self->handle); return result; } @@ -2878,11 +2877,9 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s goto result_err; } - _Py_RemoteDebug_ClearCache(&self->handle); return result; result_err: - _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } @@ -2949,11 +2946,9 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject goto cleanup; } - _Py_RemoteDebug_ClearCache(&self->handle); return result; cleanup: - _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } @@ -2979,7 +2974,6 @@ RemoteUnwinder_dealloc(PyObject *op) } #endif if (self->handle.pid != 0) { - _Py_RemoteDebug_ClearCache(&self->handle); _Py_RemoteDebug_CleanupProcHandle(&self->handle); } PyObject_Del(self); diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 8f9b6cd4c4960f..d1fcb478d2b035 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -110,14 +110,6 @@ get_page_size(void) { return page_size; } -typedef struct page_cache_entry { - uintptr_t page_addr; // page-aligned base address - char *data; - int valid; - struct page_cache_entry *next; -} page_cache_entry_t; - -#define MAX_PAGES 1024 // Define a platform-independent process handle structure typedef struct { @@ -129,27 +121,9 @@ typedef struct { #elif defined(__linux__) int memfd; #endif - page_cache_entry_t pages[MAX_PAGES]; Py_ssize_t page_size; } proc_handle_t; -static void -_Py_RemoteDebug_FreePageCache(proc_handle_t *handle) -{ - for (int i = 0; i < MAX_PAGES; i++) { - PyMem_RawFree(handle->pages[i].data); - handle->pages[i].data = NULL; - handle->pages[i].valid = 0; - } -} - -UNUSED static void -_Py_RemoteDebug_ClearCache(proc_handle_t *handle) -{ - for (int i = 0; i < MAX_PAGES; i++) { - handle->pages[i].valid = 0; - } -} #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static mach_port_t pid_to_task(pid_t pid); @@ -178,10 +152,6 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->memfd = -1; #endif handle->page_size = get_page_size(); - for (int i = 0; i < MAX_PAGES; i++) { - handle->pages[i].data = NULL; - handle->pages[i].valid = 0; - } return 0; } @@ -200,7 +170,6 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { } #endif handle->pid = 0; - _Py_RemoteDebug_FreePageCache(handle); } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -1066,53 +1035,6 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, size_t size, void *out) { - size_t page_size = handle->page_size; - uintptr_t page_base = addr & ~(page_size - 1); - size_t offset_in_page = addr - page_base; - - if (offset_in_page + size > page_size) { - return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); - } - - // Search for valid cached page - for (int i = 0; i < MAX_PAGES; i++) { - page_cache_entry_t *entry = &handle->pages[i]; - if (entry->valid && entry->page_addr == page_base) { - memcpy(out, entry->data + offset_in_page, size); - return 0; - } - } - - // Find reusable slot - for (int i = 0; i < MAX_PAGES; i++) { - page_cache_entry_t *entry = &handle->pages[i]; - if (!entry->valid) { - if (entry->data == NULL) { - entry->data = PyMem_RawMalloc(page_size); - if (entry->data == NULL) { - _set_debug_exception_cause(PyExc_MemoryError, - "Cannot allocate %zu bytes for page cache entry " - "during read from PID %d at address 0x%lx", - page_size, handle->pid, addr); - return -1; - } - } - - if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { - // Try to just copy the exact ammount as a fallback - PyErr_Clear(); - goto fallback; - } - - entry->page_addr = page_base; - entry->valid = 1; - memcpy(out, entry->data + offset_in_page, size); - return 0; - } - } - -fallback: - // Cache full — fallback to uncached read return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } From ab299f4d691c3d46f4fb609ac034936403bc8288 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 27 Jun 2025 02:22:40 +0100 Subject: [PATCH 3/9] Fix perf script --- Tools/inspection/benchmark_external_inspection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index 5fae83806dbb7b..0ac7ac4d385792 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -174,6 +174,7 @@ def benchmark(unwinder, duration_seconds=10): total_work_time = 0.0 start_time = time.perf_counter() end_time = start_time + duration_seconds + total_attempts = 0 colors = get_colors(can_colorize()) @@ -183,6 +184,7 @@ def benchmark(unwinder, duration_seconds=10): try: while time.perf_counter() < end_time: + total_attempts += 1 work_start = time.perf_counter() try: stack_trace = unwinder.get_stack_trace() @@ -194,7 +196,6 @@ def benchmark(unwinder, duration_seconds=10): work_end = time.perf_counter() total_work_time += work_end - work_start - total_attempts = sample_count + fail_count if total_attempts % 10000 == 0: avg_work_time_us = (total_work_time / total_attempts) * 1e6 work_rate = ( @@ -221,7 +222,6 @@ def benchmark(unwinder, duration_seconds=10): actual_end_time = time.perf_counter() wall_time = actual_end_time - start_time - total_attempts = sample_count + fail_count # Return final statistics return { From d166146a575f01e215cb757cca1180e15b0e80d0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 27 Jun 2025 02:34:00 +0100 Subject: [PATCH 4/9] Regen files --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 ++ Lib/test/test_external_inspection.py | 40 +++++++++---------- Modules/_remote_debugging_module.c | 4 +- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index e118b86db50754..c461bc1786ddf4 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1136,6 +1136,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(offset_src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(on_type_read)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(onceregistry)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_active_thread)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 36f3d23d095d59..72c2051bd97660 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -627,6 +627,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(offset_src) STRUCT_FOR_ID(on_type_read) STRUCT_FOR_ID(onceregistry) + STRUCT_FOR_ID(only_active_thread) STRUCT_FOR_ID(only_keys) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index d172cc1485d426..d378fcae26cf35 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1134,6 +1134,7 @@ extern "C" { INIT_ID(offset_src), \ INIT_ID(on_type_read), \ INIT_ID(onceregistry), \ + INIT_ID(only_active_thread), \ INIT_ID(only_keys), \ INIT_ID(oparg), \ INIT_ID(opcode), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0a9be4e41ace89..e516211f6c6cbc 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2296,6 +2296,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(only_active_thread); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(only_keys); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index cf57cea631f978..3748b3148a4cf0 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -888,17 +888,17 @@ def test_only_active_thread(self): script = textwrap.dedent( f"""\ import time, sys, socket, threading - + # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) - + def worker_thread(name, barrier, ready_event): barrier.wait() # Synchronize thread start ready_event.wait() # Wait for main thread signal # Sleep to keep thread alive time.sleep(10_000) - + def main_work(): # Do busy work to hold the GIL count = 0 @@ -906,36 +906,36 @@ def main_work(): count += 1 if count % 10000000 == 0: pass # Keep main thread busy - + # Create synchronization primitives num_threads = 3 barrier = threading.Barrier(num_threads + 1) # +1 for main thread ready_event = threading.Event() - + # Start worker threads threads = [] for i in range(num_threads): t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) t.start() threads.append(t) - + # Wait for all threads to be ready barrier.wait() - + # Signal ready to parent process sock.sendall(b"ready\\n") - + # Signal threads to start waiting ready_event.set() - + # Give threads time to start sleeping time.sleep(0.1) - + # Now do busy work to hold the GIL main_work() """ ) - + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) @@ -953,23 +953,23 @@ def main_work(): p = subprocess.Popen([sys.executable, script_name]) client_socket, _ = server_socket.accept() server_socket.close() - + # Wait for ready signal response = b"" while b"ready" not in response: response += client_socket.recv(1024) - + # Give threads a moment to start their busy work time.sleep(0.1) - + # Get stack trace with all threads unwinder_all = RemoteUnwinder(p.pid, all_threads=True) all_traces = unwinder_all.get_stack_trace() - + # Get stack trace with only GIL holder unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) gil_traces = unwinder_gil.get_stack_trace() - + except PermissionError: self.skipTest( "Insufficient permissions to read the stack trace" @@ -980,17 +980,17 @@ def main_work(): p.kill() p.terminate() p.wait(timeout=SHORT_TIMEOUT) - + # Verify we got multiple threads in all_traces self.assertGreater(len(all_traces), 1, "Should have multiple threads") - + # Verify we got exactly one thread in gil_traces self.assertEqual(len(gil_traces), 1, "Should have exactly one GIL holder") - + # The GIL holder should be in the all_traces list gil_thread_id = gil_traces[0][0] all_thread_ids = [trace[0] for trace in all_traces] - self.assertIn(gil_thread_id, all_thread_ids, + self.assertIn(gil_thread_id, all_thread_ids, "GIL holder should be among all threads") diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 9984c316940db8..b20b575ebc0ef8 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -2532,11 +2532,11 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { - PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "all_threads and only_active_thread cannot both be True"); return -1; } - + self->debug = debug; self->only_active_thread = only_active_thread; self->cached_state = NULL; From 03f447e57eb43ba69c4675ea00aaa03e81c5b97d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 27 Jun 2025 12:00:35 +0100 Subject: [PATCH 5/9] FIx race --- Lib/test/test_external_inspection.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 3748b3148a4cf0..572679173a3ebf 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -901,11 +901,13 @@ def worker_thread(name, barrier, ready_event): def main_work(): # Do busy work to hold the GIL + sock.sendall(b"working\\n") count = 0 while count < 100000000: count += 1 if count % 10000000 == 0: pass # Keep main thread busy + sock.sendall(b"done\\n") # Create synchronization primitives num_threads = 3 @@ -959,8 +961,9 @@ def main_work(): while b"ready" not in response: response += client_socket.recv(1024) - # Give threads a moment to start their busy work - time.sleep(0.1) + # Wait for the main thread to start its busy work + while b"working" not in response: + response += client_socket.recv(1024) # Get stack trace with all threads unwinder_all = RemoteUnwinder(p.pid, all_threads=True) From f1b1e4f330c894f57723d91feed83f54d04261b3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 02:44:39 +0100 Subject: [PATCH 6/9] FIx race --- Lib/test/test_external_inspection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 572679173a3ebf..daf9c335feda25 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -8,7 +8,7 @@ import time from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry +from test.support import os_helper, SHORT_TIMEOUT, busy_retry, requires_gil_enabled from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -882,6 +882,7 @@ def test_self_trace(self): sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support", ) + @requires_gil_enabled def test_only_active_thread(self): # Test that only_active_thread parameter works correctly port = find_unused_port() From 04f874816bf2f7589206ce85bbd94b606e5b9c9e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 03:04:25 +0100 Subject: [PATCH 7/9] FIx race --- Lib/test/test_external_inspection.py | 2 +- Lib/test/test_external_inspection_fixed.py | 1001 ++++++++++++++++++++ 2 files changed, 1002 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_external_inspection_fixed.py diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index daf9c335feda25..6e8c6f5823dec1 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -877,12 +877,12 @@ def test_self_trace(self): ], ) + @requires_gil_enabled("Free threaded builds don't have an 'active thread'") @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support", ) - @requires_gil_enabled def test_only_active_thread(self): # Test that only_active_thread parameter works correctly port = find_unused_port() diff --git a/Lib/test/test_external_inspection_fixed.py b/Lib/test/test_external_inspection_fixed.py new file mode 100644 index 00000000000000..572679173a3ebf --- /dev/null +++ b/Lib/test/test_external_inspection_fixed.py @@ -0,0 +1,1001 @@ +import unittest +import os +import textwrap +import importlib +import sys +import socket +import threading +import time +from asyncio import staggered, taskgroups, base_events, tasks +from unittest.mock import ANY +from test.support import os_helper, SHORT_TIMEOUT, busy_retry +from test.support.script_helper import make_script +from test.support.socket_helper import find_unused_port + +import subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + from _remote_debugging import RemoteUnwinder + from _remote_debugging import FrameInfo, CoroInfo, TaskInfo +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + + +def _make_test_script(script_dir, script_basename, source): + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + + +skip_if_not_supported = unittest.skipIf( + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), + "Test only runs on Linux, Windows and MacOS", +) + + +def get_stack_trace(pid): + unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) + return unwinder.get_stack_trace() + + +def get_async_stack_trace(pid): + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_async_stack_trace() + + +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_all_awaited_by() + + +class TestGetStackTrace(unittest.TestCase): + maxDiff = None + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_remote_stack_trace(self): + # Spawn a process with some realistic Python code + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import time, sys, socket, threading + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def bar(): + for x in range(100): + if x == 50: + baz() + + def baz(): + foo() + + def foo(): + sock.sendall(b"ready:thread\\n"); time.sleep(10_000) # same line number + + t = threading.Thread(target=bar) + t.start() + sock.sendall(b"ready:main\\n"); t.join() # same line number + """ + ) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + response = b"" + while ( + b"ready:main" not in response + or b"ready:thread" not in response + ): + response += client_socket.recv(1024) + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + thread_expected_stack_trace = [ + FrameInfo([script_name, 15, "foo"]), + FrameInfo([script_name, 12, "baz"]), + FrameInfo([script_name, 9, "bar"]), + FrameInfo([threading.__file__, ANY, "Thread.run"]), + ] + # Is possible that there are more threads, so we check that the + # expected stack traces are in the result (looking at you Windows!) + self.assertIn((ANY, thread_expected_stack_trace), stack_trace) + + # Check that the main thread stack trace is in the result + frame = FrameInfo([script_name, 19, ""]) + for _, stack in stack_trace: + if frame in stack: + break + else: + self.fail("Main thread stack trace not found in result") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_remote_stack_trace(self): + # Spawn a process with some realistic Python code + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import time + import sys + import socket + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def c5(): + sock.sendall(b"ready"); time.sleep(10_000) # same line number + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + def new_eager_loop(): + loop = asyncio.new_event_loop() + eager_task_factory = asyncio.create_eager_task_factory( + asyncio.Task) + loop.set_task_factory(eager_task_factory) + return loop + + asyncio.run(main(), loop_factory={{TASK_FACTORY}}) + """ + ) + stack_trace = None + for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": + with ( + self.subTest(task_factory_variant=task_factory_variant), + os_helper.temp_dir() as work_dir, + ): + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + server_socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM + ) + server_socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 + ) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + script_name = _make_test_script( + script_dir, + "script", + script.format(TASK_FACTORY=task_factory_variant), + ) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + self.assertEqual(response, b"ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [ + [ + FrameInfo([script_name, 10, "c5"]), + FrameInfo([script_name, 14, "c4"]), + FrameInfo([script_name, 17, "c3"]), + FrameInfo([script_name, 20, "c2"]), + ], + "c2_root", + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo([script_name, 26, "main"]), + ], + "Task-1", + ] + ), + CoroInfo( + [ + [FrameInfo([script_name, 23, "c1"])], + "sub_main_1", + ] + ), + CoroInfo( + [ + [FrameInfo([script_name, 23, "c1"])], + "sub_main_2", + ] + ), + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_asyncgen_remote_stack_trace(self): + # Spawn a process with some realistic Python code + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import time + import sys + import socket + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + async def gen_nested_call(): + sock.sendall(b"ready"); time.sleep(10_000) # same line number + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + asyncio.run(main()) + """ + ) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + self.assertEqual(response, b"ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [ + [ + FrameInfo([script_name, 10, "gen_nested_call"]), + FrameInfo([script_name, 16, "gen"]), + FrameInfo([script_name, 19, "main"]), + ], + "Task-1", + [], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_gather_remote_stack_trace(self): + # Spawn a process with some realistic Python code + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import time + import sys + import socket + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + async def deep(): + await asyncio.sleep(0) + sock.sendall(b"ready"); time.sleep(10_000) # same line number + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + asyncio.run(main()) + """ + ) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + self.assertEqual(response, b"ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [ + [ + FrameInfo([script_name, 11, "deep"]), + FrameInfo([script_name, 15, "c1"]), + ], + "Task-2", + [CoroInfo([[FrameInfo([script_name, 21, "main"])], "Task-1"])], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_staggered_race_remote_stack_trace(self): + # Spawn a process with some realistic Python code + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio.staggered + import time + import sys + import socket + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + async def deep(): + await asyncio.sleep(0) + sock.sendall(b"ready"); time.sleep(10_000) # same line number + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(10_000) + + async def main(): + await asyncio.staggered.staggered_race( + [c1, c2], + delay=None, + ) + + asyncio.run(main()) + """ + ) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + self.assertEqual(response, b"ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + expected_stack_trace = [ + [ + FrameInfo([script_name, 11, "deep"]), + FrameInfo([script_name, 15, "c1"]), + FrameInfo( + [ + staggered.__file__, + ANY, + "staggered_race..run_one_coro", + ] + ), + ], + "Task-2", + [ + CoroInfo( + [ + [ + FrameInfo( + [staggered.__file__, ANY, "staggered_race"] + ), + FrameInfo([script_name, 21, "main"]), + ], + "Task-1", + ] + ) + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_global_awaited_by(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import os + import random + import sys + import socket + from string import ascii_lowercase, digits + from test.support import socket_helper, SHORT_TIMEOUT + + HOST = '127.0.0.1' + PORT = socket_helper.find_unused_port() + connections = 0 + + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + class EchoServerProtocol(asyncio.Protocol): + def connection_made(self, transport): + global connections + connections += 1 + self.transport = transport + + def data_received(self, data): + self.transport.write(data) + self.transport.close() + + async def echo_client(message): + reader, writer = await asyncio.open_connection(HOST, PORT) + writer.write(message.encode()) + await writer.drain() + + data = await reader.read(100) + assert message == data.decode() + writer.close() + await writer.wait_closed() + # Signal we are ready to sleep + sock.sendall(b"ready") + await asyncio.sleep(SHORT_TIMEOUT) + + async def echo_client_spam(server): + async with asyncio.TaskGroup() as tg: + while connections < 1000: + msg = list(ascii_lowercase + digits) + random.shuffle(msg) + tg.create_task(echo_client("".join(msg))) + await asyncio.sleep(0) + # at least a 1000 tasks created. Each task will signal + # when is ready to avoid the race caused by the fact that + # tasks are waited on tg.__exit__ and we cannot signal when + # that happens otherwise + # at this point all client tasks completed without assertion errors + # let's wrap up the test + server.close() + await server.wait_closed() + + async def main(): + loop = asyncio.get_running_loop() + server = await loop.create_server(EchoServerProtocol, HOST, PORT) + async with server: + async with asyncio.TaskGroup() as tg: + tg.create_task(server.serve_forever(), name="server task") + tg.create_task(echo_client_spam(server), name="echo client spam") + + asyncio.run(main()) + """ + ) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + for _ in range(1000): + expected_response = b"ready" + response = client_socket.recv(len(expected_response)) + self.assertEqual(response, expected_response) + for _ in busy_retry(SHORT_TIMEOUT): + try: + all_awaited_by = get_all_awaited_by(p.pid) + except RuntimeError as re: + # This call reads a linked list in another process with + # no synchronization. That occasionally leads to invalid + # reads. Here we avoid making the test flaky. + msg = str(re) + if msg.startswith("Task list appears corrupted"): + continue + elif msg.startswith( + "Invalid linked list structure reading remote memory" + ): + continue + elif msg.startswith("Unknown error reading memory"): + continue + elif msg.startswith("Unhandled frame owner"): + continue + raise # Unrecognized exception, safest not to ignore it + else: + break + # expected: a list of two elements: 1 thread, 1 interp + self.assertEqual(len(all_awaited_by), 2) + # expected: a tuple with the thread ID and the awaited_by list + self.assertEqual(len(all_awaited_by[0]), 2) + # expected: no tasks in the fallback per-interp task list + self.assertEqual(all_awaited_by[1], (0, [])) + entries = all_awaited_by[0][1] + # expected: at least 1000 pending tasks + self.assertGreaterEqual(len(entries), 1000) + # the first three tasks stem from the code structure + main_stack = [ + FrameInfo([taskgroups.__file__, ANY, "TaskGroup._aexit"]), + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] + ), + FrameInfo([script_name, 60, "main"]), + ] + self.assertIn( + TaskInfo( + [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] + ), + entries, + ) + self.assertIn( + TaskInfo( + [ + ANY, + "server task", + [ + CoroInfo( + [ + [ + FrameInfo( + [ + base_events.__file__, + ANY, + "Server.serve_forever", + ] + ) + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, ANY, "main"] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + self.assertIn( + TaskInfo( + [ + ANY, + "Task-4", + [ + CoroInfo( + [ + [ + FrameInfo( + [tasks.__file__, ANY, "sleep"] + ), + FrameInfo( + [ + script_name, + 38, + "echo_client", + ] + ), + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [ + script_name, + 41, + "echo_client_spam", + ] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + + expected_awaited_by = [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, 41, "echo_client_spam"] + ), + ], + ANY, + ] + ) + ] + tasks_with_awaited = [ + task + for task in entries + if task.awaited_by == expected_awaited_by + ] + self.assertGreaterEqual(len(tasks_with_awaited), 1000) + + # the final task will have some random number, but it should for + # sure be one of the echo client spam horde (In windows this is not true + # for some reason) + if sys.platform != "win32": + self.assertEqual( + tasks_with_awaited[-1].awaited_by, + entries[-1].awaited_by, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_self_trace(self): + stack_trace = get_stack_trace(os.getpid()) + # Is possible that there are more threads, so we check that the + # expected stack traces are in the result (looking at you Windows!) + this_tread_stack = None + for thread_id, stack in stack_trace: + if thread_id == threading.get_native_id(): + this_tread_stack = stack + break + self.assertIsNotNone(this_tread_stack) + self.assertEqual( + stack[:2], + [ + FrameInfo( + [ + __file__, + get_stack_trace.__code__.co_firstlineno + 2, + "get_stack_trace", + ] + ), + FrameInfo( + [ + __file__, + self.test_self_trace.__code__.co_firstlineno + 6, + "TestGetStackTrace.test_self_trace", + ] + ), + ], + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_only_active_thread(self): + # Test that only_active_thread parameter works correctly + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import time, sys, socket, threading + + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def worker_thread(name, barrier, ready_event): + barrier.wait() # Synchronize thread start + ready_event.wait() # Wait for main thread signal + # Sleep to keep thread alive + time.sleep(10_000) + + def main_work(): + # Do busy work to hold the GIL + sock.sendall(b"working\\n") + count = 0 + while count < 100000000: + count += 1 + if count % 10000000 == 0: + pass # Keep main thread busy + sock.sendall(b"done\\n") + + # Create synchronization primitives + num_threads = 3 + barrier = threading.Barrier(num_threads + 1) # +1 for main thread + ready_event = threading.Event() + + # Start worker threads + threads = [] + for i in range(num_threads): + t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) + t.start() + threads.append(t) + + # Wait for all threads to be ready + barrier.wait() + + # Signal ready to parent process + sock.sendall(b"ready\\n") + + # Signal threads to start waiting + ready_event.set() + + # Give threads time to start sleeping + time.sleep(0.1) + + # Now do busy work to hold the GIL + main_work() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + + # Wait for ready signal + response = b"" + while b"ready" not in response: + response += client_socket.recv(1024) + + # Wait for the main thread to start its busy work + while b"working" not in response: + response += client_socket.recv(1024) + + # Get stack trace with all threads + unwinder_all = RemoteUnwinder(p.pid, all_threads=True) + all_traces = unwinder_all.get_stack_trace() + + # Get stack trace with only GIL holder + unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) + gil_traces = unwinder_gil.get_stack_trace() + + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # Verify we got multiple threads in all_traces + self.assertGreater(len(all_traces), 1, "Should have multiple threads") + + # Verify we got exactly one thread in gil_traces + self.assertEqual(len(gil_traces), 1, "Should have exactly one GIL holder") + + # The GIL holder should be in the all_traces list + gil_thread_id = gil_traces[0][0] + all_thread_ids = [trace[0] for trace in all_traces] + self.assertIn(gil_thread_id, all_thread_ids, + "GIL holder should be among all threads") + + +if __name__ == "__main__": + unittest.main() From 63261de582871e0c4eaa9c9fdde0bfeed944fe25 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 03:21:59 +0100 Subject: [PATCH 8/9] FIx race --- Lib/test/test_external_inspection.py | 2 +- Lib/test/test_external_inspection_fixed.py | 1001 -------------------- 2 files changed, 1 insertion(+), 1002 deletions(-) delete mode 100644 Lib/test/test_external_inspection_fixed.py diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 6e8c6f5823dec1..e29d44e209b6a4 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -877,12 +877,12 @@ def test_self_trace(self): ], ) - @requires_gil_enabled("Free threaded builds don't have an 'active thread'") @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support", ) + @requires_gil_enabled("Free threaded builds don't have an 'active thread'") def test_only_active_thread(self): # Test that only_active_thread parameter works correctly port = find_unused_port() diff --git a/Lib/test/test_external_inspection_fixed.py b/Lib/test/test_external_inspection_fixed.py deleted file mode 100644 index 572679173a3ebf..00000000000000 --- a/Lib/test/test_external_inspection_fixed.py +++ /dev/null @@ -1,1001 +0,0 @@ -import unittest -import os -import textwrap -import importlib -import sys -import socket -import threading -import time -from asyncio import staggered, taskgroups, base_events, tasks -from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry -from test.support.script_helper import make_script -from test.support.socket_helper import find_unused_port - -import subprocess - -PROCESS_VM_READV_SUPPORTED = False - -try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - from _remote_debugging import RemoteUnwinder - from _remote_debugging import FrameInfo, CoroInfo, TaskInfo -except ImportError: - raise unittest.SkipTest( - "Test only runs when _remote_debugging is available" - ) - - -def _make_test_script(script_dir, script_basename, source): - to_return = make_script(script_dir, script_basename, source) - importlib.invalidate_caches() - return to_return - - -skip_if_not_supported = unittest.skipIf( - ( - sys.platform != "darwin" - and sys.platform != "linux" - and sys.platform != "win32" - ), - "Test only runs on Linux, Windows and MacOS", -) - - -def get_stack_trace(pid): - unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) - return unwinder.get_stack_trace() - - -def get_async_stack_trace(pid): - unwinder = RemoteUnwinder(pid, debug=True) - return unwinder.get_async_stack_trace() - - -def get_all_awaited_by(pid): - unwinder = RemoteUnwinder(pid, debug=True) - return unwinder.get_all_awaited_by() - - -class TestGetStackTrace(unittest.TestCase): - maxDiff = None - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_remote_stack_trace(self): - # Spawn a process with some realistic Python code - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import time, sys, socket, threading - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - def bar(): - for x in range(100): - if x == 50: - baz() - - def baz(): - foo() - - def foo(): - sock.sendall(b"ready:thread\\n"); time.sleep(10_000) # same line number - - t = threading.Thread(target=bar) - t.start() - sock.sendall(b"ready:main\\n"); t.join() # same line number - """ - ) - stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = b"" - while ( - b"ready:main" not in response - or b"ready:thread" not in response - ): - response += client_socket.recv(1024) - stack_trace = get_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - thread_expected_stack_trace = [ - FrameInfo([script_name, 15, "foo"]), - FrameInfo([script_name, 12, "baz"]), - FrameInfo([script_name, 9, "bar"]), - FrameInfo([threading.__file__, ANY, "Thread.run"]), - ] - # Is possible that there are more threads, so we check that the - # expected stack traces are in the result (looking at you Windows!) - self.assertIn((ANY, thread_expected_stack_trace), stack_trace) - - # Check that the main thread stack trace is in the result - frame = FrameInfo([script_name, 19, ""]) - for _, stack in stack_trace: - if frame in stack: - break - else: - self.fail("Main thread stack trace not found in result") - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_async_remote_stack_trace(self): - # Spawn a process with some realistic Python code - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import asyncio - import time - import sys - import socket - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - def c5(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number - - async def c4(): - await asyncio.sleep(0) - c5() - - async def c3(): - await c4() - - async def c2(): - await c3() - - async def c1(task): - await task - - async def main(): - async with asyncio.TaskGroup() as tg: - task = tg.create_task(c2(), name="c2_root") - tg.create_task(c1(task), name="sub_main_1") - tg.create_task(c1(task), name="sub_main_2") - - def new_eager_loop(): - loop = asyncio.new_event_loop() - eager_task_factory = asyncio.create_eager_task_factory( - asyncio.Task) - loop.set_task_factory(eager_task_factory) - return loop - - asyncio.run(main(), loop_factory={{TASK_FACTORY}}) - """ - ) - stack_trace = None - for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": - with ( - self.subTest(task_factory_variant=task_factory_variant), - os_helper.temp_dir() as work_dir, - ): - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - server_socket = socket.socket( - socket.AF_INET, socket.SOCK_STREAM - ) - server_socket.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 - ) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script( - script_dir, - "script", - script.format(TASK_FACTORY=task_factory_variant), - ) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [ - FrameInfo([script_name, 10, "c5"]), - FrameInfo([script_name, 14, "c4"]), - FrameInfo([script_name, 17, "c3"]), - FrameInfo([script_name, 20, "c2"]), - ], - "c2_root", - [ - CoroInfo( - [ - [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo([script_name, 26, "main"]), - ], - "Task-1", - ] - ), - CoroInfo( - [ - [FrameInfo([script_name, 23, "c1"])], - "sub_main_1", - ] - ), - CoroInfo( - [ - [FrameInfo([script_name, 23, "c1"])], - "sub_main_2", - ] - ), - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_asyncgen_remote_stack_trace(self): - # Spawn a process with some realistic Python code - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import asyncio - import time - import sys - import socket - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - async def gen_nested_call(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number - - async def gen(): - for num in range(2): - yield num - if num == 1: - await gen_nested_call() - - async def main(): - async for el in gen(): - pass - - asyncio.run(main()) - """ - ) - stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [ - FrameInfo([script_name, 10, "gen_nested_call"]), - FrameInfo([script_name, 16, "gen"]), - FrameInfo([script_name, 19, "main"]), - ], - "Task-1", - [], - ] - self.assertEqual(stack_trace, expected_stack_trace) - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_async_gather_remote_stack_trace(self): - # Spawn a process with some realistic Python code - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import asyncio - import time - import sys - import socket - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - async def deep(): - await asyncio.sleep(0) - sock.sendall(b"ready"); time.sleep(10_000) # same line number - - async def c1(): - await asyncio.sleep(0) - await deep() - - async def c2(): - await asyncio.sleep(0) - - async def main(): - await asyncio.gather(c1(), c2()) - - asyncio.run(main()) - """ - ) - stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [ - FrameInfo([script_name, 11, "deep"]), - FrameInfo([script_name, 15, "c1"]), - ], - "Task-2", - [CoroInfo([[FrameInfo([script_name, 21, "main"])], "Task-1"])], - ] - self.assertEqual(stack_trace, expected_stack_trace) - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_async_staggered_race_remote_stack_trace(self): - # Spawn a process with some realistic Python code - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import asyncio.staggered - import time - import sys - import socket - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - async def deep(): - await asyncio.sleep(0) - sock.sendall(b"ready"); time.sleep(10_000) # same line number - - async def c1(): - await asyncio.sleep(0) - await deep() - - async def c2(): - await asyncio.sleep(10_000) - - async def main(): - await asyncio.staggered.staggered_race( - [c1, c2], - delay=None, - ) - - asyncio.run(main()) - """ - ) - stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [ - [ - FrameInfo([script_name, 11, "deep"]), - FrameInfo([script_name, 15, "c1"]), - FrameInfo( - [ - staggered.__file__, - ANY, - "staggered_race..run_one_coro", - ] - ), - ], - "Task-2", - [ - CoroInfo( - [ - [ - FrameInfo( - [staggered.__file__, ANY, "staggered_race"] - ), - FrameInfo([script_name, 21, "main"]), - ], - "Task-1", - ] - ) - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_async_global_awaited_by(self): - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import asyncio - import os - import random - import sys - import socket - from string import ascii_lowercase, digits - from test.support import socket_helper, SHORT_TIMEOUT - - HOST = '127.0.0.1' - PORT = socket_helper.find_unused_port() - connections = 0 - - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - class EchoServerProtocol(asyncio.Protocol): - def connection_made(self, transport): - global connections - connections += 1 - self.transport = transport - - def data_received(self, data): - self.transport.write(data) - self.transport.close() - - async def echo_client(message): - reader, writer = await asyncio.open_connection(HOST, PORT) - writer.write(message.encode()) - await writer.drain() - - data = await reader.read(100) - assert message == data.decode() - writer.close() - await writer.wait_closed() - # Signal we are ready to sleep - sock.sendall(b"ready") - await asyncio.sleep(SHORT_TIMEOUT) - - async def echo_client_spam(server): - async with asyncio.TaskGroup() as tg: - while connections < 1000: - msg = list(ascii_lowercase + digits) - random.shuffle(msg) - tg.create_task(echo_client("".join(msg))) - await asyncio.sleep(0) - # at least a 1000 tasks created. Each task will signal - # when is ready to avoid the race caused by the fact that - # tasks are waited on tg.__exit__ and we cannot signal when - # that happens otherwise - # at this point all client tasks completed without assertion errors - # let's wrap up the test - server.close() - await server.wait_closed() - - async def main(): - loop = asyncio.get_running_loop() - server = await loop.create_server(EchoServerProtocol, HOST, PORT) - async with server: - async with asyncio.TaskGroup() as tg: - tg.create_task(server.serve_forever(), name="server task") - tg.create_task(echo_client_spam(server), name="echo client spam") - - asyncio.run(main()) - """ - ) - stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - for _ in range(1000): - expected_response = b"ready" - response = client_socket.recv(len(expected_response)) - self.assertEqual(response, expected_response) - for _ in busy_retry(SHORT_TIMEOUT): - try: - all_awaited_by = get_all_awaited_by(p.pid) - except RuntimeError as re: - # This call reads a linked list in another process with - # no synchronization. That occasionally leads to invalid - # reads. Here we avoid making the test flaky. - msg = str(re) - if msg.startswith("Task list appears corrupted"): - continue - elif msg.startswith( - "Invalid linked list structure reading remote memory" - ): - continue - elif msg.startswith("Unknown error reading memory"): - continue - elif msg.startswith("Unhandled frame owner"): - continue - raise # Unrecognized exception, safest not to ignore it - else: - break - # expected: a list of two elements: 1 thread, 1 interp - self.assertEqual(len(all_awaited_by), 2) - # expected: a tuple with the thread ID and the awaited_by list - self.assertEqual(len(all_awaited_by[0]), 2) - # expected: no tasks in the fallback per-interp task list - self.assertEqual(all_awaited_by[1], (0, [])) - entries = all_awaited_by[0][1] - # expected: at least 1000 pending tasks - self.assertGreaterEqual(len(entries), 1000) - # the first three tasks stem from the code structure - main_stack = [ - FrameInfo([taskgroups.__file__, ANY, "TaskGroup._aexit"]), - FrameInfo( - [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] - ), - FrameInfo([script_name, 60, "main"]), - ] - self.assertIn( - TaskInfo( - [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] - ), - entries, - ) - self.assertIn( - TaskInfo( - [ - ANY, - "server task", - [ - CoroInfo( - [ - [ - FrameInfo( - [ - base_events.__file__, - ANY, - "Server.serve_forever", - ] - ) - ], - ANY, - ] - ) - ], - [ - CoroInfo( - [ - [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo( - [script_name, ANY, "main"] - ), - ], - ANY, - ] - ) - ], - ] - ), - entries, - ) - self.assertIn( - TaskInfo( - [ - ANY, - "Task-4", - [ - CoroInfo( - [ - [ - FrameInfo( - [tasks.__file__, ANY, "sleep"] - ), - FrameInfo( - [ - script_name, - 38, - "echo_client", - ] - ), - ], - ANY, - ] - ) - ], - [ - CoroInfo( - [ - [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo( - [ - script_name, - 41, - "echo_client_spam", - ] - ), - ], - ANY, - ] - ) - ], - ] - ), - entries, - ) - - expected_awaited_by = [ - CoroInfo( - [ - [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo( - [script_name, 41, "echo_client_spam"] - ), - ], - ANY, - ] - ) - ] - tasks_with_awaited = [ - task - for task in entries - if task.awaited_by == expected_awaited_by - ] - self.assertGreaterEqual(len(tasks_with_awaited), 1000) - - # the final task will have some random number, but it should for - # sure be one of the echo client spam horde (In windows this is not true - # for some reason) - if sys.platform != "win32": - self.assertEqual( - tasks_with_awaited[-1].awaited_by, - entries[-1].awaited_by, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_self_trace(self): - stack_trace = get_stack_trace(os.getpid()) - # Is possible that there are more threads, so we check that the - # expected stack traces are in the result (looking at you Windows!) - this_tread_stack = None - for thread_id, stack in stack_trace: - if thread_id == threading.get_native_id(): - this_tread_stack = stack - break - self.assertIsNotNone(this_tread_stack) - self.assertEqual( - stack[:2], - [ - FrameInfo( - [ - __file__, - get_stack_trace.__code__.co_firstlineno + 2, - "get_stack_trace", - ] - ), - FrameInfo( - [ - __file__, - self.test_self_trace.__code__.co_firstlineno + 6, - "TestGetStackTrace.test_self_trace", - ] - ), - ], - ) - - @skip_if_not_supported - @unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", - ) - def test_only_active_thread(self): - # Test that only_active_thread parameter works correctly - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import time, sys, socket, threading - - # Connect to the test process - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - def worker_thread(name, barrier, ready_event): - barrier.wait() # Synchronize thread start - ready_event.wait() # Wait for main thread signal - # Sleep to keep thread alive - time.sleep(10_000) - - def main_work(): - # Do busy work to hold the GIL - sock.sendall(b"working\\n") - count = 0 - while count < 100000000: - count += 1 - if count % 10000000 == 0: - pass # Keep main thread busy - sock.sendall(b"done\\n") - - # Create synchronization primitives - num_threads = 3 - barrier = threading.Barrier(num_threads + 1) # +1 for main thread - ready_event = threading.Event() - - # Start worker threads - threads = [] - for i in range(num_threads): - t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) - t.start() - threads.append(t) - - # Wait for all threads to be ready - barrier.wait() - - # Signal ready to parent process - sock.sendall(b"ready\\n") - - # Signal threads to start waiting - ready_event.set() - - # Give threads time to start sleeping - time.sleep(0.1) - - # Now do busy work to hold the GIL - main_work() - """ - ) - - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - - # Wait for ready signal - response = b"" - while b"ready" not in response: - response += client_socket.recv(1024) - - # Wait for the main thread to start its busy work - while b"working" not in response: - response += client_socket.recv(1024) - - # Get stack trace with all threads - unwinder_all = RemoteUnwinder(p.pid, all_threads=True) - all_traces = unwinder_all.get_stack_trace() - - # Get stack trace with only GIL holder - unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) - gil_traces = unwinder_gil.get_stack_trace() - - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # Verify we got multiple threads in all_traces - self.assertGreater(len(all_traces), 1, "Should have multiple threads") - - # Verify we got exactly one thread in gil_traces - self.assertEqual(len(gil_traces), 1, "Should have exactly one GIL holder") - - # The GIL holder should be in the all_traces list - gil_thread_id = gil_traces[0][0] - all_thread_ids = [trace[0] for trace in all_traces] - self.assertIn(gil_thread_id, all_thread_ids, - "GIL holder should be among all threads") - - -if __name__ == "__main__": - unittest.main() From 99622fbff5bb350f45babd8c9f14a90c0251e427 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 28 Jun 2025 03:51:25 +0100 Subject: [PATCH 9/9] FIx race --- Lib/test/test_external_inspection.py | 1 - Modules/_remote_debugging_module.c | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index e29d44e209b6a4..0f31c225e68de3 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -5,7 +5,6 @@ import sys import socket import threading -import time from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry, requires_gil_enabled diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index b20b575ebc0ef8..ce7189637c2d69 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -2537,6 +2537,14 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return -1; } +#ifdef Py_GIL_DISABLED + if (only_active_thread) { + PyErr_SetString(PyExc_ValueError, + "only_active_thread is not supported when Py_GIL_DISABLED is not defined"); + return -1; + } +#endif + self->debug = debug; self->only_active_thread = only_active_thread; self->cached_state = NULL;