From cbe85e7c319b534f127c5235e973b694fef48312 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 20 Nov 2024 18:52:09 -0800 Subject: [PATCH 1/4] gh-127086: Ignore memory mmap in FileIO testing `mmap`, `munmap`, and `mprotect` are used by CPython for memory management, which may occur in the middle of the FileIO tests. The system calls can also be used with files, so `strace` includes them in its `%file` and `%desc` filters. Filter out the `mmap` system calls related to memory allocation for the file tests. Currently FileIO doesn't do `mmap` at all, so didn't add code to track from `mmap` through `munmap` since it wouldn't be used. For now if an `mmap` on a fd happens, the call will be included (which may cause test to fail), and at that time support for tracking the address throug `munmap` could be added. --- Lib/test/support/strace_helper.py | 34 ++++++++++++++++--- Lib/test/test_fileio.py | 8 +++-- ...-11-20-18-49-01.gh-issue-127076.DHnXxo.rst | 2 ++ 3 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index 7fb4581b2a7390..9317a045412c53 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -71,6 +71,27 @@ def sections(self): return sections +def filter_memory(syscalls): + """Filter out memory allocation calls from File I/O calls. + + Some calls (mmap, munmap, etc) can be used on files or to just get a block + of memory. Use this function to filter out the memory related calls from + other calls.""" + + def _filter(call): + # mmap can operate on a fd or `NULL` which gives a block of memory. + # Ignore the `NULL` ones. + if call.syscall == 'mmap' and call.args[0] == 'NULL': + return False + + if call.syscall in ('munmap', 'mprotect'): + return False + + return True + + return [call for call in syscalls if _filter(call)] + + @support.requires_subprocess() def strace_python(code, strace_flags, check=True): """Run strace and return the trace. @@ -92,8 +113,6 @@ def _make_error(reason, details): "-c", textwrap.dedent(code), __run_using_command=[_strace_binary] + strace_flags, - # Don't want to trace our JIT's own mmap and mprotect calls: - PYTHON_JIT="0", ) except OSError as err: return _make_error("Caught OSError", err) @@ -144,9 +163,14 @@ def get_events(code, strace_flags, prelude, cleanup): return all_sections['code'] -def get_syscalls(code, strace_flags, prelude="", cleanup=""): +def get_syscalls(code, strace_flags, prelude="", cleanup="", + ignore_memory=True): """Get the syscalls which a given chunk of python code generates""" events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup) + + if ignore_memory: + events = filter_memory(events) + return [ev.syscall for ev in events] @@ -169,5 +193,5 @@ def requires_strace(): return unittest.skipUnless(_can_strace(), "Requires working strace") -__all__ = ["get_events", "get_syscalls", "requires_strace", "strace_python", - "StraceEvent", "StraceResult"] +__all__ = ["filter_memory", "get_events", "get_syscalls", "requires_strace", + "strace_python", "StraceEvent", "StraceResult"] diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index d60aabcdf1ae22..e681417e15d34b 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -364,8 +364,7 @@ def testErrnoOnClosedReadinto(self, f): @strace_helper.requires_strace() def test_syscalls_read(self): - """Check that the set of system calls produced by the I/O stack is what - is expected for various read cases. + """Check set of system calls during common I/O patterns It's expected as bits of the I/O implementation change, this will need to change. The goal is to catch changes that unintentionally add @@ -383,6 +382,11 @@ def check_readall(name, code, prelude="", cleanup="", prelude=prelude, cleanup=cleanup) + # Some system calls (ex. mmap) can be used for both File I/O and + # memory allocation. Filter out the ones used for memory + # allocation. + syscalls = strace_helper.filter_memory(syscalls) + # The first call should be an open that returns a # file descriptor (fd). Afer that calls may vary. Once the file # is opened, check calls refer to it by fd as the filename diff --git a/Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst b/Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst new file mode 100644 index 00000000000000..39323604bbef56 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst @@ -0,0 +1,2 @@ +Filter out memory-related ``mmap``, ``munmap``, and ``mprotect`` calls from +file-related ones when testing :mod:`io` behavior using strace. From 57cd5dbafeaa058d7fe51e337b85d432a1df28d6 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 21 Nov 2024 16:23:28 -0800 Subject: [PATCH 2/4] Change to look for MAP_ANON --- Lib/test/support/strace_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index 9317a045412c53..deb5d288f4dd19 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -81,10 +81,10 @@ def filter_memory(syscalls): def _filter(call): # mmap can operate on a fd or `NULL` which gives a block of memory. # Ignore the `NULL` ones. - if call.syscall == 'mmap' and call.args[0] == 'NULL': + if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: return False - if call.syscall in ('munmap', 'mprotect'): + if call.syscall in ("munmap", "mprotect"): return False return True From 8670b31935672f53bd506f2faf8f266b3bb99632 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 21 Nov 2024 16:25:36 -0800 Subject: [PATCH 3/4] fixup comment --- Lib/test/support/strace_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index 5232b025933d8f..8aca8ef8e6d6cd 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -80,8 +80,8 @@ def filter_memory(syscalls): other calls.""" def _filter(call): - # mmap can operate on a fd or `NULL` which gives a block of memory. - # Ignore the `NULL` ones. + # mmap can operate on a fd or "MAP_ANON" which gives a block of memory. + # Ignore the "MAP_ANON" ones. if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: return False From e733fa0a31243be696f31ae10705a22e37d50f53 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Fri, 22 Nov 2024 00:19:50 -0800 Subject: [PATCH 4/4] Review fixes --- Lib/test/support/strace_helper.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index 8aca8ef8e6d6cd..eab16ea3e2889f 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -71,6 +71,17 @@ def sections(self): return sections +def _filter_memory_call(call): + # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory. + # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias. + if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: + return True + + if call.syscall in ("munmap", "mprotect"): + return True + + return False + def filter_memory(syscalls): """Filter out memory allocation calls from File I/O calls. @@ -79,18 +90,7 @@ def filter_memory(syscalls): of memory. Use this function to filter out the memory related calls from other calls.""" - def _filter(call): - # mmap can operate on a fd or "MAP_ANON" which gives a block of memory. - # Ignore the "MAP_ANON" ones. - if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: - return False - - if call.syscall in ("munmap", "mprotect"): - return False - - return True - - return [call for call in syscalls if _filter(call)] + return [call for call in syscalls if not _filter_memory_call(call)] @support.requires_subprocess()