diff --git a/.gitignore b/.gitignore index 00813f9d1b6b00..a0be74c9851204 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ Tools/unicode/data/ /dist/ /jit_stencils*.h /jit_unwind_info*.h +/trampoline_ehframe.c* .jit-stamp /platform /profile-clean-stamp diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h index 7099b88812ce7b..aec088ff4530e3 100644 --- a/Include/internal/pycore_jit_unwind.h +++ b/Include/internal/pycore_jit_unwind.h @@ -49,6 +49,33 @@ enum { DWRF_EH_PE_indirect = 0x80 }; +#if defined(PY_HAVE_PERF_TRAMPOLINE) +/* The perf trampoline's .eh_frame, extracted from the compiled trampoline + * object by Tools/jit/_trampoline_ehframe.py into trampoline_ehframe.c. + * The FDE's initial_location and address_range are zeroed placeholders, + * fde_field_size bytes each at fde_pc_offset and fde_range_offset, that + * _PyJitUnwind_BuildEhFrame() fills in. */ +typedef struct { + const uint8_t *data; + size_t size; + size_t fde_pc_offset; + size_t fde_range_offset; + size_t fde_field_size; +} _PyTrampolineEhFrame; + +PyAPI_DATA(const _PyTrampolineEhFrame) _Py_trampoline_ehframe; + +/* Copy eh's .eh_frame into buffer and fill in the FDE's initial_location + * and address_range for code_size bytes of code that perf maps right + * before the frame (see perf_jit_trampoline.c). Returns the number of + * bytes written, or 0 when the data is absent (the bootstrap programs' + * stub), inconsistent, larger than the buffer, or when code_size does not + * fit the offsets. Export for '_testinternalcapi'. */ +PyAPI_FUNC(size_t) _PyJitUnwind_PatchTrampolineEhFrame( + const _PyTrampolineEhFrame *eh, uint8_t *buffer, size_t buffer_size, + size_t code_size); +#endif + /* Return the size of the generated .eh_frame data for the given encoding. */ size_t _PyJitUnwind_EhFrameSize(int absolute_addr); diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index a2e67726e95959..54f4ddc57e2605 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -1,3 +1,5 @@ +import glob +import struct import unittest import string import subprocess @@ -11,7 +13,9 @@ assert_python_failure, assert_python_ok, ) +from test.support import import_helper from test.support.os_helper import temp_dir +from test import test_tools if not support.has_subprocess_support: @@ -685,5 +689,266 @@ def tearDown(self) -> None: file.unlink() +JITDUMP_MAGIC = 0x4A695444 # "JiTD" +JITDUMP_VERSION = 1 +PERF_LOAD = 0 +PERF_UNWINDING_INFO = 4 +JITDUMP_ENDIAN = "<" if sys.byteorder == "little" else ">" +# Every jitdump record starts with event(u32), size(u32), timestamp(u64). +JITDUMP_RECORD_HEADER_SIZE = 16 +# CodeLoadEvent: record header, pid(u32), tid(u32), vma(u64), code_addr(u64), +# code_size(u64), code_id(u64), then the NUL-terminated name. +CODE_LOAD_CODE_SIZE_OFFSET = JITDUMP_RECORD_HEADER_SIZE + 4 + 4 + 8 + 8 +CODE_LOAD_NAME_OFFSET = CODE_LOAD_CODE_SIZE_OFFSET + 8 + 8 +# CodeUnwindingInfoEvent: record header, unwind_data_size(u64), +# eh_frame_hdr_size(u64), mapped_size(u64), then the .eh_frame bytes followed +# by perf's 20-byte eh_frame_hdr (EhFrameHeader in perf_jit_trampoline.c), +# whose "from" field is the signed distance back to the code. +UNWIND_DATA_SIZE_OFFSET = JITDUMP_RECORD_HEADER_SIZE +UNWIND_EH_FRAME_OFFSET = JITDUMP_RECORD_HEADER_SIZE + 3 * 8 +EH_FRAME_HDR_SIZE = 20 +EH_FRAME_HDR_FROM_OFFSET = 12 +# DWARF FDE pointer encodings: DW_EH_PE_pcrel | DW_EH_PE_sdata4 (ELF +# assemblers) and DW_EH_PE_pcrel | DW_EH_PE_absptr (Darwin assemblers). +DW_EH_PE_PCREL_SDATA4 = 0x1B +DW_EH_PE_PCREL_ABSPTR = 0x10 + + +def _jitdump_records(data): + """Yield (event, offset, size) for each record of a jitdump file.""" + header_size = struct.unpack_from(f"{JITDUMP_ENDIAN}I", data, 8)[0] + pos = header_size + while pos < len(data): + if pos + JITDUMP_RECORD_HEADER_SIZE > len(data): + raise ValueError(f"truncated record header at offset {pos}") + event, size = struct.unpack_from(f"{JITDUMP_ENDIAN}II", data, pos) + if size < JITDUMP_RECORD_HEADER_SIZE or pos + size > len(data): + raise ValueError(f"record at offset {pos} has a bad size {size}") + yield event, pos, size + pos += size + + +def _fde_pointer_encoding(eh_frame): + """Return the FDE pointer encoding byte of a version 1 "zR" CIE.""" + cie_length = struct.unpack_from(f"{JITDUMP_ENDIAN}I", eh_frame, 0)[0] + cie_total = 4 + cie_length + pos = 12 # past length, CIE_id, version and "zR\0" + for _ in range(2): # code and data alignment factors (LEB128) + while eh_frame[pos] & 0x80: + pos += 1 + pos += 1 + pos += 1 # return address column + while eh_frame[pos] & 0x80: # augmentation data length (LEB128) + pos += 1 + pos += 1 + if pos >= cie_total: + raise ValueError("truncated CIE augmentation data") + return eh_frame[pos] + + +@unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") +class TestJitdumpFileFormat(unittest.TestCase): + """Validate the jitdump written by -Xperf_jit without requiring perf.""" + + def _run_and_get_jitdump(self, code): + # The child prints its pid so we open exactly its own jitdump file + # rather than whatever another test worker left in /tmp. + code = "import os, sys\nsys.stdout.write(str(os.getpid()))\n" + code + rc, out, err = assert_python_ok("-Xperf_jit", "-c", code, PYTHON_JIT="0") + path = f"/tmp/jit-{int(out)}.dump" + if not os.path.exists(path): + # perf_map_jit_init() gives up silently when it cannot create + # the file (for example an unwritable /tmp). + self.skipTest("jitdump file was not created") + self.addCleanup(os.unlink, path) + with open(path, "rb") as f: + data = f.read() + if not data: + # The file is created before the executable mapping of the + # jitdump; if that mapping fails (for example a noexec /tmp) the + # backend gives up silently and never writes the header. + self.skipTest("jitdump could not be initialized") + return data + + def _check_unwinding_records(self, data): + """Check every unwinding record against the code load record it + describes; return {name: code_size} for the regions seen.""" + records = list(_jitdump_records(data)) + regions = {} + for index, (event, pos, size) in enumerate(records): + if event != PERF_UNWINDING_INFO: + continue + # The unwinding info record is immediately followed by the + # code load record it describes. + self.assertLess(index + 1, len(records)) + load_event, load_pos, load_size = records[index + 1] + self.assertEqual(load_event, PERF_LOAD) + code_size = struct.unpack_from( + f"{JITDUMP_ENDIAN}Q", data, load_pos + CODE_LOAD_CODE_SIZE_OFFSET)[0] + name_start = load_pos + CODE_LOAD_NAME_OFFSET + name_end = data.find(b"\x00", name_start, load_pos + load_size) + self.assertGreater(name_end, 0) + name = data[name_start:name_end].decode("utf-8", errors="replace") + # The machine code follows the name inside the load record. + self.assertLessEqual(name_end + 1 + code_size, load_pos + load_size, name) + unwind_data_size, eh_frame_hdr_size = struct.unpack_from( + f"{JITDUMP_ENDIAN}QQ", data, pos + UNWIND_DATA_SIZE_OFFSET) + self.assertEqual(eh_frame_hdr_size, EH_FRAME_HDR_SIZE) + self.assertLessEqual(UNWIND_EH_FRAME_OFFSET + unwind_data_size, size) + eh_frame_size = unwind_data_size - eh_frame_hdr_size + self.assertGreater(eh_frame_size, 0) + start = pos + UNWIND_EH_FRAME_OFFSET + eh_frame = data[start:start + eh_frame_size] + + cie_length, cie_id = struct.unpack_from(f"{JITDUMP_ENDIAN}II", eh_frame, 0) + self.assertEqual(cie_id, 0, "first entry must be a CIE") + self.assertEqual(eh_frame[8], 1, "CIE version must be 1") + self.assertEqual(eh_frame[9:12], b"zR\x00") + encoding = _fde_pointer_encoding(eh_frame) + if encoding == DW_EH_PE_PCREL_SDATA4: + fields = "iI" + elif encoding == DW_EH_PE_PCREL_ABSPTR: + fields = "qQ" + else: + self.fail(f"unexpected FDE pointer encoding {encoding:#x}") + # jit_unwind.c patches initial_location and address_range for + # perf's DSO layout, where the .eh_frame follows the code at + # code_size rounded up to 8 bytes. + pc_offset = 4 + cie_length + 8 + initial_location, address_range = struct.unpack_from( + f"{JITDUMP_ENDIAN}{fields}", eh_frame, pc_offset) + self.assertEqual(address_range, code_size, name) + rounded_code_size = (code_size + 7) & ~7 + self.assertEqual(initial_location, -(rounded_code_size + pc_offset), name) + # perf's eh_frame_hdr must point back at the code with the same + # rounding as the FDE. + hdr_from = struct.unpack_from( + f"{JITDUMP_ENDIAN}i", data, + start + eh_frame_size + EH_FRAME_HDR_FROM_OFFSET)[0] + self.assertEqual(hdr_from, -(rounded_code_size + eh_frame_size), name) + regions[name] = code_size + self.assertTrue(regions, "no CodeUnwindingInfoEvent found") + return regions + + def test_jitdump_unwinding_info(self): + """Each region's .eh_frame is patched for that region's size.""" + data = self._run_and_get_jitdump("def my_test_func(): pass\nmy_test_func()") + magic, version = struct.unpack_from(f"{JITDUMP_ENDIAN}II", data, 0) + self.assertEqual((magic, version), (JITDUMP_MAGIC, JITDUMP_VERSION)) + regions = self._check_unwinding_records(data) + self.assertTrue(any("my_test_func" in name for name in regions)) + + +try: + with test_tools.imports_under_tool("jit"): + import _trampoline_ehframe +except ImportError: + # Installed Python without the Tools directory. + _trampoline_ehframe = None + +try: + from test.test_tools.test_trampoline_ehframe import fake_cie, fake_fde +except (ImportError, unittest.SkipTest): + # Installed Python without the Tools directory. + fake_cie = fake_fde = None + + +@unittest.skipIf(_trampoline_ehframe is None, + "Tools/jit/_trampoline_ehframe.py not found") +class TestTrampolineEhframeScript(unittest.TestCase): + """The generator against this build's trampoline object. The parsers are + tested with synthetic objects in test.test_tools.test_trampoline_ehframe.""" + + ehframe = _trampoline_ehframe + + def _build_trampoline_objects(self): + """The object(s) the Makefile fed to the generator.""" + builddir = sysconfig.get_config_var("abs_builddir") or "." + universal2 = os.path.join(builddir, "Python", "asm_trampoline_universal2.o") + if os.path.exists(universal2): + return [universal2] + return sorted( + path for path in glob.glob( + os.path.join(builddir, "Python", "asm_trampoline_*.o")) + if "apple-darwin" not in os.path.basename(path)) + + def test_generated_source_is_current(self): + """The C file in the build directory matches a fresh generation.""" + objects = self._build_trampoline_objects() + builddir = sysconfig.get_config_var("abs_builddir") or "." + source = os.path.join(builddir, "trampoline_ehframe.c") + if not objects or not os.path.exists(source): + self.skipTest("trampoline object or generated C file not found") + with open(source) as f: + current = f.read() + with temp_dir() as tmp: + fresh_path = os.path.join(tmp, "trampoline_ehframe.c") + self.ehframe.generate(objects, fresh_path) + with open(fresh_path) as f: + fresh = f.read() + self.assertEqual(current, fresh) + + +class TestTrampolineEhframeData(unittest.TestCase): + """Checks on the linked trampoline_ehframe.c data and the runtime patching.""" + + def setUp(self): + self.capi = import_helper.import_module("_testinternalcapi") + if not hasattr(self.capi, "test_trampoline_ehframe"): + self.skipTest("_testinternalcapi built without the perf trampoline") + + def test_generated_data_structure(self): + # Raises AssertionError describing the first failed check. + self.capi.test_trampoline_ehframe() + + @unittest.skipIf(fake_cie is None, "test_tools.test_trampoline_ehframe not importable") + def test_patch_both_widths(self): + patch = self.capi.patch_trampoline_ehframe + for encoding, field_size in ((DW_EH_PE_PCREL_SDATA4, 4), + (DW_EH_PE_PCREL_ABSPTR, 8)): + cie = fake_cie(encoding=encoding) + data = cie + fake_fde(len(cie), field_size=field_size) + pc = len(cie) + 8 + rng = pc + field_size + for code_size in (1, 8, 9, 4096): + with self.subTest(field_size=field_size, code_size=code_size): + out = patch(data, pc, rng, field_size, code_size, 1024) + self.assertEqual(len(out), len(data)) + rounded = (code_size + 7) & ~7 + self.assertEqual( + int.from_bytes(out[pc:rng], sys.byteorder, signed=True), + -(rounded + pc)) + self.assertEqual( + int.from_bytes(out[rng:rng + field_size], sys.byteorder), + code_size) + self.assertEqual(out[:pc] + out[rng + field_size:], + data[:pc] + data[rng + field_size:]) + + @unittest.skipIf(fake_cie is None, "test_tools.test_trampoline_ehframe not importable") + def test_patch_rejects_bad_input(self): + patch = self.capi.patch_trampoline_ehframe + cie = fake_cie() + data = cie + fake_fde(len(cie)) + pc = len(cie) + 8 + rng = pc + 4 + # The largest code size whose offsets still fit a signed 32-bit field. + limit = 2**31 - 1 - 8 - len(data) + self.assertIsNotNone(patch(data, pc, rng, 4, limit, 1024)) + cases = { + "empty data, as in the bootstrap stub": (b"", 8, 12, 4, 8, 1024), + "buffer too small": (data, pc, rng, 4, 8, len(data) - 1), + "bad field size": (data, pc, rng, 2, 8, 1024), + "fields not adjacent": (data, pc, rng + 4, 4, 8, 1024), + "fields past the end": (data, len(data) - 4, len(data), 4, 8, 1024), + "zero code size": (data, pc, rng, 4, 0, 1024), + "code size past INT32_MAX": (data, pc, rng, 4, limit + 1, 1024), + } + for name, args in cases.items(): + with self.subTest(name): + self.assertIsNone(patch(*args)) + with self.assertRaises(ValueError): + patch(data, -1, rng, 4, 8, 1024) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tools/test_trampoline_ehframe.py b/Lib/test/test_tools/test_trampoline_ehframe.py new file mode 100644 index 00000000000000..66dc154f7fe7eb --- /dev/null +++ b/Lib/test/test_tools/test_trampoline_ehframe.py @@ -0,0 +1,344 @@ +"""Tests for Tools/jit/_trampoline_ehframe.py with synthetic object files. + +The generator's output for this build's own trampoline object, and the +runtime that patches it, are tested in test.test_perf_profiler. +""" + +import os +import struct +import unittest + +from test.support import os_helper +from test.support.script_helper import assert_python_failure, assert_python_ok +from test.test_tools import imports_under_tool, skip_if_missing, toolsdir + +skip_if_missing("jit") +with imports_under_tool("jit"): + import _trampoline_ehframe as ehframe + +SCRIPT = os.path.join(toolsdir, "jit", "_trampoline_ehframe.py") +PCREL_SDATA4 = ehframe._DW_EH_PE_pcrel | ehframe._DW_EH_PE_sdata4 +PCREL_ABSPTR = ehframe._DW_EH_PE_pcrel | ehframe._DW_EH_PE_absptr + + +def fake_cie(*, version=1, augmentation=b"zR", ra_column=16, + encoding=PCREL_SDATA4, cie_id=0, endian="<"): + """A CIE like the assembler's: code align 1, data align -8, one + DW_CFA_def_cfa instruction, padded with DW_CFA_nop to 8 bytes.""" + body = bytes([version]) + augmentation + b"\x00" + body += bytes([1, 0x78, ra_column, 1, encoding]) + body += bytes([0x0C, 7, 8]) # DW_CFA_def_cfa: r7 (rsp) ofs 8 + body += b"\x00" * (-(8 + len(body)) % 8) + return struct.pack(f"{endian}II", 4 + len(body), cie_id) + body + + +def fake_fde(cie_total, *, field_size=4, address_range=8, + instructions=b"\x41\x0e\x10\x86\x02", endian="<"): + """An FDE right after a CIE of cie_total bytes, padded to 8 bytes.""" + byteorder = "little" if endian == "<" else "big" + body = struct.pack(f"{endian}I", cie_total + 4) # CIE pointer, relative to itself + # initial_location as an assembler would leave it, the parser zeroes it. + body += (-40).to_bytes(field_size, byteorder, signed=True) + body += address_range.to_bytes(field_size, byteorder) + body += b"\x00" # augmentation data length + body += instructions + body += b"\x00" * (-(4 + len(body)) % 8) + return struct.pack(f"{endian}I", len(body)) + body + + +def fake_ehframe(text_size=8, *, field_size=4, endian="<"): + """A CIE and an FDE for text_size bytes of code.""" + encoding = PCREL_SDATA4 if field_size == 4 else PCREL_ABSPTR + cie = fake_cie(encoding=encoding, endian=endian) + return cie + fake_fde(len(cie), field_size=field_size, + address_range=text_size, endian=endian) + + +def fake_elf(*, endian="<", e_machine=ehframe._EM_X86_64, text=b"\x55\xc3", + eh_frame=None, text_type=1, extra_sections=()): + """A minimal ELF64 relocatable object with .text, .eh_frame and .shstrtab.""" + E = ehframe + if eh_frame is None: + eh_frame = fake_ehframe(len(text), endian=endian) + section_list = [(b".text", text_type, text), (b".eh_frame", 1, eh_frame)] + section_list += [(name, 1, data) for name, data in extra_sections] + shstrtab = (b"\x00" + b"".join(name + b"\x00" for name, _, _ in section_list) + + b".shstrtab\x00") + section_list.append((b".shstrtab", 3, shstrtab)) + offset = E._ELF64_HEADER_SIZE + blobs = b"" + headers = [b"\x00" * E._ELF64_SECTION_HEADER_SIZE] # the null section + for name, sh_type, data in section_list: + # Elf64_Shdr: name, type, flags, addr, offset, size, link, info, + # addralign, entsize. + headers.append(struct.pack(f"{endian}IIQQQQIIQQ", + shstrtab.index(name + b"\x00"), sh_type, + 0, 0, offset, len(data), 0, 0, 1, 0)) + blobs += data + offset += len(data) + data_encoding = E._ELFDATA2LSB if endian == "<" else E._ELFDATA2MSB + ident = b"\x7fELF" + bytes([E._ELFCLASS64, data_encoding, 1, 0]) + bytes(8) + # Elf64_Ehdr after e_ident: type, machine, version, entry, phoff, shoff, + # flags, ehsize, phentsize, phnum, shentsize, shnum, shstrndx. + header = ident + struct.pack(f"{endian}HHIQQQIHHHHHH", 1, e_machine, 1, 0, 0, + offset, 0, E._ELF64_HEADER_SIZE, 0, 0, + E._ELF64_SECTION_HEADER_SIZE, len(headers), + len(headers) - 1) + return header + blobs + b"".join(headers) + + +def fake_macho(cputype, text, eh_frame, *, text_flags=0): + """A minimal MH_OBJECT: one __TEXT segment with __text and __eh_frame + sections, section data right after the load command.""" + E = ehframe + segment_size = E._MACHO64_SEGMENT_COMMAND_SIZE + 2 * E._MACHO64_SECTION_SIZE + text_offset = E._MACHO64_HEADER_SIZE + segment_size + eh_offset = text_offset + len(text) + sections = b"" + for name, size, offset, flags in (("__text", len(text), text_offset, text_flags), + ("__eh_frame", len(eh_frame), eh_offset, 0)): + sections += struct.pack("<16s16sQQIIIIIIII", name.encode(), b"__TEXT", + 0, size, offset, 0, 0, 0, flags, 0, 0, 0) + segment = struct.pack("II", magic, len(blobs)) + entries + body + + +def load(blob): + """Run load_object() on an object given as bytes.""" + with os_helper.temp_dir() as tmp: + path = os.path.join(tmp, "object.o") + with open(path, "wb") as f: + f.write(blob) + return ehframe.load_object(path) + + +class TestParseEhframe(unittest.TestCase): + def parse(self, data, text_size=8, endian="<"): + return ehframe.parse_ehframe(bytes(data), endian, text_size) + + def test_parse(self): + """Both FDE pointer encodings, in both byte orders.""" + cases = [(PCREL_SDATA4, 4, 8), (PCREL_ABSPTR, 8, 20)] + for encoding, field_size, text_size in cases: + for endian in ("<", ">"): + with self.subTest(encoding=hex(encoding), endian=endian): + cie = fake_cie(encoding=encoding, endian=endian) + fde = fake_fde(len(cie), field_size=field_size, + address_range=text_size, endian=endian) + result = self.parse(cie + fde, text_size, endian) + self.assertEqual(result.field_size, field_size) + self.assertEqual(result.fde_pc_offset, len(cie) + 8) + self.assertEqual(result.fde_range_offset, + len(cie) + 8 + field_size) + # Both patchable fields zeroed, everything else untouched. + expected = bytearray(cie + fde) + start = len(cie) + 8 + expected[start:start + 2 * field_size] = bytes(2 * field_size) + self.assertEqual(result.data, bytes(expected)) + + def test_parse_rejects_malformed(self): + cie = fake_cie() + fde = fake_fde(len(cie)) + cases = [ + ("version", fake_cie(version=3) + fde, 8), + ("augmentation", fake_cie(augmentation=b"zPLR") + fde, 8), + ("encoding", fake_cie(encoding=0x1A) + fde, 8), + ("exactly one FDE", cie + fde + fde, 8), + ("address_range", cie + fde, 12), + ("no FDE", cie, 8), + ("no CIE", b"", 8), + ("bad CIE length", b"\xff\xff\xff\xff" + cie[4:] + fde, 8), + ("empty", cie + fake_fde(len(cie), address_range=0), 0), + ("larger than", + cie + fake_fde(len(cie), instructions=b"\x00" * 1100), 8), + ] + for message, data, text_size in cases: + with self.subTest(message): + with self.assertRaisesRegex(ValueError, message): + self.parse(data, text_size) + + +class TestElfObjects(unittest.TestCase): + def test_both_byte_orders(self): + cases = [("<", ehframe._EM_X86_64, "__x86_64__"), + (">", ehframe._EM_AARCH64, "__aarch64__")] + for endian, e_machine, macro in cases: + with self.subTest(endian=endian): + (obj,) = load(fake_elf(endian=endian, e_machine=e_machine)) + self.assertEqual(obj.endian, endian) + self.assertEqual(obj.arch_macro, macro) + self.assertEqual(obj.sections[".text"], b"\x55\xc3") + frame = ehframe.build_ehframe(obj) + cie_size = len(fake_cie(endian=endian)) + self.assertEqual( + (frame.fde_pc_offset, frame.fde_range_offset, frame.field_size), + (cie_size + 8, cie_size + 12, 4)) + self.assertEqual(frame.data[cie_size + 8:cie_size + 16], bytes(8)) + + def test_rejects_malformed(self): + cases = [ + ("unsupported ELF machine", fake_elf(e_machine=243)), + ("no contents", fake_elf(text_type=ehframe._SHT_NOBITS)), + ("more than one .text", fake_elf(extra_sections=((b".text", b"\x90"),))), + ("truncated ELF header", fake_elf()[:40]), + ("section headers extend", fake_elf()[:-8]), + ("not an ELF64", b"\x7fELF\x01" + bytes(59)), + ("empty", fake_elf(text=b"", eh_frame=fake_ehframe(0))), + ] + for message, blob in cases: + with self.subTest(message): + with self.assertRaisesRegex(ValueError, message): + for obj in load(blob): + ehframe.build_ehframe(obj) + + +class TestMachoObjects(unittest.TestCase): + X86 = ehframe._CPU_TYPE_X86_64 + ARM64 = ehframe._CPU_TYPE_ARM64 + + def test_thin_and_fat(self): + """Mach-O objects and both fat container formats.""" + x86 = fake_macho(self.X86, b"\x55\xc3", b"x86 eh_frame") + arm = fake_macho(self.ARM64, b"\xc0\x03\x5f\xd6", b"arm64 eh_frame") + (thin,) = load(arm) + self.assertEqual(thin.arch_macro, "__aarch64__") + self.assertEqual(thin.sections[".text"], b"\xc0\x03\x5f\xd6") + self.assertEqual(thin.sections[".eh_frame"], b"arm64 eh_frame") + for magic in (ehframe._FAT_MAGIC, ehframe._FAT_MAGIC_64): + with self.subTest(magic=hex(magic)): + fat = fake_fat([(self.X86, x86), (self.ARM64, arm)], magic=magic) + slices = load(fat) + self.assertEqual([s.arch_macro for s in slices], + ["__x86_64__", "__aarch64__"]) + self.assertEqual(slices[0].sections[".eh_frame"], b"x86 eh_frame") + self.assertEqual(slices[1].sections[".text"], b"\xc0\x03\x5f\xd6") + + def test_rejects_malformed(self): + text, frame = b"\x55\xc3", b"eh_frame" + good = fake_macho(self.X86, text, frame) + header_size = ehframe._MACHO64_HEADER_SIZE + segment_size = ehframe._MACHO64_SEGMENT_COMMAND_SIZE + + def patched(offset, value): + return good[:offset] + struct.pack(" 0, "no trampoline .eh_frame is linked in"); + CHECK(field_size == 4 || field_size == 8, "unsupported FDE field size"); + CHECK(size >= 8 + 8 + 2 * field_size + 1, "data too small for a CIE and an FDE"); + + /* CIE: length, CIE_id == 0, version 1, augmentation "zR". */ + uint32_t cie_length; + memcpy(&cie_length, data, sizeof(cie_length)); + CHECK(cie_length > 0 && cie_length < size, "bad CIE length"); + + uint32_t cie_id; + memcpy(&cie_id, data + 4, sizeof(cie_id)); + CHECK(cie_id == 0, "first entry is not a CIE"); + + CHECK(data[8] == 1, "CIE version is not 1"); + CHECK(data[9] == 'z' && data[10] == 'R' && data[11] == '\0', + "CIE augmentation is not \"zR\""); + + /* Exactly one FDE must follow the CIE, and the walk below must stay + * inside the data. */ + size_t cie_total = 4 + cie_length; + CHECK(cie_total + 8 + 2 * field_size <= size, + "no room for an FDE after the CIE"); + + /* FDE pointer encoding byte: skip version(1), "zR\0"(3), code_align + * (ULEB128), data_align (SLEB128), RA column (1 byte in version 1), + * augmentation data length (ULEB128). */ + size_t pos = 12; + while (pos < cie_total && (data[pos] & 0x80)) { /* code alignment */ + pos++; + } + pos++; + while (pos < cie_total && (data[pos] & 0x80)) { /* data alignment */ + pos++; + } + pos++; + pos++; /* RA column */ + /* Augmentation data length (ULEB128) must be 1: the encoding byte. */ + uint32_t aug_length = 0; + int shift = 0; + while (pos < cie_total && (data[pos] & 0x80)) { + CHECK(shift < 28, "CIE augmentation data length is too long"); + aug_length |= (uint32_t)(data[pos] & 0x7f) << shift; + shift += 7; + pos++; + } + CHECK(pos < cie_total, "CIE augmentation data length is truncated"); + CHECK((uint32_t)(data[pos] & 0x7f) <= (UINT32_MAX >> shift), + "CIE augmentation data length overflows"); + aug_length |= (uint32_t)(data[pos] & 0x7f) << shift; + pos++; + CHECK(aug_length == 1 && pos < cie_total, + "CIE augmentation data length is not 1"); + uint8_t fde_enc = data[pos]; + CHECK(fde_enc == (DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4) + || fde_enc == (DWRF_EH_PE_pcrel | DWRF_EH_PE_absptr), + "unsupported FDE pointer encoding"); + size_t enc_field_size = (fde_enc & 0x0f) == DWRF_EH_PE_sdata4 ? 4 : 8; + CHECK(enc_field_size == field_size, + "FDE field size does not match the CIE pointer encoding"); + + /* The FDE ends exactly at the end of the data. */ + uint32_t fde_length; + memcpy(&fde_length, data + cie_total, sizeof(fde_length)); + CHECK(fde_length > 0, "FDE length is zero"); + CHECK(cie_total + 4 + fde_length == size, + "FDE does not end at the end of the data"); + + /* The CIE pointer is the distance from its own field back to the CIE. */ + uint32_t fde_cie_ptr; + memcpy(&fde_cie_ptr, data + cie_total + 4, sizeof(fde_cie_ptr)); + CHECK(fde_cie_ptr == cie_total + 4, "FDE CIE pointer does not point at the CIE"); + + /* The runtime patches initial_location and address_range at the + * recorded offsets. They must be the two fields after the CIE pointer + * and must be zeroed placeholders in the data. */ + CHECK(pc_offset == cie_total + 8, "FDE initial_location offset is wrong"); + CHECK(range_offset == pc_offset + field_size, + "FDE address_range offset is wrong"); + for (size_t i = 0; i < 2 * field_size; i++) { + CHECK(data[pc_offset + i] == 0, "FDE placeholder fields are not zero"); + } + /* The FDE's own augmentation data length must follow and be 0. */ + size_t fde_aug_offset = range_offset + field_size; + CHECK(fde_aug_offset < size, "FDE augmentation data length byte is missing"); + CHECK(data[fde_aug_offset] == 0, "FDE augmentation data length is not 0"); + +#undef CHECK + Py_RETURN_NONE; +} + +/* Run the runtime's FDE patching on caller-supplied data, so the tests can + * cover both field widths and the rejection paths on any platform. + * Returns the patched bytes, or None when the runtime refuses the input. */ +static PyObject * +patch_trampoline_ehframe(PyObject *self, PyObject *args) +{ + Py_buffer data; + Py_ssize_t pc_offset, range_offset, field_size, code_size, buffer_size; + if (!PyArg_ParseTuple(args, "y*nnnnn", &data, &pc_offset, &range_offset, + &field_size, &code_size, &buffer_size)) { + return NULL; + } + PyObject *result = NULL; + uint8_t *buffer = NULL; + if (pc_offset < 0 || range_offset < 0 || field_size < 0 || code_size < 0 + || buffer_size < 0) { + PyErr_SetString(PyExc_ValueError, "arguments must not be negative"); + goto done; + } + buffer = PyMem_Malloc(buffer_size > 0 ? (size_t)buffer_size : 1); + if (buffer == NULL) { + PyErr_NoMemory(); + goto done; + } + _PyTrampolineEhFrame eh = { + data.buf, (size_t)data.len, (size_t)pc_offset, (size_t)range_offset, + (size_t)field_size, + }; + size_t written = _PyJitUnwind_PatchTrampolineEhFrame( + &eh, buffer, (size_t)buffer_size, (size_t)code_size); + if (written == 0) { + result = Py_NewRef(Py_None); + } + else { + result = PyBytes_FromStringAndSize((const char *)buffer, + (Py_ssize_t)written); + } +done: + PyMem_Free(buffer); + PyBuffer_Release(&data); + return result; +} +#endif /* PY_HAVE_PERF_TRAMPOLINE */ + static PyObject * compile_perf_trampoline_entry(PyObject *self, PyObject *args) { @@ -3353,6 +3511,10 @@ static PyMethodDef module_functions[] = { {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, +#if defined(PY_HAVE_PERF_TRAMPOLINE) + {"test_trampoline_ehframe", test_trampoline_ehframe, METH_NOARGS}, + {"patch_trampoline_ehframe", patch_trampoline_ehframe, METH_VARARGS}, +#endif {"get_crossinterp_data", _PyCFunction_CAST(get_crossinterp_data), METH_VARARGS | METH_KEYWORDS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6443d814a22dab..458fa7f02a2068 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -1,13 +1,14 @@ /* Frozen modules bootstrap * - * Limited and restricted Python interpreter to run - * "Tools/build/deepfreeze.py" on systems with no or older Python - * interpreter. + * Limited and restricted Python interpreter to run the build's own + * scripts, Programs/_freeze_module.py and Tools/jit/_trampoline_ehframe.py, + * on systems with no or an older Python interpreter. */ #include "Python.h" #include "pycore_import.h" +#include "pycore_jit_unwind.h" // _PyTrampolineEhFrame /* Includes for frozen modules: */ #include "Python/frozen_modules/importlib._bootstrap.h" @@ -45,6 +46,15 @@ const struct _module_alias *_PyImport_FrozenAliases = aliases; const struct _frozen *PyImport_FrozenModules = NULL; +#ifdef PY_HAVE_PERF_TRAMPOLINE +/* Python/jit_unwind.o references the trampoline's unwind data, but + trampoline_ehframe.c is generated later in the build, by _bootstrap_python, + so it cannot be linked in here. The build never activates the perf + trampoline in this program, and with this stub activating it would + only write no unwind data. */ +const _PyTrampolineEhFrame _Py_trampoline_ehframe = {NULL, 0, 0, 0, 0}; +#endif + int #ifdef MS_WINDOWS wmain(int argc, wchar_t **argv) diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 27a60171f3eca8..ecd207c07780d0 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -13,6 +13,7 @@ #include #include "pycore_fileutils.h" // _Py_stat_struct #include +#include "pycore_jit_unwind.h" // _PyTrampolineEhFrame #include #include // malloc() @@ -39,6 +40,15 @@ const struct _frozen *_PyImport_FrozenTest; const struct _frozen *PyImport_FrozenModules; const struct _module_alias *_PyImport_FrozenAliases; +#ifdef PY_HAVE_PERF_TRAMPOLINE +/* Python/jit_unwind.o references the trampoline's unwind data, but + trampoline_ehframe.c is generated later in the build, by _bootstrap_python, + so it cannot be linked in here. The build never activates the perf + trampoline in this program, and with this stub activating it would + only write no unwind data. */ +const _PyTrampolineEhFrame _Py_trampoline_ehframe = {NULL, 0, 0, 0, 0}; +#endif + static const char header[] = "/* Auto-generated by Programs/_freeze_module.c */"; diff --git a/Python/asm_trampoline_aarch64.S b/Python/asm_trampoline_aarch64.S index b3aeb728de200c..883c3f1b0394a8 100644 --- a/Python/asm_trampoline_aarch64.S +++ b/Python/asm_trampoline_aarch64.S @@ -44,13 +44,28 @@ __Py_trampoline_func_start: .globl _Py_trampoline_func_start _Py_trampoline_func_start: #endif + .cfi_startproc SIGN_LR +#if defined(__ARM_FEATURE_PAC_DEFAULT) + .cfi_negate_ra_state +#endif stp x29, x30, [sp, -16]! + .cfi_def_cfa_offset 16 + .cfi_offset x29, -16 + .cfi_offset x30, -8 mov x29, sp + .cfi_def_cfa_register x29 blr x3 ldp x29, x30, [sp], 16 + .cfi_restore x30 + .cfi_restore x29 + .cfi_def_cfa sp, 0 VERIFY_LR +#if defined(__ARM_FEATURE_PAC_DEFAULT) + .cfi_negate_ra_state +#endif ret + .cfi_endproc #if defined(__APPLE__) .globl __Py_trampoline_func_end __Py_trampoline_func_end: diff --git a/Python/asm_trampoline_x86_64.S b/Python/asm_trampoline_x86_64.S index 0e6b11589eafc8..999728653aa534 100644 --- a/Python/asm_trampoline_x86_64.S +++ b/Python/asm_trampoline_x86_64.S @@ -7,14 +7,20 @@ __Py_trampoline_func_start: .globl _Py_trampoline_func_start _Py_trampoline_func_start: #endif + .cfi_startproc #if defined(__CET__) && (__CET__ & 1) endbr64 #endif push %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 mov %rsp, %rbp + .cfi_def_cfa_register %rbp call *%rcx pop %rbp + .cfi_def_cfa %rsp, 8 ret + .cfi_endproc #if defined(__APPLE__) .globl __Py_trampoline_func_end __Py_trampoline_func_end: diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c index 0941ed593ff7d1..4c64dfe3afb9a1 100644 --- a/Python/jit_unwind.c +++ b/Python/jit_unwind.c @@ -1,8 +1,8 @@ /* * Python JIT - DWARF .eh_frame builder * - * This file contains the DWARF CFI generator used to build .eh_frame - * data for JIT code (perf jitdump and other unwinders). + * Builds the .eh_frame data attached to JIT code regions, for the perf + * jitdump (from trampoline_ehframe.c) and for the GDB JIT interface. */ #include "Python.h" @@ -35,6 +35,33 @@ void __deregister_frame(const void *); #include #include + +// ============================================================================= +// ELF OBJECT CONTEXT +// ============================================================================= + +/* + * Context for building ELF/DWARF structures + * + * This structure maintains state while constructing DWARF unwind information. + * It acts as a simple buffer manager with pointers to track current position + * and important landmarks within the buffer. + */ +typedef struct ELFObjectContext { + uint8_t* p; // Current write position in buffer + uint8_t* startp; // Start of buffer (for offset calculations) + uintptr_t code_addr; // Address of the code section + size_t code_size; // Size of the code section +} ELFObjectContext; + +// ============================================================================= +// DWARF GENERATION UTILITIES +// ============================================================================= + +/* The perf path copies pre-built .eh_frame bytes (see elf_init_ehframe_perf), + * so the DWARF constants and writers below are only needed by the GDB path. */ +#if defined(PY_HAVE_JIT_GDB_UNWIND) + // ============================================================================= // DWARF CONSTANTS // ============================================================================= @@ -60,72 +87,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction -#if defined(__aarch64__) - DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state -#endif DWRF_CFA_restore = 0xc0 // Restore register }; -/* - * Architecture-specific DWARF register numbers - * - * These constants define the register numbering scheme used by DWARF - * for each supported architecture. The numbers must match the ABI - * specification for proper stack unwinding. - */ -enum { -#ifdef __x86_64__ - /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ - DWRF_REG_AX, // RAX - DWRF_REG_DX, // RDX - DWRF_REG_CX, // RCX - DWRF_REG_BX, // RBX - DWRF_REG_SI, // RSI - DWRF_REG_DI, // RDI - DWRF_REG_BP, // RBP - DWRF_REG_SP, // RSP - DWRF_REG_8, // R8 - DWRF_REG_9, // R9 - DWRF_REG_10, // R10 - DWRF_REG_11, // R11 - DWRF_REG_12, // R12 - DWRF_REG_13, // R13 - DWRF_REG_14, // R14 - DWRF_REG_15, // R15 - DWRF_REG_RA, // Return address (RIP) -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 register numbering */ - DWRF_REG_FP = 29, // Frame Pointer - DWRF_REG_RA = 30, // Link register (return address) - DWRF_REG_SP = 31, // Stack pointer -#else -# error "Unsupported target architecture" -#endif -}; - -// ============================================================================= -// ELF OBJECT CONTEXT -// ============================================================================= - -/* - * Context for building ELF/DWARF structures - * - * This structure maintains state while constructing DWARF unwind information. - * It acts as a simple buffer manager with pointers to track current position - * and important landmarks within the buffer. - */ -typedef struct ELFObjectContext { - uint8_t* p; // Current write position in buffer - uint8_t* startp; // Start of buffer (for offset calculations) - uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) - uintptr_t code_addr; // Address of the code section - size_t code_size; // Size of the code section -} ELFObjectContext; - -// ============================================================================= -// DWARF GENERATION UTILITIES -// ============================================================================= - /* * Append a null-terminated string to the ELF context buffer. * @@ -221,45 +185,42 @@ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ } +#endif /* PY_HAVE_JIT_GDB_UNWIND */ + // ============================================================================= // DWARF EH FRAME GENERATION // ============================================================================= -static void elf_init_ehframe_perf(ELFObjectContext* ctx); #if defined(PY_HAVE_JIT_GDB_UNWIND) static void elf_init_ehframe_gdb(ELFObjectContext* ctx); #endif -static inline void elf_init_ehframe(ELFObjectContext* ctx, int absolute_addr) { - if (absolute_addr) { -#if defined(PY_HAVE_JIT_GDB_UNWIND) - elf_init_ehframe_gdb(ctx); -#else - Py_UNREACHABLE(); -#endif - } - else { - elf_init_ehframe_perf(ctx); - } -} - size_t _PyJitUnwind_EhFrameSize(int absolute_addr) { - /* The .eh_frame we emit is small and bounded; keep a generous buffer. */ + if (!absolute_addr) { +#if defined(PY_HAVE_PERF_TRAMPOLINE) + return _Py_trampoline_ehframe.size; +#else + return 0; +#endif + } +#if defined(PY_HAVE_JIT_GDB_UNWIND) + /* GDB path: generate into scratch to learn the required size. */ uint8_t scratch[512]; _Static_assert(sizeof(scratch) >= 256, - "scratch buffer may be too small for elf_init_ehframe"); + "scratch buffer may be too small for elf_init_ehframe_gdb"); ELFObjectContext ctx; ctx.code_size = 1; ctx.code_addr = 0; ctx.startp = ctx.p = scratch; - ctx.fde_p = NULL; - /* Generate once into scratch to learn the required size. */ - elf_init_ehframe(&ctx, absolute_addr); + elf_init_ehframe_gdb(&ctx); ptrdiff_t size = ctx.p - ctx.startp; assert(size <= (ptrdiff_t)sizeof(scratch)); return (size_t)size; +#else + return 0; +#endif } size_t @@ -270,7 +231,16 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, if (buffer == NULL || code_addr == NULL || code_size == 0) { return 0; } - /* Generate the frame twice: once to size-check, once to write. */ + if (!absolute_addr) { +#if defined(PY_HAVE_PERF_TRAMPOLINE) + return _PyJitUnwind_PatchTrampolineEhFrame( + &_Py_trampoline_ehframe, buffer, buffer_size, code_size); +#else + return 0; +#endif + } +#if defined(PY_HAVE_JIT_GDB_UNWIND) + /* Size the frame first, then write it. */ size_t required = _PyJitUnwind_EhFrameSize(absolute_addr); if (required == 0 || required > buffer_size) { return 0; @@ -279,12 +249,14 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, ctx.code_size = code_size; ctx.code_addr = (uintptr_t)code_addr; ctx.startp = ctx.p = buffer; - ctx.fde_p = NULL; - elf_init_ehframe(&ctx, absolute_addr); + elf_init_ehframe_gdb(&ctx); size_t written = (size_t)(ctx.p - ctx.startp); /* The frame size is independent of code_addr/code_size (fixed-width fields). */ assert(written == required); return written; +#else + return 0; +#endif } /* @@ -300,11 +272,35 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, * * Two flavors are emitted, dispatched on the absolute_addr flag: * - * - absolute_addr == 0 (elf_init_ehframe_perf): PC-relative FDE address - * encoding for perf's synthesized DSO layout. The CIE describes the - * trampoline's entry state and the FDE walks through the prologue and - * epilogue with advance_loc instructions. This matches the pre-existing - * perf_jit_trampoline behavior byte-for-byte. + * - absolute_addr == 0 (_PyJitUnwind_PatchTrampolineEhFrame): PC-relative FDE address + * encoding for perf's synthesized DSO layout. The bytes come from + * trampoline_ehframe.c, generated at build time from the compiled + * trampoline object. Only the FDE's initial_location and address_range + * are filled in here. + * + * To add an architecture, write Python/asm_trampoline_.S with + * .cfi directives describing the frame at every instruction (the + * x86_64 and AArch64 files are the reference), wire its object into + * configure.ac and Makefile.pre.in, and add its ELF machine type or + * Mach-O CPU type to the tables at the top of + * Tools/jit/_trampoline_ehframe.py. Compiling an equivalent C trampoline + * with "-O2 -fno-omit-frame-pointer -fno-optimize-sibling-calls" and + * running "readelf --debug-dump=frames" on it shows the CFI the + * directives must be equivalent to, for example with GCC on AArch64: + * + * CIE: Code alignment factor 4, Data alignment factor -8, + * Return address column 30, DW_CFA_def_cfa: r31 (sp) ofs 0 + * FDE: DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * GCC keeps the CFA relative to sp there, the assembly moves it to the + * frame pointer after "mov x29, sp", which describes the same frame. * * - absolute_addr == 1 (elf_init_ehframe_gdb): absolute FDE address * encoding for the GDB JIT in-memory ELF. The CIE describes the @@ -315,338 +311,60 @@ _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, * across the region). This is the GDB-side fix; see elf_init_ehframe_gdb * for details. */ -static void elf_init_ehframe_perf(ELFObjectContext* ctx) { - int fde_ptr_enc = DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4; - uint8_t* p = ctx->p; - uint8_t* framep = p; // Remember start of frame data - - /* - * DWARF Unwind Table for Trampoline Function - * - * This section defines DWARF Call Frame Information (CFI) using encoded macros - * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function - * preserves and restores registers. This is used by profiling tools (e.g., `perf`) - * and debuggers for stack unwinding in JIT-compiled code. - * - * ------------------------------------------------- - * TO REGENERATE THIS TABLE FROM GCC OBJECTS: - * ------------------------------------------------- - * - * 1. Create a trampoline source file (e.g., `trampoline.c`): - * - * #include - * typedef PyObject* (*py_evaluator)(void*, void*, int); - * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { - * return evaluator(ts, f, throwflag); - * } - * - * 2. Compile to an object file with frame pointer preservation: - * - * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c - * - * 3. Extract DWARF unwind info from the object file: - * - * readelf -w trampoline.o - * - * Example output from `.eh_frame`: - * - * 00000000 CIE - * Version: 1 - * Augmentation: "zR" - * Code alignment factor: 4 - * Data alignment factor: -8 - * Return address column: 30 - * DW_CFA_def_cfa: r31 (sp) ofs 0 - * - * 00000014 FDE cie=00000000 pc=0..14 - * DW_CFA_advance_loc: 4 - * DW_CFA_def_cfa_offset: 16 - * DW_CFA_offset: r29 at cfa-16 - * DW_CFA_offset: r30 at cfa-8 - * DW_CFA_advance_loc: 12 - * DW_CFA_restore: r30 - * DW_CFA_restore: r29 - * DW_CFA_def_cfa_offset: 0 - * - * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. - * - * ---------------------------------- - * HOW TO TRANSLATE TO DWRF_* MACROS: - * ---------------------------------- - * - * After compiling your trampoline with: - * - * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c - * - * run: - * - * readelf -w trampoline.o - * - * to inspect the generated `.eh_frame` data. You will see two main components: - * - * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. - * 2. An FDE (Frame Description Entry): function-specific unwind instructions. - * - * --------------------- - * Translating the CIE: - * --------------------- - * From `readelf -w`, you might see: - * - * 00000000 0000000000000010 00000000 CIE - * Version: 1 - * Augmentation: "zR" - * Code alignment factor: 4 - * Data alignment factor: -8 - * Return address column: 30 - * Augmentation data: 1b - * DW_CFA_def_cfa: r31 (sp) ofs 0 - * - * Map this to: - * - * DWRF_SECTION(CIE, - * DWRF_U32(0); // CIE ID (always 0 for CIEs) - * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 - * DWRF_STR("zR"); // Augmentation string "zR" - * DWRF_UV(4); // Code alignment factor = 4 - * DWRF_SV(-8); // Data alignment factor = -8 - * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) - * DWRF_UV(1); // Augmentation data length = 1 - * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers - * - * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa - * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) - * DWRF_UV(0); // Offset = 0 - * - * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary - * ) - * - * Notes: - * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. - * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. - * - * --------------------- - * Translating the FDE: - * --------------------- - * From `readelf -w`: - * - * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 - * DW_CFA_advance_loc: 4 - * DW_CFA_def_cfa_offset: 16 - * DW_CFA_offset: r29 at cfa-16 - * DW_CFA_offset: r30 at cfa-8 - * DW_CFA_advance_loc: 12 - * DW_CFA_restore: r30 - * DW_CFA_restore: r29 - * DW_CFA_def_cfa_offset: 0 - * - * Map the FDE header and instructions to: - * - * DWRF_SECTION(FDE, - * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) - * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) - * DWRF_U32(ctx->code_size); // Code range covered by this FDE - * DWRF_U8(0); // Augmentation data length (none) - * - * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) - * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 - * DWRF_UV(16); - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) - * DWRF_UV(2); // At offset 2 * 8 = 16 bytes - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) - * DWRF_UV(1); // At offset 1 * 8 = 8 bytes - * - * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) - * - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 - * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 - * - * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP - * DWRF_UV(0); - * ) - * - * To regenerate: - * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. - * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as - * the code is in a different address space every time. - * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: - * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) - * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) - * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset - * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) - * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. - * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. - */ - - /* - * Emit DWARF EH CIE (Common Information Entry) - * - * The CIE describes the calling conventions and basic unwinding rules - * that apply to all functions in this compilation unit. - */ - DWRF_SECTION(CIE, - DWRF_U32(0); // CIE ID (0 indicates this is a CIE) - DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) - DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) -#ifdef __x86_64__ - DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) -#endif - DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) - DWRF_U8(DWRF_REG_RA); // Return address register number - DWRF_UV(1); // Augmentation data length - DWRF_U8(fde_ptr_enc); // FDE pointer encoding - - /* Initial CFI instructions - describe default calling convention */ -#ifdef __x86_64__ - /* x86_64 initial CFI state */ - DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) - DWRF_UV(DWRF_REG_SP); // CFA = SP register - DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size - DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved - DWRF_UV(1); // At offset 1 from CFA -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 initial CFI state */ - DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) - DWRF_UV(DWRF_REG_SP); // CFA = SP register - DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) - // No initial register saves in AArch64 CIE -#endif - DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary - ) - - /* - * Emit DWARF EH FDE (Frame Description Entry) - * - * The FDE describes unwinding information specific to this function. - * It references the CIE and provides function-specific CFI instructions. - * - * The PC-relative offset is calculated after the entire EH frame is built - * to ensure accurate positioning relative to the synthesized DSO layout. - */ - DWRF_SECTION(FDE, - DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) - /* - * In perf jitdump mode the FDE PC field is encoded PC-relative and - * points back to code_start. Record where that field lives so we can - * patch in the final offset after the rest of the synthetic DSO - * layout is known. - */ - ctx->fde_p = p; // Remember where PC offset field is located for later calculation - DWRF_U32(0); // Placeholder for PC-relative offset (calculated below) - DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) - DWRF_U8(0); // Augmentation data length (none) - - /* - * Architecture-specific CFI instructions - * - * These instructions describe how registers are saved and restored - * during function calls. Each architecture has different calling - * conventions and register usage patterns. - */ -#ifdef __x86_64__ - /* x86_64 calling convention unwinding rules */ -# if defined(__CET__) && (__CET__ & 1) - DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) -# endif - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) - DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 - DWRF_UV(16); // New offset: SP + 16 - DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 - DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) - DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 - DWRF_UV(DWRF_REG_BP); // Use base pointer register - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 - DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 - DWRF_UV(DWRF_REG_SP); // Use stack pointer register - DWRF_UV(8); // New offset: SP + 8 -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - /* AArch64 calling convention unwinding rules */ -#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ - (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) -#endif -#if defined(__ARM_FEATURE_PAC_DEFAULT) - DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here -#endif - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) - DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 - DWRF_UV(16); // Stack pointer moved by 16 bytes - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved - DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved - DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) -#if defined(__ARM_FEATURE_PAC_DEFAULT) - DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed -#endif - DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 - DWRF_UV(DWRF_REG_FP); - DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! - DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! - DWRF_U8(DWRF_CFA_def_cfa); // CFA = SP + 0 (stack restored) - DWRF_UV(DWRF_REG_SP); - DWRF_UV(0); - -#else -# error "Unsupported target architecture" -#endif - - DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary - ) - - ctx->p = p; // Update context pointer to end of generated data - - /* Calculate and update the PC-relative offset in the FDE - * - * When perf processes the jitdump, it creates a synthesized DSO with this layout: - * - * Synthesized DSO Memory Layout: - * ┌─────────────────────────────────────────────────────────────┐ < code_start - * │ Code Section │ - * │ (round_up(code_size, 8) bytes) │ - * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data - * │ EH Frame Data │ - * │ ┌─────────────────────────────────────────────────────┐ │ - * │ │ CIE data │ │ - * │ └─────────────────────────────────────────────────────┘ │ - * │ ┌─────────────────────────────────────────────────────┐ │ - * │ │ FDE Header: │ │ - * │ │ - CIE offset (4 bytes) │ │ - * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start - * │ │ - address range (4 bytes) │ │ (this specific field) - * │ │ CFI Instructions... │ │ - * │ └─────────────────────────────────────────────────────┘ │ - * ├─────────────────────────────────────────────────────────────┤ < reference_point - * │ EhFrameHeader │ - * │ (navigation metadata) │ - * └─────────────────────────────────────────────────────────────┘ - * - * The PC offset field in the FDE must contain the distance from itself to code_start: - * - * distance = code_start - fde_pc_field - * - * Where: - * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame - * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) - * - * Therefore: - * distance = code_start_location - fde_pc_field_location - * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) - * = -rounded_code_size - fde_offset_in_frame - * = -(round_up(code_size, 8) + fde_offset_in_frame) - * - * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field. - * - */ - int32_t rounded_code_size = - (int32_t)_Py_SIZE_ROUND_UP(ctx->code_size, 8); - int32_t fde_offset_in_frame = (int32_t)(ctx->fde_p - framep); - *(int32_t *)ctx->fde_p = -(rounded_code_size + fde_offset_in_frame); +#if defined(PY_HAVE_PERF_TRAMPOLINE) +size_t +_PyJitUnwind_PatchTrampolineEhFrame(const _PyTrampolineEhFrame *eh, + uint8_t *buffer, size_t buffer_size, + size_t code_size) +{ + /* The data can only come from Tools/jit/_trampoline_ehframe.py, but + * check it here as well instead of trusting the linked object: the + * bootstrap programs link an empty stub, and a malformed artifact must + * not turn into out-of-bounds writes. The bounds checks subtract so + * they cannot overflow. */ + if (eh == NULL || eh->data == NULL || buffer == NULL || code_size == 0) { + return 0; + } + size_t size = eh->size; + size_t field_size = eh->fde_field_size; + if (size == 0 || size > buffer_size + || (field_size != 4 && field_size != 8) + || size < 2 * field_size + || eh->fde_pc_offset > size - 2 * field_size + || eh->fde_range_offset != eh->fde_pc_offset + field_size) { + return 0; + } + /* perf's EhFrameHeader stores the distance from the end of the frame + * back to the code as a signed 4-byte offset (see perf_jit_trampoline.c), + * and so does the FDE when its fields are 4 bytes wide. Refuse sizes + * those cannot hold rather than writing truncated offsets. */ + if (size > (size_t)INT32_MAX - 8 + || code_size > (size_t)INT32_MAX - 8 - size) { + return 0; + } + memcpy(buffer, eh->data, size); + + /* perf maps this .eh_frame right after the code, at code_size rounded + * up to 8 bytes (see EhFrameHeader in perf_jit_trampoline.c), and + * initial_location is relative to its own field. */ + int64_t initial_location = -(int64_t)( + _Py_SIZE_ROUND_UP(code_size, 8) + eh->fde_pc_offset); + uint64_t address_range = code_size; + if (field_size == 4) { + int32_t pc_field = (int32_t)initial_location; + uint32_t range_field = (uint32_t)address_range; + memcpy(buffer + eh->fde_pc_offset, &pc_field, sizeof(pc_field)); + memcpy(buffer + eh->fde_range_offset, &range_field, sizeof(range_field)); + } + else { + memcpy(buffer + eh->fde_pc_offset, &initial_location, + sizeof(initial_location)); + memcpy(buffer + eh->fde_range_offset, &address_range, + sizeof(address_range)); + } + return size; } +#endif /* PY_HAVE_PERF_TRAMPOLINE */ /* * Build .eh_frame data for the GDB JIT interface. diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index d90b789c2b5712..fa1cb785b4126e 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -172,10 +172,8 @@ typedef PyObject *(*py_evaluator)(PyThreadState *, _PyInterpreterFrame *, typedef PyObject *(*py_trampoline)(PyThreadState *, _PyInterpreterFrame *, int, py_evaluator); -extern void *_Py_trampoline_func_start; // Start of the template of the - // assembly trampoline -extern void * - _Py_trampoline_func_end; // End of the template of the assembly trampoline +extern char _Py_trampoline_func_start; // Start of the assembly trampoline template +extern char _Py_trampoline_func_end; // End of the assembly trampoline template struct code_arena_st { char *start_addr; // Start of the memory arena @@ -334,8 +332,8 @@ new_code_arena(void) return -1; } (void)_PyAnnotateMemoryMap(memory, mem_size, "cpython:perf_trampoline"); - void *start = &_Py_trampoline_func_start; - void *end = &_Py_trampoline_func_end; + char *start = &_Py_trampoline_func_start; + char *end = &_Py_trampoline_func_end; size_t code_size = end - start; size_t unaligned_size = code_size + trampoline_api.code_padding; size_t chunk_size = round_up(unaligned_size, trampoline_api.code_alignment); diff --git a/Tools/jit/_trampoline_ehframe.py b/Tools/jit/_trampoline_ehframe.py new file mode 100644 index 00000000000000..be78976ac1fb84 --- /dev/null +++ b/Tools/jit/_trampoline_ehframe.py @@ -0,0 +1,553 @@ +"""Generate trampoline_ehframe.c from a compiled perf trampoline object. + +Copies the assembler-generated .eh_frame of the trampoline into a C file, +one block per architecture (a fat Mach-O file yields several). The FDE's +initial_location and address_range are left zeroed for Python/jit_unwind.c +to patch at runtime. + +The build runs this under _bootstrap_python, which has no shared extension +modules, so it only uses pure Python modules and built-in ones. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from dataclasses import dataclass +from typing import Literal + +# ELF constants, see . +_ELF_MAGIC = b"\x7fELF" +_ELF64_HEADER_SIZE = 64 +_ELF64_SECTION_HEADER_SIZE = 64 +_ELFCLASS64 = 2 +_ELFDATA2LSB = 1 +_ELFDATA2MSB = 2 +_SHT_NOBITS = 8 +_EM_X86_64 = 62 +_EM_AARCH64 = 183 + +# Mach-O constants, see llvm/BinaryFormat/MachO.h. +_MH_MAGIC_64 = 0xFEEDFACF +_MH_CIGAM_64 = 0xCFFAEDFE +_MACHO64_HEADER_SIZE = 32 +_MACHO64_SEGMENT_COMMAND_SIZE = 72 +_MACHO64_SECTION_SIZE = 80 +_FAT_MAGIC = 0xCAFEBABE +_FAT_MAGIC_64 = 0xCAFEBABF +_FAT_HEADER_SIZE = 8 +_LC_SEGMENT_64 = 0x19 +_SECTION_TYPE = 0x000000FF +_S_ZEROFILL = 0x01 +_S_GB_ZEROFILL = 0x0C +_S_THREAD_LOCAL_ZEROFILL = 0x12 +_ZEROFILL_SECTION_TYPES = (_S_ZEROFILL, _S_GB_ZEROFILL, _S_THREAD_LOCAL_ZEROFILL) +_CPU_ARCH_ABI64 = 0x01000000 +_CPU_TYPE_X86_64 = 7 | _CPU_ARCH_ABI64 +_CPU_TYPE_ARM64 = 12 | _CPU_ARCH_ABI64 + +# DWARF exception header pointer encodings, see +# Include/internal/pycore_jit_unwind.h. +_DW_EH_PE_absptr = 0x00 +_DW_EH_PE_sdata4 = 0x0B +_DW_EH_PE_pcrel = 0x10 + +# Largest .eh_frame accepted, in bytes. Python/perf_jit_trampoline.c builds +# each jitdump entry's frame in a buffer of this size and skips the entry +# when the frame does not fit, so a larger one would silently break perf. +_MAX_EHFRAME_SIZE = 1024 + +# Smallest CIE the parser accepts, in bytes after the length field: CIE_id, +# version, "zR\0", code and data alignment factors, return address column, +# augmentation data length and the FDE pointer encoding. +_CIE_MIN_LENGTH = 4 + 1 + 3 + 1 + 1 + 1 + 1 + 1 + +# Accepted FDE pointer encodings and the width of the initial_location and +# address_range fields they imply. GNU and LLVM ELF assemblers emit +# pcrel|sdata4, Darwin assemblers emit pcrel|absptr. +_FDE_FIELD_SIZES = { + _DW_EH_PE_pcrel | _DW_EH_PE_sdata4: 4, + _DW_EH_PE_pcrel | _DW_EH_PE_absptr: 8, +} + +# Compiler macro that selects each slice's block in the generated C file. +_ELF_ARCH_MACROS = { + _EM_X86_64: "__x86_64__", + _EM_AARCH64: "__aarch64__", +} +_MACHO_ARCH_MACROS = { + _CPU_TYPE_X86_64: "__x86_64__", + _CPU_TYPE_ARM64: "__aarch64__", +} + +# The sections the generator needs, and their Mach-O names. +_WANTED_SECTIONS = (".eh_frame", ".text") +_MACHO_SECTION_NAMES = { + "__eh_frame": ".eh_frame", + "__text": ".text", +} + +_BYTE_ORDERS: dict[str, Literal["little", "big"]] = {"<": "little", ">": "big"} + + +def _unpack( + data: bytes | bytearray, offset: int, endian: str, *sizes: int +) -> tuple[int, ...]: + """Read consecutive unsigned integers of the given byte sizes. + + endian is "<" or ">" as in the struct module, which is not available + under _bootstrap_python. + """ + end = offset + sum(sizes) + if offset < 0 or end > len(data): + raise ValueError(f"read up to offset {end} in {len(data)} bytes of data") + byteorder = _BYTE_ORDERS[endian] + values = [] + for size in sizes: + values.append(int.from_bytes(data[offset : offset + size], byteorder)) + offset += size + return tuple(values) + + +@dataclass +class ObjectSlice: + """One architecture's worth of an object file.""" + + source: str + arch_macro: str + endian: str + sections: dict[str, bytes] + + +@dataclass +class EhFrame: + """Parsed .eh_frame with the FDE fields zeroed for runtime patching.""" + + data: bytes + fde_pc_offset: int + fde_range_offset: int + field_size: int + + +def _elf_slice(data: bytes, source: str) -> ObjectSlice: + """Parse an ELF64 relocatable object.""" + if len(data) < _ELF64_HEADER_SIZE: + raise ValueError(f"{source}: truncated ELF header") + if data[4] != _ELFCLASS64: + raise ValueError(f"{source}: not an ELF64 object (class={data[4]})") + if data[5] == _ELFDATA2LSB: + endian = "<" + elif data[5] == _ELFDATA2MSB: + endian = ">" + else: + raise ValueError(f"{source}: unknown ELF byte order ({data[5]})") + + e_machine = _unpack(data, 18, endian, 2)[0] + try: + arch_macro = _ELF_ARCH_MACROS[e_machine] + except KeyError: + raise ValueError( + f"{source}: unsupported ELF machine type {e_machine}" + ) from None + + e_shoff = _unpack(data, 40, endian, 8)[0] + e_shentsize, e_shnum, e_shstrndx = _unpack(data, 58, endian, 2, 2, 2) + if e_shoff == 0 or e_shnum == 0: + raise ValueError(f"{source}: no section headers") + if e_shentsize < _ELF64_SECTION_HEADER_SIZE: + raise ValueError(f"{source}: bad section header size {e_shentsize}") + if e_shoff + e_shnum * e_shentsize > len(data): + raise ValueError(f"{source}: section headers extend beyond the end of the file") + if e_shstrndx >= e_shnum: + raise ValueError(f"{source}: bad section name string table index") + + def section_header(index: int) -> tuple[int, int, int, int]: + base = e_shoff + index * e_shentsize + sh_name, sh_type = _unpack(data, base, endian, 4, 4) + sh_offset, sh_size = _unpack(data, base + 24, endian, 8, 8) + return sh_name, sh_type, sh_offset, sh_size + + def section_bytes(name: str, sh_type: int, sh_offset: int, sh_size: int) -> bytes: + if sh_type == _SHT_NOBITS: + raise ValueError(f"{source}: section {name} has no contents in the file") + if sh_offset + sh_size > len(data): + raise ValueError( + f"{source}: section {name} extends past the end of the file" + ) + return data[sh_offset : sh_offset + sh_size] + + _, shstr_type, shstr_offset, shstr_size = section_header(e_shstrndx) + shstrtab = section_bytes(".shstrtab", shstr_type, shstr_offset, shstr_size) + + sections: dict[str, bytes] = {} + for index in range(e_shnum): + sh_name, sh_type, sh_offset, sh_size = section_header(index) + end = shstrtab.find(b"\x00", sh_name) + if end < 0: + continue + name = shstrtab[sh_name:end].decode("ascii", errors="replace") + if name in _WANTED_SECTIONS: + if name in sections: + raise ValueError(f"{source}: more than one {name} section") + sections[name] = section_bytes(name, sh_type, sh_offset, sh_size) + + return ObjectSlice(source, arch_macro, endian, sections) + + +def _macho_slice(data: bytes, source: str) -> ObjectSlice: + """Parse a thin Mach-O 64-bit object.""" + if len(data) < _MACHO64_HEADER_SIZE: + raise ValueError(f"{source}: truncated Mach-O header") + magic = _unpack(data, 0, "<", 4)[0] + if magic == _MH_MAGIC_64: + endian = "<" + elif magic == _MH_CIGAM_64: + endian = ">" + else: + raise ValueError(f"{source}: not a 64-bit Mach-O object") + + cputype = _unpack(data, 4, endian, 4)[0] + try: + arch_macro = _MACHO_ARCH_MACROS[cputype] + except KeyError: + raise ValueError( + f"{source}: unsupported Mach-O CPU type {cputype:#x}" + ) from None + + # mach_header_64: magic, cputype, cpusubtype, filetype, ncmds, + # sizeofcmds, flags, reserved (8 x uint32, 32 bytes). + ncmds, sizeofcmds = _unpack(data, 16, endian, 4, 4) + commands_end = _MACHO64_HEADER_SIZE + sizeofcmds + if commands_end > len(data): + raise ValueError(f"{source}: load commands extend beyond the end of the file") + + sections: dict[str, bytes] = {} + offset = _MACHO64_HEADER_SIZE + for _ in range(ncmds): + if offset + 8 > commands_end: + raise ValueError(f"{source}: truncated load commands") + cmd, cmdsize = _unpack(data, offset, endian, 4, 4) + if cmdsize < 8 or offset + cmdsize > commands_end: + raise ValueError(f"{source}: bad load command size {cmdsize}") + if cmd == _LC_SEGMENT_64: + # segment_command_64: cmd, cmdsize, segname[16], vmaddr, vmsize, + # fileoff, filesize, maxprot, initprot, nsects, flags (72 bytes), + # followed by nsects section_64 entries. + if cmdsize < _MACHO64_SEGMENT_COMMAND_SIZE: + raise ValueError(f"{source}: truncated segment command") + nsects = _unpack(data, offset + 64, endian, 4)[0] + if _MACHO64_SEGMENT_COMMAND_SIZE + nsects * _MACHO64_SECTION_SIZE > cmdsize: + raise ValueError( + f"{source}: section table extends beyond its segment command" + ) + sect = offset + _MACHO64_SEGMENT_COMMAND_SIZE + for _ in range(nsects): + # section_64: sectname[16], segname[16], addr, size, offset, + # align, reloff, nreloc, flags, reserved1-3 (80 bytes). The + # low byte of flags is the section type. + raw_name = data[sect : sect + 16].split(b"\x00", 1)[0] + segname = data[sect + 16 : sect + 32].split(b"\x00", 1)[0] + name = _MACHO_SECTION_NAMES.get( + raw_name.decode("ascii", errors="replace") + ) + if name is not None and segname == b"__TEXT": + if name in sections: + raise ValueError(f"{source}: more than one {name} section") + size = _unpack(data, sect + 40, endian, 8)[0] + file_offset = _unpack(data, sect + 48, endian, 4)[0] + flags = _unpack(data, sect + 64, endian, 4)[0] + if flags & _SECTION_TYPE in _ZEROFILL_SECTION_TYPES: + raise ValueError( + f"{source}: section {name} has no contents in the file" + ) + if file_offset + size > len(data): + raise ValueError( + f"{source}: section {name} extends past the end of the file" + ) + sections[name] = data[file_offset : file_offset + size] + sect += _MACHO64_SECTION_SIZE + offset += cmdsize + + return ObjectSlice(source, arch_macro, endian, sections) + + +def _fat_slices(data: bytes, source: str) -> list[ObjectSlice]: + """Split a universal (fat) Mach-O file into its thin slices.""" + # The fat header and its fat_arch entries are always big-endian. + magic, nfat_arch = _unpack(data, 0, ">", 4, 4) + entry_sizes: tuple[int, ...] + if magic == _FAT_MAGIC: + # fat_arch: cputype, cpusubtype, offset, size, align (20 bytes). + entry_sizes, entry_size = (4, 4, 4, 4, 4), 20 + elif magic == _FAT_MAGIC_64: + # fat_arch_64: cputype, cpusubtype, offset, size, align, reserved. + entry_sizes, entry_size = (4, 4, 8, 8, 4, 4), 32 + else: + raise ValueError(f"{source}: not a fat Mach-O file") + if nfat_arch == 0: + raise ValueError(f"{source}: fat Mach-O file with no architectures") + if _FAT_HEADER_SIZE + nfat_arch * entry_size > len(data): + raise ValueError(f"{source}: truncated fat header") + + slices: list[ObjectSlice] = [] + for index in range(nfat_arch): + entry = _unpack(data, _FAT_HEADER_SIZE + index * entry_size, ">", *entry_sizes) + cputype, _, offset, size = entry[:4] + thin = data[offset : offset + size] + if len(thin) != size: + raise ValueError(f"{source}: fat slice {index} is truncated") + obj_slice = _macho_slice(thin, f"{source} (slice {cputype:#x})") + if _MACHO_ARCH_MACROS.get(cputype) != obj_slice.arch_macro: + raise ValueError( + f"{source}: fat slice {index} CPU type {cputype:#x} does not match " + "its Mach-O header" + ) + slices.append(obj_slice) + return slices + + +def load_object(path: str) -> list[ObjectSlice]: + """Return the ObjectSlices (one per architecture) of an object file.""" + with open(path, "rb") as f: + data = f.read() + source = os.path.basename(path) + if data[:4] == _ELF_MAGIC: + return [_elf_slice(data, source)] + if len(data) < _FAT_HEADER_SIZE: + raise ValueError(f"{source}: file too short to be an object file") + magic_be = _unpack(data, 0, ">", 4)[0] + if magic_be in (_FAT_MAGIC, _FAT_MAGIC_64): + return _fat_slices(data, source) + magic_le = _unpack(data, 0, "<", 4)[0] + if magic_le in (_MH_MAGIC_64, _MH_CIGAM_64): + return [_macho_slice(data, source)] + raise ValueError(f"{source}: not an ELF64, Mach-O 64 or fat Mach-O object") + + +def _read_uleb128(data: bytearray, pos: int, limit: int) -> tuple[int, int]: + """Decode an unsigned LEB128 at pos; return (value, position after it).""" + value = 0 + shift = 0 + while True: + if pos >= limit: + raise ValueError("truncated LEB128 in CIE") + byte = data[pos] + pos += 1 + value |= (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return value, pos + + +def parse_ehframe(eh_frame: bytes, endian: str, text_size: int) -> EhFrame: + """Validate a one-CIE, one-FDE .eh_frame and zero the FDE's fields. + + text_size is the size of the object's .text section; it must equal the + FDE's address_range, which catches a misplaced .cfi_endproc. + """ + if text_size == 0: + raise ValueError(".text section is empty") + if len(eh_frame) > _MAX_EHFRAME_SIZE: + raise ValueError( + f".eh_frame is {len(eh_frame)} bytes, larger than the " + f"{_MAX_EHFRAME_SIZE} bytes the runtime accepts" + ) + data = bytearray(eh_frame) + if len(data) < 8: + raise ValueError("no CIE found in .eh_frame") + + # CIE header: length, CIE_id (0), version, augmentation string. + cie_length, cie_id = _unpack(data, 0, endian, 4, 4) + if cie_id != 0: + raise ValueError(f"expected a CIE at offset 0, got CIE_id={cie_id:#x}") + cie_total: int = 4 + cie_length + if cie_length < _CIE_MIN_LENGTH or cie_total > len(data): + raise ValueError(f"bad CIE length {cie_length}") + + version = data[8] + if version != 1: + raise ValueError(f"unexpected CIE version {version}") + + null_pos = data.find(0, 9, cie_total) + if null_pos < 0: + raise ValueError("CIE augmentation string not null-terminated") + augmentation = data[9:null_pos].decode("ascii", errors="replace") + if augmentation != "zR": + raise ValueError(f"CIE augmentation {augmentation!r} is not 'zR'") + + # After the augmentation string: code alignment factor (ULEB128), data + # alignment factor (SLEB128, skipped the same way), return address column + # (one byte in version 1), augmentation data length (ULEB128), then the + # augmentation data, which for "zR" is exactly the FDE pointer encoding. + pos = null_pos + 1 + _, pos = _read_uleb128(data, pos, cie_total) + _, pos = _read_uleb128(data, pos, cie_total) + pos += 1 + aug_length, pos = _read_uleb128(data, pos, cie_total) + if aug_length != 1 or pos + aug_length > cie_total: + raise ValueError(f"CIE augmentation data length {aug_length} is not 1") + fde_ptr_enc = data[pos] + try: + field_size = _FDE_FIELD_SIZES[fde_ptr_enc] + except KeyError: + raise ValueError(f"unsupported FDE pointer encoding {fde_ptr_enc:#x}") from None + + # FDE: length, CIE pointer, initial_location, address_range, augmentation + # data length (0 for a "zR" CIE), then the CFI instructions. + fde_start = cie_total + fde_min_length = 4 + 2 * field_size + 1 + if fde_start + 4 + fde_min_length > len(data): + raise ValueError("no FDE after the CIE") + fde_length, fde_cie_ptr = _unpack(data, fde_start, endian, 4, 4) + if fde_length < fde_min_length: + raise ValueError(f"FDE too short ({fde_length} bytes)") + if fde_cie_ptr != fde_start + 4: + raise ValueError(f"FDE CIE pointer {fde_cie_ptr} does not point at the CIE") + if fde_start + 4 + fde_length != len(data): + raise ValueError("expected exactly one FDE ending at the end of .eh_frame") + + fde_pc_offset = fde_start + 8 + fde_range_offset = fde_pc_offset + field_size + fde_aug_length = data[fde_range_offset + field_size] + if fde_aug_length != 0: + raise ValueError(f"FDE augmentation data length {fde_aug_length} is not 0") + address_range = int.from_bytes( + data[fde_range_offset : fde_range_offset + field_size], + "little" if endian == "<" else "big", + ) + if address_range != text_size: + raise ValueError( + f"FDE address_range {address_range} != .text size {text_size}; " + "check the .cfi_startproc/.cfi_endproc placement" + ) + + # Zero the placeholders. The runtime fills in the real values. + data[fde_pc_offset : fde_range_offset + field_size] = bytes(2 * field_size) + return EhFrame(bytes(data), fde_pc_offset, fde_range_offset, field_size) + + +def build_ehframe(obj_slice: ObjectSlice) -> EhFrame: + """Extract and validate the .eh_frame of one object slice.""" + sections = obj_slice.sections + if ".eh_frame" not in sections: + raise ValueError( + f"{obj_slice.source}: no .eh_frame section; does the assembly " + "have .cfi_startproc/.cfi_endproc directives?" + ) + if ".text" not in sections: + raise ValueError(f"{obj_slice.source}: no .text section") + try: + return parse_ehframe( + sections[".eh_frame"], obj_slice.endian, len(sections[".text"]) + ) + except ValueError as exc: + raise ValueError(f"{obj_slice.source}: {exc}") from None + + +def _format_bytes(data: bytes, per_line: int = 12) -> str: + lines = [] + for start in range(0, len(data), per_line): + chunk = data[start : start + per_line] + lines.append(" " + ", ".join(f"0x{b:02x}" for b in chunk) + ",") + return "\n".join(lines) + + +def write_source(entries: list[tuple[ObjectSlice, EhFrame]], output_path: str) -> None: + """Write the C file; entries is a list of (ObjectSlice, EhFrame).""" + if not entries: + raise ValueError("no architectures to write") + sources = sorted({obj_slice.source.split(" (")[0] for obj_slice, _ in entries}) + lines = [ + "/* Auto-generated by Tools/jit/_trampoline_ehframe.py" + f" from {', '.join(sources)}. Do not edit. */", + "", + '#include "Python.h"', + '#include "pycore_jit_unwind.h"', + "", + "/* .eh_frame of the perf trampoline. The FDE's initial_location and", + " * address_range are zeroed placeholders patched by Python/jit_unwind.c. */", + ] + keyword = "#if" + for obj_slice, eh_frame in sorted(entries, key=lambda e: e[0].arch_macro): + lines += [ + f"{keyword} defined({obj_slice.arch_macro})", + f"/* From {obj_slice.source}. */", + "static const uint8_t trampoline_ehframe[] = {", + _format_bytes(eh_frame.data), + "};", + "const _PyTrampolineEhFrame _Py_trampoline_ehframe = {", + " trampoline_ehframe,", + " sizeof(trampoline_ehframe),", + f" {eh_frame.fde_pc_offset}, /* fde_pc_offset */", + f" {eh_frame.fde_range_offset}, /* fde_range_offset */", + f" {eh_frame.field_size}, /* fde_field_size */", + "};", + ] + keyword = "#elif" + lines += [ + "#else", + '# error "trampoline_ehframe.c was not generated for this architecture"', + "#endif", + "", + ] + output = "\n".join(lines) + + tmp_path = output_path + ".tmp" + try: + with open(tmp_path, "w") as f: + f.write(output) + os.replace(tmp_path, output_path) + except BaseException: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + + +def generate( + object_paths: list[str], output_path: str +) -> list[tuple[ObjectSlice, EhFrame]]: + """Generate the C file from the given objects; return what was written.""" + entries: list[tuple[ObjectSlice, EhFrame]] = [] + seen: dict[str, str] = {} + for path in object_paths: + for obj_slice in load_object(path): + if obj_slice.arch_macro in seen: + raise ValueError( + f"{obj_slice.source} and {seen[obj_slice.arch_macro]} " + f"are both {obj_slice.arch_macro}" + ) + seen[obj_slice.arch_macro] = obj_slice.source + entries.append((obj_slice, build_ehframe(obj_slice))) + write_source(entries, output_path) + return entries + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate trampoline_ehframe.c from trampoline objects" + ) + parser.add_argument( + "objects", + nargs="+", + metavar="OBJECT", + help="compiled trampoline object (ELF64, Mach-O 64, or fat Mach-O)", + ) + parser.add_argument( + "-o", "--output", required=True, help="path of the C file to write" + ) + args = parser.parse_args() + try: + entries = generate(args.objects, args.output) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + for obj_slice, eh_frame in entries: + print( + f"Generated {args.output}: {obj_slice.arch_macro} from " + f"{obj_slice.source}, {len(eh_frame.data)} bytes" + ) + + +if __name__ == "__main__": + main()