From 3578ca6ef92a4e8173f2a26b84be26f1d63bedeb Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 11 May 2024 15:12:04 -0700 Subject: [PATCH 1/5] Make the generation of jit_stencils.h atomic --- Tools/jit/_targets.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 023ef498a21d7c..58966cf19cfbcf 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -180,18 +180,18 @@ async def _compile( await _llvm.run("clang", args_o, echo=self.verbose) return await self._parse(o) - async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: + async def _build_stencils( + self, work: pathlib.Path + ) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) tasks = [] - with tempfile.TemporaryDirectory() as tempdir: - work = pathlib.Path(tempdir).resolve() - async with asyncio.TaskGroup() as group: - coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) - tasks.append(group.create_task(coro, name="trampoline")) - for opname in opnames: - coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) - tasks.append(group.create_task(coro, name=opname)) + async with asyncio.TaskGroup() as group: + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) + for opname in opnames: + coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} def build( @@ -211,14 +211,18 @@ def build( and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = asyncio.run(self._build_stencils()) - with jit_stencils.open("w") as file: - file.write(digest) - if comment: - file.write(f"// {comment}\n\n") - file.write("") - for line in _writer.dump(stencil_groups): - file.write(f"{line}\n") + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + stencil_groups = asyncio.run(self._build_stencils(work)) + new_jit_stencils = work / "jit_stencils.h" + with new_jit_stencils.open("w") as file: + file.write(digest) + if comment: + file.write(f"// {comment}\n\n") + file.write("") + for line in _writer.dump(stencil_groups): + file.write(f"{line}\n") + new_jit_stencils.replace(jit_stencils) class _COFF( From bc322276a5b94e6f6ef7764964724914265af3df Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 11 May 2024 15:12:50 -0700 Subject: [PATCH 2/5] blurb add --- .../next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst diff --git a/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst b/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst new file mode 100644 index 00000000000000..4e886be034fb82 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-11-15-11-30.gh-issue-118943.VI_MnY.rst @@ -0,0 +1,3 @@ +Fix a possible race condition affecting parallel builds configured with +``--enable-experimental-jit``, in which compilation errors could be caused +by an incompletely-generated header file. From dc9404c5457a33da24f2564f60ae35d3fe9ae99a Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 11 May 2024 15:45:38 -0700 Subject: [PATCH 3/5] Don't stick jit_stencils.h in a temporary directory --- Tools/jit/_targets.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 58966cf19cfbcf..ebbd017657ea4c 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -180,18 +180,18 @@ async def _compile( await _llvm.run("clang", args_o, echo=self.verbose) return await self._parse(o) - async def _build_stencils( - self, work: pathlib.Path - ) -> dict[str, _stencils.StencilGroup]: + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases)) tasks = [] - async with asyncio.TaskGroup() as group: - coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) - tasks.append(group.create_task(coro, name="trampoline")) - for opname in opnames: - coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) - tasks.append(group.create_task(coro, name=opname)) + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + async with asyncio.TaskGroup() as group: + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) + for opname in opnames: + coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) + tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} def build( @@ -211,18 +211,19 @@ def build( and jit_stencils.read_text().startswith(digest) ): return - with tempfile.TemporaryDirectory() as tempdir: - work = pathlib.Path(tempdir).resolve() - stencil_groups = asyncio.run(self._build_stencils(work)) - new_jit_stencils = work / "jit_stencils.h" - with new_jit_stencils.open("w") as file: + stencil_groups = asyncio.run(self._build_stencils()) + jit_stencils_new = out / "jit_stencils.h.new" + try: + with jit_stencils_new.open("w") as file: file.write(digest) if comment: file.write(f"// {comment}\n\n") file.write("") for line in _writer.dump(stencil_groups): file.write(f"{line}\n") - new_jit_stencils.replace(jit_stencils) + jit_stencils_new.replace(jit_stencils) + finally: + jit_stencils_new.unlink(True) class _COFF( From 04ff73343b897722d97a119a8951ab0b8e24ff8a Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 11 May 2024 16:03:05 -0700 Subject: [PATCH 4/5] Remove empty write --- Tools/jit/_targets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index ebbd017657ea4c..b4c0f33887ef23 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -218,7 +218,6 @@ def build( file.write(digest) if comment: file.write(f"// {comment}\n\n") - file.write("") for line in _writer.dump(stencil_groups): file.write(f"{line}\n") jit_stencils_new.replace(jit_stencils) From 526612947d465e42bfe1aba5b9d02be95569d946 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 16 May 2024 11:32:58 -0400 Subject: [PATCH 5/5] Feedback from code review --- Tools/jit/_targets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index b4c0f33887ef23..d289e8a262ff76 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -217,12 +217,13 @@ def build( with jit_stencils_new.open("w") as file: file.write(digest) if comment: - file.write(f"// {comment}\n\n") + file.write(f"// {comment}\n") + file.write("\n") for line in _writer.dump(stencil_groups): file.write(f"{line}\n") jit_stencils_new.replace(jit_stencils) finally: - jit_stencils_new.unlink(True) + jit_stencils_new.unlink(missing_ok=True) class _COFF(