From 4410aeb08dc14a4f29c9ec0e8730a1bde3386665 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Mon, 8 Sep 2025 00:19:44 +0530 Subject: [PATCH 1/6] [clang-repl] Adding custom lambda in launchExecutor --- clang/include/clang/Interpreter/Interpreter.h | 4 +++- clang/lib/Interpreter/IncrementalExecutor.cpp | 6 +++++- clang/lib/Interpreter/IncrementalExecutor.h | 3 ++- clang/lib/Interpreter/Interpreter.cpp | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 61af7bf762d5e..54be57684c03f 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -135,11 +135,13 @@ class Interpreter { std::string OrcRuntimePath = ""; /// PID of the out-of-process JIT executor. uint32_t ExecutorPID = 0; + /// Custom lambda to be executed inside child process/executor + std::function CustomizeFork = nullptr; JITConfig() : IsOutOfProcess(false), OOPExecutor(""), OOPExecutorConnect(""), UseSharedMemory(false), SlabAllocateSize(0), OrcRuntimePath(""), - ExecutorPID(0) {} + ExecutorPID(0), CustomizeFork(nullptr) {} }; protected: diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index b0eb7d0e9f072..0cf11939fefd1 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -138,7 +138,8 @@ IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, Expected> createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC, - unsigned SlabAllocateSize) { + unsigned SlabAllocateSize, + std::function CustomizeFork) { llvm::orc::SharedMemoryMapper::SymbolAddrs SAs; if (auto Err = SREPC.getBootstrapSymbols( {{SAs.Instance, @@ -215,6 +216,9 @@ IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath, close(ToExecutor[WriteEnd]); close(FromExecutor[ReadEnd]); + if (CustomizeFork) + CustomizeFork(); + // Execute the child process. std::unique_ptr ExecutorPath, FDSpecifier; { diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index d091535166770..bb1ec33452515 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -79,7 +79,8 @@ class IncrementalExecutor { static llvm::Expected< std::pair, uint32_t>> launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory, - unsigned SlabAllocateSize); + unsigned SlabAllocateSize, + std::function CustomizeFork = nullptr); #if LLVM_ON_UNIX && LLVM_ENABLE_THREADS static llvm::Expected> diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 043e0c1e5754e..e17229a853a6f 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -355,7 +355,7 @@ Interpreter::outOfProcessJITBuilder(JITConfig Config) { if (!Config.OOPExecutor.empty()) { // Launch an out-of-process executor locally in a child process. auto ResultOrErr = IncrementalExecutor::launchExecutor( - Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize); + Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize, Config.CustomizeFork); if (!ResultOrErr) return ResultOrErr.takeError(); childPid = ResultOrErr->second; From 0a09e011672db57c4a041a3719144dd90afdeb8d Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Mon, 8 Sep 2025 00:20:09 +0530 Subject: [PATCH 2/6] Formatting changes --- clang/lib/Interpreter/IncrementalExecutor.cpp | 2 +- clang/lib/Interpreter/Interpreter.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 0cf11939fefd1..792ecb08c5f33 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -139,7 +139,7 @@ IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, Expected> createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC, unsigned SlabAllocateSize, - std::function CustomizeFork) { + std::function CustomizeFork) { llvm::orc::SharedMemoryMapper::SymbolAddrs SAs; if (auto Err = SREPC.getBootstrapSymbols( {{SAs.Instance, diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index e17229a853a6f..2425a628b59b9 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -355,7 +355,8 @@ Interpreter::outOfProcessJITBuilder(JITConfig Config) { if (!Config.OOPExecutor.empty()) { // Launch an out-of-process executor locally in a child process. auto ResultOrErr = IncrementalExecutor::launchExecutor( - Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize, Config.CustomizeFork); + Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize, + Config.CustomizeFork); if (!ResultOrErr) return ResultOrErr.takeError(); childPid = ResultOrErr->second; From 268826a35221f15549d595226d709922bca98abc Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Mon, 8 Sep 2025 00:35:44 +0530 Subject: [PATCH 3/6] Formatting changes & fixing bug --- clang/lib/Interpreter/IncrementalExecutor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 792ecb08c5f33..5bec3b44a0dc0 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -173,7 +173,8 @@ createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC, llvm::Expected, uint32_t>> IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory, - unsigned SlabAllocateSize) { + unsigned SlabAllocateSize, + std::function CustomizeFork) { #ifndef LLVM_ON_UNIX // FIXME: Add support for Windows. return llvm::make_error( From cf4c8766088524ad3b7ffe4d927325185c8c262a Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Mon, 8 Sep 2025 00:49:32 +0530 Subject: [PATCH 4/6] Removing extra arg from sharedMem --- clang/lib/Interpreter/IncrementalExecutor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 5bec3b44a0dc0..45620fcd358c8 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -138,8 +138,7 @@ IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, Expected> createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC, - unsigned SlabAllocateSize, - std::function CustomizeFork) { + unsigned SlabAllocateSize) { llvm::orc::SharedMemoryMapper::SymbolAddrs SAs; if (auto Err = SREPC.getBootstrapSymbols( {{SAs.Instance, From df67ed1320ebef0997ed78a27b8275034b3b50a8 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 13 Sep 2025 03:23:53 +0530 Subject: [PATCH 5/6] ClangRepl Interpreter test for out-of-process --- clang/unittests/Interpreter/CMakeLists.txt | 23 +- .../OutOfProcessInterpreterTests.cpp | 203 ++++++++++++++++++ 2 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt index db9f80d9f53fe..7b8dcfc9b0546 100644 --- a/clang/unittests/Interpreter/CMakeLists.txt +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -29,12 +29,25 @@ set(CLANG_LIBS_TO_LINK ) endif() -add_distinct_clang_unittest(ClangReplInterpreterTests +set(CLANG_REPL_TEST_SOURCES IncrementalCompilerBuilderTest.cpp IncrementalProcessingTest.cpp InterpreterTest.cpp InterpreterExtensionsTest.cpp CodeCompletionTest.cpp +) + +if(TARGET compiler-rt) + list(APPEND CLANG_REPL_TEST_SOURCES + OutOfProcessInterpreterTests.cpp + ) + message(STATUS "Compiler-RT found, enabling out of process JIT tests") +endif() + +add_distinct_clang_unittest(ClangReplInterpreterTests + ${CLANG_REPL_TEST_SOURCES} + + PARTIAL_SOURCES_INTENDED EXPORT_SYMBOLS @@ -48,6 +61,14 @@ add_distinct_clang_unittest(ClangReplInterpreterTests ${LLVM_COMPONENTS_TO_LINK} ) +if(TARGET compiler-rt) + add_dependencies(ClangReplInterpreterTests + llvm-jitlink-executor + compiler-rt + ) + message(STATUS "Adding dependency on compiler-rt for out of process JIT tests") +endif() + if(EMSCRIPTEN) # Without the above you try to link to LLVMSupport twice, and end # up with a duplicate symbol error when creating the main module diff --git a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp new file mode 100644 index 0000000000000..271820e4e5f25 --- /dev/null +++ b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp @@ -0,0 +1,203 @@ +//===- unittests/Interpreter/OutOfProcessInterpreterTest.cpp --- Interpreter +// tests when Out-of-Process ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Unit tests for Clang's Interpreter library. +// +//===----------------------------------------------------------------------===// + +#include "InterpreterTestFixture.h" +#include "clang/AST/Decl.h" +#include "clang/AST/DeclGroup.h" +#include "clang/AST/Mangle.h" +#include "clang/Basic/Version.h" +#include "clang/Config/config.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Interpreter/Interpreter.h" +#include "clang/Interpreter/Value.h" +#include "clang/Sema/Lookup.h" +#include "clang/Sema/Sema.h" +#include "llvm/Support/Error.h" +#include "llvm/TargetParser/Host.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include +#include +#include +#include + +using namespace clang; + +llvm::ExitOnError ExitOnError; + +namespace { + +using Args = std::vector; + +struct FileDeleter { + void operator()(FILE *f) { + if (f) + fclose(f); + } +}; + +struct IOContext { + std::unique_ptr stdin_file; + std::unique_ptr stdout_file; + std::unique_ptr stderr_file; + + bool initializeTempFiles() { + stdin_file.reset(tmpfile()); + stdout_file.reset(tmpfile()); + stderr_file.reset(tmpfile()); + return stdin_file && stdout_file && stderr_file; + } + + std::string readStdoutContent() { + if (!stdout_file) + return ""; + rewind(stdout_file.get()); + std::ostringstream content; + char buffer[1024]; + size_t bytes_read; + while ((bytes_read = fread(buffer, 1, sizeof(buffer), stdout_file.get())) > + 0) { + content.write(buffer, bytes_read); + } + return content.str(); + } + + std::string readStderrContent() { + if (!stderr_file) + return ""; + rewind(stderr_file.get()); + std::ostringstream content; + char buffer[1024]; + size_t bytes_read; + while ((bytes_read = fread(buffer, 1, sizeof(buffer), stderr_file.get())) > + 0) { + content.write(buffer, bytes_read); + } + return content.str(); + } +}; + +static void removePathComponent(unsigned N, llvm::SmallString<256> &Path) { + for (unsigned i = 0; i < N; ++i) + llvm::sys::path::remove_filename(Path); +} + +static std::string getExecutorPath() { + llvm::SmallString<256> ExecutorPath(llvm::sys::fs::getMainExecutable( + nullptr, reinterpret_cast(&getExecutorPath))); + removePathComponent(5, ExecutorPath); + llvm::sys::path::append(ExecutorPath, "bin", "llvm-jitlink-executor"); + return ExecutorPath.str().str(); +} + +static std::string getOrcRuntimePath() { + llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable( + nullptr, reinterpret_cast(&getOrcRuntimePath))); + removePathComponent(5, RuntimePath); + llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang", + CLANG_VERSION_MAJOR_STRING, "lib"); + + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + if (SystemTriple.isOSBinFormatMachO()) { + llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a"); + } else if (SystemTriple.isOSBinFormatELF()) { + llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu", + "liborc_rt.a"); + } + return RuntimePath.str().str(); +} + +static std::unique_ptr +createInterpreterWithRemoteExecution(std::shared_ptr io_ctx, + const Args &ExtraArgs = {}) { + Args ClangArgs = {"-Xclang", "-emit-llvm-only"}; + llvm::append_range(ClangArgs, ExtraArgs); + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetCompilerArgs(ClangArgs); + auto CI = cantFail(CB.CreateCpp()); + + clang::Interpreter::JITConfig Config; + llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); + + if (SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO()) { + Config.IsOutOfProcess = true; + Config.OOPExecutor = getExecutorPath(); + Config.UseSharedMemory = false; + Config.SlabAllocateSize = 0; + Config.OrcRuntimePath = getOrcRuntimePath(); + + int stdin_fd = fileno(io_ctx->stdin_file.get()); + int stdout_fd = fileno(io_ctx->stdout_file.get()); + int stderr_fd = fileno(io_ctx->stderr_file.get()); + + Config.CustomizeFork = [=] { + auto redirect = [](int from, int to) { + if (from != to) { + dup2(from, to); + close(from); + } + }; + + redirect(stdin_fd, STDIN_FILENO); + redirect(stdout_fd, STDOUT_FILENO); + redirect(stderr_fd, STDERR_FILENO); + + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + + printf("CustomizeFork executed\n"); + fflush(stdout); + }; + } + + return cantFail(clang::Interpreter::create(std::move(CI), Config)); +} + +static size_t DeclsSize(TranslationUnitDecl *PTUDecl) { + return std::distance(PTUDecl->decls().begin(), PTUDecl->decls().end()); +} + +TEST_F(InterpreterTestBase, SanityWithRemoteExecution) { + if (!HostSupportsJIT()) + GTEST_SKIP(); + + std::string OrcRuntimePath = getOrcRuntimePath(); + std::string ExecutorPath = getExecutorPath(); + + if (!llvm::sys::fs::exists(OrcRuntimePath) || + !llvm::sys::fs::exists(ExecutorPath)) + GTEST_SKIP(); + + auto io_ctx = std::make_shared(); + ASSERT_TRUE(io_ctx->initializeTempFiles()); + + std::unique_ptr Interp = + createInterpreterWithRemoteExecution(io_ctx); + ASSERT_TRUE(Interp); + + using PTU = PartialTranslationUnit; + PTU &R1(cantFail(Interp->Parse("void g(); void g() {}"))); + EXPECT_EQ(2U, DeclsSize(R1.TUPart)); + + PTU &R2(cantFail(Interp->Parse("int i = 42;"))); + EXPECT_EQ(1U, DeclsSize(R2.TUPart)); + + std::string captured_stdout = io_ctx->readStdoutContent(); + std::string captured_stderr = io_ctx->readStderrContent(); + + EXPECT_TRUE(captured_stdout.find("CustomizeFork executed") != + std::string::npos); +} + +} // end anonymous namespace \ No newline at end of file From d7f67d0e3519892e589d425df2ed92a0ecc7cf3d Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 13 Sep 2025 07:40:58 +0530 Subject: [PATCH 6/6] Formatting changes --- clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp index 271820e4e5f25..704ddc37e642e 100644 --- a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp +++ b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp @@ -174,7 +174,7 @@ TEST_F(InterpreterTestBase, SanityWithRemoteExecution) { std::string OrcRuntimePath = getOrcRuntimePath(); std::string ExecutorPath = getExecutorPath(); - + if (!llvm::sys::fs::exists(OrcRuntimePath) || !llvm::sys::fs::exists(ExecutorPath)) GTEST_SKIP();