From 2da048b3b95ef3073675a77b57760fe5e6c90c3f Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 29 Feb 2024 12:30:49 +0530 Subject: [PATCH 01/22] Add --jit option to execute without creation of executable file --- src/bin/lpython.cpp | 110 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index ea8c873c88..27d645b5f5 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -891,6 +891,106 @@ int compile_python_to_object_file( return 0; } +int execute_python_using_jit( + const std::string &infile, + const std::string &runtime_library_dir, + LCompilers::PassManager& pass_manager, + CompilerOptions &compiler_options, + bool time_report) +{ + Allocator al(4*1024); + LCompilers::diag::Diagnostics diagnostics; + LCompilers::LocationManager lm; + std::vector>times; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = infile; + lm.files.push_back(fl); + + auto file_reading_start = std::chrono::high_resolution_clock::now(); + std::string input = LCompilers::read_file(infile); + auto file_reading_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("File reading", std::chrono::duration + (file_reading_end - file_reading_start).count())); + + lm.init_simple(input); + lm.file_ends.push_back(input.size()); + } + auto parsing_start = std::chrono::high_resolution_clock::now(); + LCompilers::Result r = parse_python_file( + al, runtime_library_dir, infile, diagnostics, 0, compiler_options.new_parser); + auto parsing_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("Parsing", std::chrono::duration(parsing_end - parsing_start).count())); + std::cerr << diagnostics.render(lm, compiler_options); + if (!r.ok) { + print_time_report(times, time_report); + return 1; + } + + // Src -> AST -> ASR + LCompilers::LPython::AST::ast_t* ast = r.result; + diagnostics.diagnostics.clear(); + auto ast_to_asr_start = std::chrono::high_resolution_clock::now(); + LCompilers::Result + r1 = LCompilers::LPython::python_ast_to_asr(al, lm, nullptr, *ast, diagnostics, compiler_options, + !(false && compiler_options.po.disable_main), "__main__", infile); + + auto ast_to_asr_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("AST to ASR", std::chrono::duration(ast_to_asr_end - ast_to_asr_start).count())); + std::cerr << diagnostics.render(lm, compiler_options); + if (!r1.ok) { + LCOMPILERS_ASSERT(diagnostics.has_error()) + print_time_report(times, time_report); + return 2; + } + LCompilers::ASR::TranslationUnit_t* asr = r1.result; + if( compiler_options.po.disable_main ) { + int err = LCompilers::LPython::save_pyc_files(*asr, infile); + if( err ) { + return err; + } + } + diagnostics.diagnostics.clear(); + + // ASR -> LLVM + if (compiler_options.emit_debug_info) { +#ifndef HAVE_RUNTIME_STACKTRACE + diagnostics.add(LCompilers::diag::Diagnostic( + "The `runtime stacktrace` is not enabled. To get the stacktraces, " + "re-build LPython with `-DWITH_RUNTIME_STACKTRACE=yes`", + LCompilers::diag::Level::Error, + LCompilers::diag::Stage::Semantic, {}) + ); + std::cerr << diagnostics.render(lm, compiler_options); + return 1; +#endif + } + LCompilers::PythonCompiler fe(compiler_options); + LCompilers::LLVMEvaluator e(compiler_options.target); + auto asr_to_llvm_start = std::chrono::high_resolution_clock::now(); + LCompilers::Result> + res = fe.get_llvm3(*asr, pass_manager, diagnostics, infile); + auto asr_to_llvm_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("ASR to LLVM", std::chrono::duration(asr_to_llvm_end - asr_to_llvm_start).count())); + + std::cerr << diagnostics.render(lm, compiler_options); + if (!res.ok) { + LCOMPILERS_ASSERT(diagnostics.has_error()) + print_time_report(times, time_report); + return 3; + } + + auto llvm_start = std::chrono::high_resolution_clock::now(); + e.add_module(std::move(res.result)); + try { + e.voidfn("__module___main_____main__global_stmts"); + } catch (LCompilers::LCompilersException&) {} + auto llvm_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("LLVM to binary", std::chrono::duration(llvm_end - llvm_start).count())); + print_time_report(times, time_report); + return 0; +} + #endif void do_print_rtl_header_dir() { @@ -1560,6 +1660,7 @@ int main(int argc, char *argv[]) bool print_rtl_header_dir = false; bool print_rtl_dir = false; bool separate_compilation = false; + bool to_jit = false; std::string arg_fmt_file; // int arg_fmt_indent = 4; @@ -1593,6 +1694,7 @@ int main(int argc, char *argv[]) app.add_option("-I", compiler_options.import_paths, "Specify the paths" "to look for the module")->allow_extra_args(false); // app.add_option("-J", arg_J, "Where to save mod files"); + app.add_flag("--jit", to_jit, "Execute the program using just-in-time (JIT) compiler"); app.add_flag("-g", compiler_options.emit_debug_info, "Compile with debugging information"); app.add_flag("--debug-with-line-column", compiler_options.emit_debug_line_column, "Convert the linear location info into line + column in the debugging information"); @@ -1795,6 +1897,14 @@ int main(int argc, char *argv[]) return 1; } + if (to_jit) { + compiler_options.emit_debug_info = false; + compiler_options.emit_debug_line_column = false; + compiler_options.generate_object_code = false; + return execute_python_using_jit(arg_file, runtime_library_dir, + lpython_pass_manager, compiler_options, time_report); + } + std::string outfile; std::string basename; basename = remove_extension(arg_file); From 5a96c18e05a560fd59fb5ebd01c805be16ab6587 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Fri, 8 Mar 2024 15:23:09 +0530 Subject: [PATCH 02/22] execute global_init & global_stmts after checking their existence remove the try catch block around the execution of global_init & global_stmts --- src/bin/lpython.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 27d645b5f5..a3df140bf8 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -979,12 +979,23 @@ int execute_python_using_jit( print_time_report(times, time_report); return 3; } + std::unique_ptr m = std::move(res.result); auto llvm_start = std::chrono::high_resolution_clock::now(); - e.add_module(std::move(res.result)); - try { + + bool call_init = false; + bool call_stmts = false; + if (m->get_return_type("__module___main_____main__global_init") == "void") + call_init = true; + if (m->get_return_type("__module___main_____main__global_stmts") == "void") + call_stmts = true; + + e.add_module(std::move(m)); + if (call_init) + e.voidfn("__module___main_____main__global_init"); + if (call_stmts) e.voidfn("__module___main_____main__global_stmts"); - } catch (LCompilers::LCompilersException&) {} + auto llvm_end = std::chrono::high_resolution_clock::now(); times.push_back(std::make_pair("LLVM to binary", std::chrono::duration(llvm_end - llvm_start).count())); print_time_report(times, time_report); From 8a0a847ce9416a7a471001f1b07de801365b1254 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Fri, 8 Mar 2024 15:26:28 +0530 Subject: [PATCH 03/22] CI fix & raise error for non-llvm backend with jit --- src/bin/lpython.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index a3df140bf8..5be1670080 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -1908,14 +1908,6 @@ int main(int argc, char *argv[]) return 1; } - if (to_jit) { - compiler_options.emit_debug_info = false; - compiler_options.emit_debug_line_column = false; - compiler_options.generate_object_code = false; - return execute_python_using_jit(arg_file, runtime_library_dir, - lpython_pass_manager, compiler_options, time_report); - } - std::string outfile; std::string basename; basename = remove_extension(arg_file); @@ -2032,6 +2024,20 @@ int main(int argc, char *argv[]) if (endswith(arg_file, ".py")) { int err = 0; + if (to_jit) { +#ifdef HAVE_LFORTRAN_LLVM + if (backend != Backend::llvm) + std::cerr << "JIT option is only available with LLVM backend" << std::endl; + + compiler_options.emit_debug_info = false; + compiler_options.emit_debug_line_column = false; + compiler_options.generate_object_code = false; + return execute_python_using_jit(arg_file, runtime_library_dir, + lpython_pass_manager, compiler_options, time_report); +#else + std::cerr << "Just-In-Time Compilation of Python files requires the LLVM backend to be enabled. Recompile with `WITH_LLVM=yes`." << std::endl; +#endif + } if (backend == Backend::x86) { err = compile_to_binary_x86(arg_file, outfile, runtime_library_dir, compiler_options, time_report); From 895ae8f73ce84b8d33857175b5c9033bc7eb5203 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Sat, 9 Mar 2024 20:35:28 +0530 Subject: [PATCH 04/22] combined `compile_python_to_object_file` and `execute_python_using_jit` to `compile_python_using_llvm` removed duplicated code --- src/bin/lpython.cpp | 151 ++++++++++---------------------------------- 1 file changed, 33 insertions(+), 118 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 5be1670080..429c3f844b 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -792,13 +792,17 @@ int emit_llvm(const std::string &infile, return 0; } -int compile_python_to_object_file( +/* + Compiles python to object file, if `to_jit` is false + otherwise execute python code using llvm JIT +*/ +int compile_python_using_llvm( const std::string &infile, const std::string &outfile, const std::string &runtime_library_dir, LCompilers::PassManager& pass_manager, CompilerOptions &compiler_options, - bool time_report, bool arg_c=false) + bool time_report, bool arg_c=false, bool to_jit=false) { Allocator al(4*1024); LCompilers::diag::Diagnostics diagnostics; @@ -869,7 +873,6 @@ int compile_python_to_object_file( } LCompilers::PythonCompiler fe(compiler_options); LCompilers::LLVMEvaluator e(compiler_options.target); - std::unique_ptr m; auto asr_to_llvm_start = std::chrono::high_resolution_clock::now(); LCompilers::Result> res = fe.get_llvm3(*asr, pass_manager, diagnostics, infile); @@ -882,123 +885,34 @@ int compile_python_to_object_file( print_time_report(times, time_report); return 3; } - m = std::move(res.result); - auto llvm_start = std::chrono::high_resolution_clock::now(); - e.save_object_file(*(m->m_m), outfile); - auto llvm_end = std::chrono::high_resolution_clock::now(); - times.push_back(std::make_pair("LLVM to binary", std::chrono::duration(llvm_end - llvm_start).count())); - print_time_report(times, time_report); - return 0; -} - -int execute_python_using_jit( - const std::string &infile, - const std::string &runtime_library_dir, - LCompilers::PassManager& pass_manager, - CompilerOptions &compiler_options, - bool time_report) -{ - Allocator al(4*1024); - LCompilers::diag::Diagnostics diagnostics; - LCompilers::LocationManager lm; - std::vector>times; - { - LCompilers::LocationManager::FileLocations fl; - fl.in_filename = infile; - lm.files.push_back(fl); + std::unique_ptr m = std::move(res.result); - auto file_reading_start = std::chrono::high_resolution_clock::now(); - std::string input = LCompilers::read_file(infile); - auto file_reading_end = std::chrono::high_resolution_clock::now(); - times.push_back(std::make_pair("File reading", std::chrono::duration - (file_reading_end - file_reading_start).count())); + if (to_jit) { + auto llvm_start = std::chrono::high_resolution_clock::now(); - lm.init_simple(input); - lm.file_ends.push_back(input.size()); - } - auto parsing_start = std::chrono::high_resolution_clock::now(); - LCompilers::Result r = parse_python_file( - al, runtime_library_dir, infile, diagnostics, 0, compiler_options.new_parser); - auto parsing_end = std::chrono::high_resolution_clock::now(); - times.push_back(std::make_pair("Parsing", std::chrono::duration(parsing_end - parsing_start).count())); - std::cerr << diagnostics.render(lm, compiler_options); - if (!r.ok) { - print_time_report(times, time_report); - return 1; - } + bool call_init = false; + bool call_stmts = false; + if (m->get_return_type("__module___main_____main__global_init") == "void") + call_init = true; + if (m->get_return_type("__module___main_____main__global_stmts") == "void") + call_stmts = true; - // Src -> AST -> ASR - LCompilers::LPython::AST::ast_t* ast = r.result; - diagnostics.diagnostics.clear(); - auto ast_to_asr_start = std::chrono::high_resolution_clock::now(); - LCompilers::Result - r1 = LCompilers::LPython::python_ast_to_asr(al, lm, nullptr, *ast, diagnostics, compiler_options, - !(false && compiler_options.po.disable_main), "__main__", infile); + e.add_module(std::move(m)); + if (call_init) + e.voidfn("__module___main_____main__global_init"); + if (call_stmts) + e.voidfn("__module___main_____main__global_stmts"); - auto ast_to_asr_end = std::chrono::high_resolution_clock::now(); - times.push_back(std::make_pair("AST to ASR", std::chrono::duration(ast_to_asr_end - ast_to_asr_start).count())); - std::cerr << diagnostics.render(lm, compiler_options); - if (!r1.ok) { - LCOMPILERS_ASSERT(diagnostics.has_error()) + auto llvm_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("LLVM JIT execution", std::chrono::duration(llvm_end - llvm_start).count())); print_time_report(times, time_report); - return 2; - } - LCompilers::ASR::TranslationUnit_t* asr = r1.result; - if( compiler_options.po.disable_main ) { - int err = LCompilers::LPython::save_pyc_files(*asr, infile); - if( err ) { - return err; - } - } - diagnostics.diagnostics.clear(); - - // ASR -> LLVM - if (compiler_options.emit_debug_info) { -#ifndef HAVE_RUNTIME_STACKTRACE - diagnostics.add(LCompilers::diag::Diagnostic( - "The `runtime stacktrace` is not enabled. To get the stacktraces, " - "re-build LPython with `-DWITH_RUNTIME_STACKTRACE=yes`", - LCompilers::diag::Level::Error, - LCompilers::diag::Stage::Semantic, {}) - ); - std::cerr << diagnostics.render(lm, compiler_options); - return 1; -#endif - } - LCompilers::PythonCompiler fe(compiler_options); - LCompilers::LLVMEvaluator e(compiler_options.target); - auto asr_to_llvm_start = std::chrono::high_resolution_clock::now(); - LCompilers::Result> - res = fe.get_llvm3(*asr, pass_manager, diagnostics, infile); - auto asr_to_llvm_end = std::chrono::high_resolution_clock::now(); - times.push_back(std::make_pair("ASR to LLVM", std::chrono::duration(asr_to_llvm_end - asr_to_llvm_start).count())); - - std::cerr << diagnostics.render(lm, compiler_options); - if (!res.ok) { - LCOMPILERS_ASSERT(diagnostics.has_error()) + } else { + auto llvm_start = std::chrono::high_resolution_clock::now(); + e.save_object_file(*(m->m_m), outfile); + auto llvm_end = std::chrono::high_resolution_clock::now(); + times.push_back(std::make_pair("LLVM to binary", std::chrono::duration(llvm_end - llvm_start).count())); print_time_report(times, time_report); - return 3; } - std::unique_ptr m = std::move(res.result); - - auto llvm_start = std::chrono::high_resolution_clock::now(); - - bool call_init = false; - bool call_stmts = false; - if (m->get_return_type("__module___main_____main__global_init") == "void") - call_init = true; - if (m->get_return_type("__module___main_____main__global_stmts") == "void") - call_stmts = true; - - e.add_module(std::move(m)); - if (call_init) - e.voidfn("__module___main_____main__global_init"); - if (call_stmts) - e.voidfn("__module___main_____main__global_stmts"); - - auto llvm_end = std::chrono::high_resolution_clock::now(); - times.push_back(std::make_pair("LLVM to binary", std::chrono::duration(llvm_end - llvm_start).count())); - print_time_report(times, time_report); return 0; } @@ -2010,7 +1924,7 @@ int main(int argc, char *argv[]) if (arg_c) { if (backend == Backend::llvm) { #ifdef HAVE_LFORTRAN_LLVM - return compile_python_to_object_file(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report, + return compile_python_using_llvm(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report, arg_c); #else std::cerr << "The -c option requires the LLVM backend to be enabled. Recompile with `WITH_LLVM=yes`." << std::endl; @@ -2032,10 +1946,11 @@ int main(int argc, char *argv[]) compiler_options.emit_debug_info = false; compiler_options.emit_debug_line_column = false; compiler_options.generate_object_code = false; - return execute_python_using_jit(arg_file, runtime_library_dir, - lpython_pass_manager, compiler_options, time_report); + err = compile_python_using_llvm(arg_file, nullptr, runtime_library_dir, + lpython_pass_manager, compiler_options, time_report, false, true); #else - std::cerr << "Just-In-Time Compilation of Python files requires the LLVM backend to be enabled. Recompile with `WITH_LLVM=yes`." << std::endl; + std::cerr << "Just-In-Time Compilation of Python files requires the LLVM backend to be enabled." + " Recompile with `WITH_LLVM=yes`." << std::endl; #endif } if (backend == Backend::x86) { @@ -2058,7 +1973,7 @@ int main(int argc, char *argv[]) } else if (backend == Backend::llvm) { #ifdef HAVE_LFORTRAN_LLVM std::string tmp_o = outfile + ".tmp.o"; - err = compile_python_to_object_file(arg_file, tmp_o, runtime_library_dir, + err = compile_python_using_llvm(arg_file, tmp_o, runtime_library_dir, lpython_pass_manager, compiler_options, time_report); if (err != 0) return err; err = link_executable({tmp_o}, outfile, runtime_library_dir, From 865994879a56e86f576070e4df4574d4bb867124 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Sun, 10 Mar 2024 19:11:37 +0530 Subject: [PATCH 05/22] Add testing mechanism --- integration_tests/CMakeLists.txt | 10 ++++++++++ src/bin/lpython.cpp | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index e5f6ff3aa8..3704020d81 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -101,6 +101,9 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) target_link_libraries(${name} lpython_rtlib) add_test(${name} ${CMAKE_CURRENT_BINARY_DIR}/${name}) + add_test( + NAME "${name}_jit" + COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") endif() @@ -360,6 +363,13 @@ macro(RUN) list(REMOVE_ITEM RUN_LABELS cpython cpython_sym) # remove cpython, cpython_sym, from --fast test RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) endif() + + # if (llvm IN_LIST RUN_LABELS) + # unset(RUN_EXTRA_ARGS) + # set(RUN_EXTRA_ARGS --jit) + # set(RUN_NAME "${RUN_NAME}_JIT") + # RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) + # endif() endmacro(RUN) # only compiles till object file diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 429c3f844b..8e8572ec00 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -1921,7 +1921,7 @@ int main(int argc, char *argv[]) } } - if (arg_c) { + if (arg_c && !to_jit) { if (backend == Backend::llvm) { #ifdef HAVE_LFORTRAN_LLVM return compile_python_using_llvm(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report, @@ -1946,7 +1946,7 @@ int main(int argc, char *argv[]) compiler_options.emit_debug_info = false; compiler_options.emit_debug_line_column = false; compiler_options.generate_object_code = false; - err = compile_python_using_llvm(arg_file, nullptr, runtime_library_dir, + return compile_python_using_llvm(arg_file, "", runtime_library_dir, lpython_pass_manager, compiler_options, time_report, false, true); #else std::cerr << "Just-In-Time Compilation of Python files requires the LLVM backend to be enabled." From 602359a48749f8dd36cc9cd7e6017e56f4d809dc Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Tue, 19 Mar 2024 18:30:05 +0530 Subject: [PATCH 06/22] Testing with cpython and symengine --- integration_tests/CMakeLists.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index 3704020d81..aa7dbe6561 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -121,6 +121,9 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) target_link_libraries(${name} lpython_rtlib Python::Python) add_test(${name} ${CMAKE_CURRENT_BINARY_DIR}/${name}) + add_test( + NAME "${name}_jit" + COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") endif() @@ -142,6 +145,9 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM endif() target_link_libraries(${name} lpython_rtlib ${SYMENGINE_LIB}) add_test(${name} ${CMAKE_CURRENT_BINARY_DIR}/${name}) + add_test( + NAME "${name}_jit" + COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") endif() @@ -363,13 +369,6 @@ macro(RUN) list(REMOVE_ITEM RUN_LABELS cpython cpython_sym) # remove cpython, cpython_sym, from --fast test RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) endif() - - # if (llvm IN_LIST RUN_LABELS) - # unset(RUN_EXTRA_ARGS) - # set(RUN_EXTRA_ARGS --jit) - # set(RUN_NAME "${RUN_NAME}_JIT") - # RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) - # endif() endmacro(RUN) # only compiles till object file From 4ae8463dea018efd54a8b0748c62400f6e9bbfea Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Sat, 23 Mar 2024 19:32:27 +0530 Subject: [PATCH 07/22] skipping jit tests with external dependency --- integration_tests/CMakeLists.txt | 34 ++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index aa7dbe6561..254ae3f2ca 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -76,8 +76,9 @@ message("LPYTHON_RTLIB_DIR: ${LPYTHON_RTLIB_DIR}") message("LPYTHON_RTLIB_LIBRARY: ${LPYTHON_RTLIB_LIBRARY}") -macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) +macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN RUN_SKIP_ON_JIT) set(fail ${${RUN_FAIL}}) + set(skip_jit ${${RUN_SKIP_ON_JIT}}) set(name ${${RUN_NAME}}) set(file_name ${${RUN_FILE_NAME}}) set(labels ${${RUN_LABELS}}) @@ -106,9 +107,16 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") + set_tests_properties("${name}_jit" PROPERTIES LABELS "${labels}") endif() if (${fail}) set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) + set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) + endif() + if (${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) endif() elseif (KIND STREQUAL "llvm_py") add_custom_command( @@ -126,9 +134,16 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") + set_tests_properties("${name}_jit" PROPERTIES LABELS "${labels}") endif() if (${fail}) set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) + set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) + endif() + if (${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) endif() elseif(KIND STREQUAL "llvm_sym") add_custom_command( @@ -150,9 +165,16 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") + set_tests_properties("${name}_jit" PROPERTIES LABELS "${labels}") endif() if (${fail}) set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) + set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) + endif() + if (${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) endif() elseif(KIND STREQUAL "c") add_custom_command( @@ -321,7 +343,7 @@ endmacro(RUN_UTIL) macro(RUN) set(options FAIL NOFAST NOMOD ENABLE_CPYTHON LINK_NUMPY NO_WARNINGS) - set(oneValueArgs NAME IMPORT_PATH COPY_TO_BIN REQ_PY_VER) + set(oneValueArgs NAME IMPORT_PATH COPY_TO_BIN REQ_PY_VER SKIP_ON_JIT) set(multiValueArgs LABELS EXTRAFILES) cmake_parse_arguments(RUN "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -360,14 +382,14 @@ macro(RUN) endif() if (NOT FAST) - RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) + RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN RUN_SKIP_ON_JIT) endif() if ((FAST) AND (NOT RUN_NOFAST)) set(RUN_EXTRA_ARGS ${RUN_EXTRA_ARGS} --fast) set(RUN_NAME "${RUN_NAME}_FAST") list(REMOVE_ITEM RUN_LABELS cpython cpython_sym) # remove cpython, cpython_sym, from --fast test - RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN) + RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOMOD RUN_EXTRA_ARGS RUN_COPY_TO_BIN RUN_SKIP_ON_JIT) endif() endmacro(RUN) @@ -588,7 +610,7 @@ RUN(NAME test_dict_13 LABELS cpython llvm c) RUN(NAME test_dict_bool LABELS cpython llvm) RUN(NAME test_dict_increment LABELS cpython llvm) RUN(NAME test_dict_keys_values LABELS cpython llvm) -RUN(NAME test_dict_nested1 LABELS cpython llvm) +RUN(NAME test_dict_nested1 LABELS cpython llvm SKIP_ON_JIT 139) RUN(NAME test_set_len LABELS cpython llvm) RUN(NAME test_set_add LABELS cpython llvm) RUN(NAME test_set_remove LABELS cpython llvm) @@ -815,7 +837,7 @@ RUN(NAME comp_01 LABELS cpython llvm c wasm wasm_x64) RUN(NAME bit_operations_i32 LABELS cpython llvm c wasm wasm_x64) RUN(NAME bit_operations_i64 LABELS cpython llvm c wasm) -RUN(NAME test_argv_01 LABELS cpython llvm NOFAST) +RUN(NAME test_argv_01 LABELS cpython llvm NOFAST SKIP_ON_JIT 1) RUN(NAME global_syms_01 LABELS cpython llvm c) RUN(NAME global_syms_02 LABELS cpython llvm c) RUN(NAME global_syms_03_b LABELS cpython llvm c) From c138728bbea3540cbc28d3d143acf05375a2db94 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Sat, 23 Mar 2024 19:53:05 +0530 Subject: [PATCH 08/22] skipping testing JIT that depend on syms or cpython --- integration_tests/CMakeLists.txt | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index 254ae3f2ca..45e271f98b 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -140,11 +140,12 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) endif() - if (${skip_jit}) - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) - elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) - endif() + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) # TODO: remove this line and uncomment below + # if (${skip_jit}) + # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + # elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) + # endif() elseif(KIND STREQUAL "llvm_sym") add_custom_command( OUTPUT ${name}.o @@ -171,11 +172,12 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) endif() - if (${skip_jit}) - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) - elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) - endif() + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) # TODO: remove this line and uncomment below + # if (${skip_jit}) + # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + # elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) + # endif() elseif(KIND STREQUAL "c") add_custom_command( OUTPUT ${name}.c From 31b4d023b114681c05f2411c09a550159afd4dd2 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Sun, 24 Mar 2024 09:37:15 +0530 Subject: [PATCH 09/22] support to use cpython and symengine with JIT --- integration_tests/CMakeLists.txt | 24 +++++++++--------- src/bin/lpython.cpp | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index 45e271f98b..8e59b420d6 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -140,12 +140,11 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) endif() - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) # TODO: remove this line and uncomment below - # if (${skip_jit}) - # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) - # elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) - # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) - # endif() + if (${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) + endif() elseif(KIND STREQUAL "llvm_sym") add_custom_command( OUTPUT ${name}.o @@ -163,7 +162,7 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM add_test(${name} ${CMAKE_CURRENT_BINARY_DIR}/${name}) add_test( NAME "${name}_jit" - COMMAND ${LPYTHON} --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) + COMMAND ${LPYTHON} --enable-symengine --jit ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py) if (labels) set_tests_properties(${name} PROPERTIES LABELS "${labels}") set_tests_properties("${name}_jit" PROPERTIES LABELS "${labels}") @@ -172,12 +171,11 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) endif() - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) # TODO: remove this line and uncomment below - # if (${skip_jit}) - # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) - # elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) - # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) - # endif() + if (${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) + endif() elseif(KIND STREQUAL "c") add_custom_command( OUTPUT ${name}.c diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 8e8572ec00..1442bd73df 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -3,6 +3,12 @@ #include #include +#if (defined (__linux__)) or (defined (__APPLE__)) +#include +#else +// TODO: support windows +#endif + #define CLI11_HAS_FILESYSTEM 0 #include @@ -888,6 +894,37 @@ int compile_python_using_llvm( std::unique_ptr m = std::move(res.result); if (to_jit) { + void *cpython_lib = nullptr; + void *symengine_lib = nullptr; + if (compiler_options.enable_cpython) { + std::string conda_prefix = std::getenv("CONDA_PREFIX"); +#if defined (__linux__) + cpython_lib = dlopen((conda_prefix + "/lib/libpython3.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); +#elif defined (__APPLE__) + cpython_lib = dlopen((conda_prefix + "/lib/libpython3.dylib").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); +#else + // TODO: support windows +#endif + if (!cpython_lib) { + std::cerr << "Could not load \"libpython3.so\"" << std::endl; + return 1; + } + } + if (compiler_options.enable_symengine) { + std::string conda_prefix = std::getenv("CONDA_PREFIX"); +#if defined (__linux__) + symengine_lib = dlopen((conda_prefix + "/lib/libsymengine.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); +#elif defined (__APPLE__) + symengine_lib = dlopen((conda_prefix + "/lib/libsymengine.dylib").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); +#else + // TODO: support windows +#endif + if (!symengine_lib) { + std::cerr << "Could not load \"libsymengine.so\"\n" << std::endl; + return 1; + } + } + auto llvm_start = std::chrono::high_resolution_clock::now(); bool call_init = false; @@ -903,6 +940,11 @@ int compile_python_using_llvm( if (call_stmts) e.voidfn("__module___main_____main__global_stmts"); + if (cpython_lib) + dlclose(cpython_lib); + if (symengine_lib) + dlclose(symengine_lib); + auto llvm_end = std::chrono::high_resolution_clock::now(); times.push_back(std::make_pair("LLVM JIT execution", std::chrono::duration(llvm_end - llvm_start).count())); print_time_report(times, time_report); From f1cd329be8af197a00953b6c594d1222b4fac765 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sun, 24 Mar 2024 10:53:10 +0530 Subject: [PATCH 10/22] Trigger CI/CD From caec554c20c33eee28d3c31927c006084fc4a13c Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sun, 24 Mar 2024 10:59:52 +0530 Subject: [PATCH 11/22] windows fix --- src/bin/lpython.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 1442bd73df..6ec2b70f37 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -6,7 +6,7 @@ #if (defined (__linux__)) or (defined (__APPLE__)) #include #else -// TODO: support windows +#include #endif #define CLI11_HAS_FILESYSTEM 0 @@ -894,8 +894,13 @@ int compile_python_using_llvm( std::unique_ptr m = std::move(res.result); if (to_jit) { +#if (defined (__linux__)) or (defined (__APPLE__)) void *cpython_lib = nullptr; void *symengine_lib = nullptr; +#else + HINSTANCE cpython_lib = nullptr; + HINSTANCE symengine_lib = nullptr; +#endif if (compiler_options.enable_cpython) { std::string conda_prefix = std::getenv("CONDA_PREFIX"); #if defined (__linux__) @@ -903,10 +908,10 @@ int compile_python_using_llvm( #elif defined (__APPLE__) cpython_lib = dlopen((conda_prefix + "/lib/libpython3.dylib").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); #else - // TODO: support windows + cpython_lib = LoadLibrary((conda_prefix + "\\python3.dll").c_str()); #endif if (!cpython_lib) { - std::cerr << "Could not load \"libpython3.so\"" << std::endl; + std::cerr << "Could not load \"python3\" library\n"; return 1; } } @@ -917,10 +922,10 @@ int compile_python_using_llvm( #elif defined (__APPLE__) symengine_lib = dlopen((conda_prefix + "/lib/libsymengine.dylib").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); #else - // TODO: support windows + symengine_lib = LoadLibrary((conda_prefix + "\\Library\\bin\\symengine-0.11.dll").c_str()); #endif if (!symengine_lib) { - std::cerr << "Could not load \"libsymengine.so\"\n" << std::endl; + std::cerr << "Could not load \"symengine\" library\n"; return 1; } } @@ -940,10 +945,17 @@ int compile_python_using_llvm( if (call_stmts) e.voidfn("__module___main_____main__global_stmts"); +#if (defined (__linux__)) or (defined (__APPLE__)) if (cpython_lib) dlclose(cpython_lib); if (symengine_lib) dlclose(symengine_lib); +#else + if (cpython_lib) + FreeLibrary(cpython_lib); + if (symengine_lib) + FreeLibrary(symengine_lib); +#endif auto llvm_end = std::chrono::high_resolution_clock::now(); times.push_back(std::make_pair("LLVM JIT execution", std::chrono::duration(llvm_end - llvm_start).count())); From ed6dd6e09901f4fa8d485c4817eb2463e1fd8891 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Sun, 24 Mar 2024 11:46:12 +0530 Subject: [PATCH 12/22] WASM ci fix --- src/bin/lpython.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 6ec2b70f37..8d5bdfa80d 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -5,7 +5,7 @@ #if (defined (__linux__)) or (defined (__APPLE__)) #include -#else +#elif (defined (WIN32)) #include #endif From 3be9723f6a38b1cd656639dd370dedf19fda2703 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Mon, 1 Apr 2024 07:57:18 +0530 Subject: [PATCH 13/22] allow few tests to fail under `--jit` --- integration_tests/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index 8e59b420d6..ab3657bcbe 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -114,7 +114,8 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) endif() if (${skip_jit}) - set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) # Remove this line and uncomment below after fixing #2595 + # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) endif() From ca42626f5a2f8fde38a00bdb106d68ca0a68771d Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 4 Apr 2024 11:02:05 +0530 Subject: [PATCH 14/22] fix for #2595 --- integration_tests/CMakeLists.txt | 5 ++--- src/libasr/codegen/llvm_utils.cpp | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index ab3657bcbe..b7f2691eda 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -114,8 +114,7 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_NOM set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) endif() if (${skip_jit}) - set_tests_properties("${name}_jit" PROPERTIES WILL_FAIL TRUE) # Remove this line and uncomment below after fixing #2595 - # set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) + set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE ${skip_jit}) elseif (DEFINED ${RUN_EXTRAFILES} AND NOT ${RUN_FAIL}) set_tests_properties("${name}_jit" PROPERTIES SKIP_RETURN_CODE 1) endif() @@ -611,7 +610,7 @@ RUN(NAME test_dict_13 LABELS cpython llvm c) RUN(NAME test_dict_bool LABELS cpython llvm) RUN(NAME test_dict_increment LABELS cpython llvm) RUN(NAME test_dict_keys_values LABELS cpython llvm) -RUN(NAME test_dict_nested1 LABELS cpython llvm SKIP_ON_JIT 139) +RUN(NAME test_dict_nested1 LABELS cpython llvm) RUN(NAME test_set_len LABELS cpython llvm) RUN(NAME test_set_add LABELS cpython llvm) RUN(NAME test_set_remove LABELS cpython llvm) diff --git a/src/libasr/codegen/llvm_utils.cpp b/src/libasr/codegen/llvm_utils.cpp index f39922d49a..9e391ae89f 100644 --- a/src/libasr/codegen/llvm_utils.cpp +++ b/src/libasr/codegen/llvm_utils.cpp @@ -2120,17 +2120,15 @@ namespace LCompilers { throw LCompilersException("list for " + type_code + " not declared yet."); } int32_t type_size = std::get<1>(typecode2listtype[type_code]); - llvm::Value* arg_size = llvm::ConstantInt::get(context, - llvm::APInt(32, type_size * initial_capacity)); - - llvm::Value* list_data = LLVM::lfortran_malloc(context, module, *builder, - arg_size); + llvm::Value* llvm_type_size = llvm::ConstantInt::get(context, llvm::APInt(32, type_size)); + llvm::Value* current_capacity = llvm::ConstantInt::get(context, llvm::APInt(32, initial_capacity)); + llvm::Value* list_data = LLVM::lfortran_calloc(context, module, *builder, + current_capacity, llvm_type_size); llvm::Type* el_type = std::get<2>(typecode2listtype[type_code]); list_data = builder->CreateBitCast(list_data, el_type->getPointerTo()); llvm::Value* list_data_ptr = get_pointer_to_list_data(list); builder->CreateStore(list_data, list_data_ptr); llvm::Value* current_end_point = llvm::ConstantInt::get(context, llvm::APInt(32, n)); - llvm::Value* current_capacity = llvm::ConstantInt::get(context, llvm::APInt(32, initial_capacity)); builder->CreateStore(current_end_point, get_pointer_to_current_end_point(list)); builder->CreateStore(current_capacity, get_pointer_to_current_capacity(list)); } @@ -2143,8 +2141,8 @@ namespace LCompilers { } int32_t type_size = std::get<1>(typecode2listtype[type_code]); llvm::Value* llvm_type_size = llvm::ConstantInt::get(context, llvm::APInt(32, type_size)); - llvm::Value* arg_size = builder->CreateMul(llvm_type_size, initial_capacity); - llvm::Value* list_data = LLVM::lfortran_malloc(context, module, *builder, arg_size); + llvm::Value* list_data = LLVM::lfortran_calloc(context, module, *builder, + initial_capacity, llvm_type_size); llvm::Type* el_type = std::get<2>(typecode2listtype[type_code]); list_data = builder->CreateBitCast(list_data, el_type->getPointerTo()); @@ -2288,10 +2286,9 @@ namespace LCompilers { builder->CreateStore(src_capacity, dest_capacity_ptr); llvm::Value* src_data = LLVM::CreateLoad(*builder, get_pointer_to_list_data(src)); int32_t type_size = std::get<1>(typecode2listtype[src_type_code]); - llvm::Value* arg_size = builder->CreateMul(llvm::ConstantInt::get(context, - llvm::APInt(32, type_size)), src_capacity); - llvm::Value* copy_data = LLVM::lfortran_malloc(context, *module, *builder, - arg_size); + llvm::Value* llvm_type_size = llvm::ConstantInt::get(context, llvm::APInt(32, type_size)); + llvm::Value* copy_data = LLVM::lfortran_calloc(context, *module, *builder, + src_capacity, llvm_type_size); llvm::Type* el_type = std::get<2>(typecode2listtype[src_type_code]); copy_data = builder->CreateBitCast(copy_data, el_type->getPointerTo()); @@ -2346,6 +2343,8 @@ namespace LCompilers { // end llvm_utils->start_new_block(loopend); } else { + llvm::Value* arg_size = builder->CreateMul(llvm::ConstantInt::get(context, + llvm::APInt(32, type_size)), src_capacity); builder->CreateMemCpy(copy_data, llvm::MaybeAlign(), src_data, llvm::MaybeAlign(), arg_size); builder->CreateStore(copy_data, get_pointer_to_list_data(dest)); From 4fdfe04e695ba7b31ee34747ad8b56e3286210f7 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 4 Apr 2024 11:28:27 +0530 Subject: [PATCH 15/22] factored out loading cpython and symengine libraries --- src/bin/lpython.cpp | 63 ++++++++----------------------------------- src/lpython/utils.cpp | 48 +++++++++++++++++++++++++++++++++ src/lpython/utils.h | 18 +++++++++++++ 3 files changed, 77 insertions(+), 52 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 8d5bdfa80d..68d4ac177a 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -3,12 +3,6 @@ #include #include -#if (defined (__linux__)) or (defined (__APPLE__)) -#include -#elif (defined (WIN32)) -#include -#endif - #define CLI11_HAS_FILESYSTEM 0 #include @@ -894,41 +888,13 @@ int compile_python_using_llvm( std::unique_ptr m = std::move(res.result); if (to_jit) { -#if (defined (__linux__)) or (defined (__APPLE__)) - void *cpython_lib = nullptr; - void *symengine_lib = nullptr; -#else - HINSTANCE cpython_lib = nullptr; - HINSTANCE symengine_lib = nullptr; -#endif - if (compiler_options.enable_cpython) { - std::string conda_prefix = std::getenv("CONDA_PREFIX"); -#if defined (__linux__) - cpython_lib = dlopen((conda_prefix + "/lib/libpython3.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); -#elif defined (__APPLE__) - cpython_lib = dlopen((conda_prefix + "/lib/libpython3.dylib").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); -#else - cpython_lib = LoadLibrary((conda_prefix + "\\python3.dll").c_str()); -#endif - if (!cpython_lib) { - std::cerr << "Could not load \"python3\" library\n"; - return 1; - } - } - if (compiler_options.enable_symengine) { - std::string conda_prefix = std::getenv("CONDA_PREFIX"); -#if defined (__linux__) - symengine_lib = dlopen((conda_prefix + "/lib/libsymengine.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); -#elif defined (__APPLE__) - symengine_lib = dlopen((conda_prefix + "/lib/libsymengine.dylib").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); -#else - symengine_lib = LoadLibrary((conda_prefix + "\\Library\\bin\\symengine-0.11.dll").c_str()); -#endif - if (!symengine_lib) { - std::cerr << "Could not load \"symengine\" library\n"; - return 1; - } - } + LCompilers::LPython::DynamicLibrary cpython_lib; + LCompilers::LPython::DynamicLibrary symengine_lib; + + if (compiler_options.enable_cpython) + LCompilers::LPython::open_cpython_library(cpython_lib); + if (compiler_options.enable_symengine) + LCompilers::LPython::open_symengine_library(symengine_lib); auto llvm_start = std::chrono::high_resolution_clock::now(); @@ -945,17 +911,10 @@ int compile_python_using_llvm( if (call_stmts) e.voidfn("__module___main_____main__global_stmts"); -#if (defined (__linux__)) or (defined (__APPLE__)) - if (cpython_lib) - dlclose(cpython_lib); - if (symengine_lib) - dlclose(symengine_lib); -#else - if (cpython_lib) - FreeLibrary(cpython_lib); - if (symengine_lib) - FreeLibrary(symengine_lib); -#endif + if (compiler_options.enable_cpython) + LCompilers::LPython::close_cpython_library(cpython_lib); + if (compiler_options.enable_symengine) + LCompilers::LPython::close_symengine_library(symengine_lib); auto llvm_end = std::chrono::high_resolution_clock::now(); times.push_back(std::make_pair("LLVM JIT execution", std::chrono::duration(llvm_end - llvm_start).count())); diff --git a/src/lpython/utils.cpp b/src/lpython/utils.cpp index 849cf540ed..79175710a4 100644 --- a/src/lpython/utils.cpp +++ b/src/lpython/utils.cpp @@ -126,6 +126,54 @@ bool path_exists(std::string path) { } } +void open_cpython_library(DynamicLibrary &l) { + std::string conda_prefix = std::getenv("CONDA_PREFIX"); +#if defined (__linux__) + l.l = dlopen((conda_prefix + "/lib/libpython3.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); +#elif defined (__APPLE__) + l.l = dlopen((conda_prefix + "/lib/libpython3.dylib").c_str(), RTLD_GLOBAL | RTLD_NOW); +#else + l.l = LoadLibrary((conda_prefix + "\\python3.dll").c_str()); +#endif + + if (l.l == nullptr) + throw "Could not open CPython library"; +} + +void close_cpython_library(DynamicLibrary &l) { +#if (defined (__linux__)) or (defined (__APPLE__)) + dlclose(l.l); + l.l = nullptr; +#else + FreeLibrary(l.l); + l.l = nullptr; +#endif +} + +void open_symengine_library(DynamicLibrary &l) { + std::string conda_prefix = std::getenv("CONDA_PREFIX"); +#if defined (__linux__) + l.l = dlopen((conda_prefix + "/lib/libsymengine.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW); +#elif defined (__APPLE__) + l.l = dlopen((conda_prefix + "/lib/libsymengine.dylib").c_str(), RTLD_GLOBAL | RTLD_NOW); +#else + l.l = LoadLibrary((conda_prefix + "\\Library\\bin\\symengine-0.11.dll").c_str()); +#endif + + if (l.l == nullptr) + throw "Could not open SymEngine library"; +} + +void close_symengine_library(DynamicLibrary &l) { +#if (defined (__linux__)) or (defined (__APPLE__)) + dlclose(l.l); + l.l = nullptr; +#else + FreeLibrary(l.l); + l.l = nullptr; +#endif +} + // Decodes the exit status code of the process (in Unix) // See `WEXITSTATUS` for more information. // https://stackoverflow.com/a/27117435/15913193 diff --git a/src/lpython/utils.h b/src/lpython/utils.h index daa3a71e0c..da1d6999dc 100644 --- a/src/lpython/utils.h +++ b/src/lpython/utils.h @@ -4,13 +4,31 @@ #include #include +#if (defined (__linux__)) or (defined (__APPLE__)) +#include +#elif (defined (WIN32)) +#include +#endif + namespace LCompilers::LPython { +union DynamicLibrary { +#if (defined (__linux__)) or (defined (__APPLE__)) + void *l; +#elif (defined (WIN32)) + HINSTANCE l; +#endif +}; + void get_executable_path(std::string &executable_path, int &dirname_length); std::string get_runtime_library_dir(); std::string get_runtime_library_header_dir(); bool is_directory(std::string path); bool path_exists(std::string path); +void open_cpython_library(DynamicLibrary &l); +void close_cpython_library(DynamicLibrary &l); +void open_symengine_library(DynamicLibrary &l); +void close_symengine_library(DynamicLibrary &l); // Decodes the exit status code of the process (in Unix) int32_t get_exit_status(int32_t err); From cbac6708028a538790a673b45f2fb2509441cbb7 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 4 Apr 2024 11:37:37 +0530 Subject: [PATCH 16/22] wasm related fix --- src/lpython/utils.cpp | 4 ++++ src/lpython/utils.h | 15 ++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lpython/utils.cpp b/src/lpython/utils.cpp index 79175710a4..f7a3d18ba1 100644 --- a/src/lpython/utils.cpp +++ b/src/lpython/utils.cpp @@ -126,6 +126,8 @@ bool path_exists(std::string path) { } } +#ifdef HAVE_LFORTRAN_LLVM + void open_cpython_library(DynamicLibrary &l) { std::string conda_prefix = std::getenv("CONDA_PREFIX"); #if defined (__linux__) @@ -174,6 +176,8 @@ void close_symengine_library(DynamicLibrary &l) { #endif } +#endif + // Decodes the exit status code of the process (in Unix) // See `WEXITSTATUS` for more information. // https://stackoverflow.com/a/27117435/15913193 diff --git a/src/lpython/utils.h b/src/lpython/utils.h index da1d6999dc..3498643a90 100644 --- a/src/lpython/utils.h +++ b/src/lpython/utils.h @@ -4,14 +4,23 @@ #include #include +#ifdef HAVE_LFORTRAN_LLVM #if (defined (__linux__)) or (defined (__APPLE__)) #include #elif (defined (WIN32)) #include #endif +#endif namespace LCompilers::LPython { +void get_executable_path(std::string &executable_path, int &dirname_length); +std::string get_runtime_library_dir(); +std::string get_runtime_library_header_dir(); +bool is_directory(std::string path); +bool path_exists(std::string path); + +#ifdef HAVE_LFORTRAN_LLVM union DynamicLibrary { #if (defined (__linux__)) or (defined (__APPLE__)) void *l; @@ -20,15 +29,11 @@ union DynamicLibrary { #endif }; -void get_executable_path(std::string &executable_path, int &dirname_length); -std::string get_runtime_library_dir(); -std::string get_runtime_library_header_dir(); -bool is_directory(std::string path); -bool path_exists(std::string path); void open_cpython_library(DynamicLibrary &l); void close_cpython_library(DynamicLibrary &l); void open_symengine_library(DynamicLibrary &l); void close_symengine_library(DynamicLibrary &l); +#endif // Decodes the exit status code of the process (in Unix) int32_t get_exit_status(int32_t err); From a35dc027fb1cf0589ff1416e39ce3272dc969f96 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 4 Apr 2024 12:19:01 +0530 Subject: [PATCH 17/22] windows ci fix --- src/lpython/utils.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lpython/utils.h b/src/lpython/utils.h index 3498643a90..3b8e97faf7 100644 --- a/src/lpython/utils.h +++ b/src/lpython/utils.h @@ -8,6 +8,9 @@ #if (defined (__linux__)) or (defined (__APPLE__)) #include #elif (defined (WIN32)) +#ifndef NOMINMAX +#define NOMINMAX +#endif // NOMINMAX #include #endif #endif From f6f62b9765e73c13700c73fee67e986e50a49a96 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 4 Apr 2024 12:42:17 +0530 Subject: [PATCH 18/22] windows ci fix --- src/lpython/utils.cpp | 2 ++ src/lpython/utils.h | 19 +++---------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/lpython/utils.cpp b/src/lpython/utils.cpp index f7a3d18ba1..319a5e05bc 100644 --- a/src/lpython/utils.cpp +++ b/src/lpython/utils.cpp @@ -3,6 +3,8 @@ #define NOMINMAX #endif // NOMINMAX #include +#else +#include #endif #include diff --git a/src/lpython/utils.h b/src/lpython/utils.h index 3b8e97faf7..0cef8e1131 100644 --- a/src/lpython/utils.h +++ b/src/lpython/utils.h @@ -4,17 +4,6 @@ #include #include -#ifdef HAVE_LFORTRAN_LLVM -#if (defined (__linux__)) or (defined (__APPLE__)) -#include -#elif (defined (WIN32)) -#ifndef NOMINMAX -#define NOMINMAX -#endif // NOMINMAX -#include -#endif -#endif - namespace LCompilers::LPython { void get_executable_path(std::string &executable_path, int &dirname_length); @@ -24,12 +13,10 @@ bool is_directory(std::string path); bool path_exists(std::string path); #ifdef HAVE_LFORTRAN_LLVM -union DynamicLibrary { -#if (defined (__linux__)) or (defined (__APPLE__)) +struct DynamicLibrary { void *l; -#elif (defined (WIN32)) - HINSTANCE l; -#endif + + DynamicLibrary(): l(nullptr) {} }; void open_cpython_library(DynamicLibrary &l); From ff5ac202c94ca9e0000260f804c4317d15173a2c Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Thu, 4 Apr 2024 13:01:51 +0530 Subject: [PATCH 19/22] windows ci fix --- src/lpython/utils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lpython/utils.cpp b/src/lpython/utils.cpp index 319a5e05bc..0dc15e71d4 100644 --- a/src/lpython/utils.cpp +++ b/src/lpython/utils.cpp @@ -149,7 +149,7 @@ void close_cpython_library(DynamicLibrary &l) { dlclose(l.l); l.l = nullptr; #else - FreeLibrary(l.l); + FreeLibrary((HMODULE)l.l); l.l = nullptr; #endif } @@ -173,7 +173,7 @@ void close_symengine_library(DynamicLibrary &l) { dlclose(l.l); l.l = nullptr; #else - FreeLibrary(l.l); + FreeLibrary((HMODULE)l.l); l.l = nullptr; #endif } From ea044d5ffde9a398384282a46b9f0769534dc48f Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Tue, 16 Apr 2024 21:13:53 +0530 Subject: [PATCH 20/22] updated for consistency Co-authored-by: Shaikh Ubaid --- src/bin/lpython.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 68d4ac177a..1bf7731d82 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -891,10 +891,12 @@ int compile_python_using_llvm( LCompilers::LPython::DynamicLibrary cpython_lib; LCompilers::LPython::DynamicLibrary symengine_lib; - if (compiler_options.enable_cpython) + if (compiler_options.enable_cpython) { LCompilers::LPython::open_cpython_library(cpython_lib); - if (compiler_options.enable_symengine) + } + if (compiler_options.enable_symengine) { LCompilers::LPython::open_symengine_library(symengine_lib); + } auto llvm_start = std::chrono::high_resolution_clock::now(); From 5ae201ec58d7b74baab94223adb5908a9b9643b5 Mon Sep 17 00:00:00 2001 From: Vipul-Cariappa Date: Wed, 17 Apr 2024 08:07:35 +0530 Subject: [PATCH 21/22] updated for consistency --- src/bin/lpython.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 1bf7731d82..44a6a0de01 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -902,21 +902,27 @@ int compile_python_using_llvm( bool call_init = false; bool call_stmts = false; - if (m->get_return_type("__module___main_____main__global_init") == "void") + if (m->get_return_type("__module___main_____main__global_init") == "void") { call_init = true; - if (m->get_return_type("__module___main_____main__global_stmts") == "void") + } + if (m->get_return_type("__module___main_____main__global_stmts") == "void") { call_stmts = true; + } e.add_module(std::move(m)); - if (call_init) + if (call_init) { e.voidfn("__module___main_____main__global_init"); - if (call_stmts) + } + if (call_stmts) { e.voidfn("__module___main_____main__global_stmts"); + } - if (compiler_options.enable_cpython) + if (compiler_options.enable_cpython) { LCompilers::LPython::close_cpython_library(cpython_lib); - if (compiler_options.enable_symengine) + } + if (compiler_options.enable_symengine) { LCompilers::LPython::close_symengine_library(symengine_lib); + } auto llvm_end = std::chrono::high_resolution_clock::now(); times.push_back(std::make_pair("LLVM JIT execution", std::chrono::duration(llvm_end - llvm_start).count())); From 6d9432bb8309ef3f28c976708930302a8c6c860a Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Wed, 17 Apr 2024 17:15:09 +0530 Subject: [PATCH 22/22] Apply suggestions from code review Co-authored-by: Shaikh Ubaid --- src/bin/lpython.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 44a6a0de01..83ea8f6ca0 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -1961,9 +1961,10 @@ int main(int argc, char *argv[]) int err = 0; if (to_jit) { #ifdef HAVE_LFORTRAN_LLVM - if (backend != Backend::llvm) + if (backend != Backend::llvm) { std::cerr << "JIT option is only available with LLVM backend" << std::endl; - + return 1; + } compiler_options.emit_debug_info = false; compiler_options.emit_debug_line_column = false; compiler_options.generate_object_code = false; @@ -1972,6 +1973,7 @@ int main(int argc, char *argv[]) #else std::cerr << "Just-In-Time Compilation of Python files requires the LLVM backend to be enabled." " Recompile with `WITH_LLVM=yes`." << std::endl; + return 1; #endif } if (backend == Backend::x86) {