From ef645e533f684d85758693e82617e1234a49fe95 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Mon, 8 Jul 2024 16:13:56 +0530 Subject: [PATCH 1/3] Squashed commit of the following: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit b1700b23047905881e98f91817b877e548e4f77c Author: Vipul Cariappa Date: Mon Jul 8 15:50:28 2024 +0530 fix commit 127955535f27aed1b9d04bc4fa57d9b0e6497e39 Author: Vipul Cariappa Date: Mon Jul 8 15:38:58 2024 +0530 in previous commit I tested by changing compiler to gcc, still fails in macOS commit 5920f1a891f64bd9030737d2804103f12246899d Author: Vipul Cariappa Date: Mon Jul 8 15:28:13 2024 +0530 . commit 81918f1d6bf626db1cf838884d6536c137d735f7 Author: Vipul Cariappa Date: Mon Jul 8 15:19:22 2024 +0530 . commit f8ed783f3c7dbef6eba899de587f8d64cad916dc Author: Vipul Cariappa Date: Mon Jul 8 15:12:23 2024 +0530 . commit 8f709d1d26c74c4d69bd5e9f26951819e217c3f8 Author: Vipul Cariappa Date: Mon Jul 8 15:05:16 2024 +0530 . commit 69b50d1c7087baa961241777cb29819fafb00568 Author: Vipul Cariappa Date: Mon Jul 8 15:01:03 2024 +0530 . commit 797f644b9307de9b72c6eb7b7eae71a3028e8d7c Author: Vipul Cariappa Date: Mon Jul 8 07:45:37 2024 +0530 fix . commit 7e1367349438e4599133649675127cd975f28dc7 Author: Vipul Cariappa Date: Mon Jul 8 07:37:19 2024 +0530 fix . commit e18a15c7ba441f8022ed32432825c197ff279130 Author: Vipul Cariappa Date: Mon Jul 8 07:34:19 2024 +0530 CI fix commit de257f59147f71302dd2eb7c7479768bbde2b2c4 Merge: 9ec6d6de4 f414b0f05 Author: Ondřej Čertík Date: Sat Jul 6 15:03:16 2024 -0600 Merge branch 'main' into ci_mamba commit 9ec6d6de420cb46c314cf0f40e6a17d63bde8fd9 Author: Ondřej Čertík Date: Sat Jul 6 14:58:15 2024 -0600 X commit e69e069b208b732e57aeb6da0ef70b8651b9e217 Author: Ondřej Čertík Date: Sat Jul 6 14:44:51 2024 -0600 Pin zstd, zlib commit 2ed3ac2dec1b87be26d1f0996cc15fd13484be81 Author: Ondřej Čertík Date: Sat Jul 6 14:36:39 2024 -0600 downgrade commit c8a80c3ec97bad027c539928dd0ddfb3ca33e076 Author: Ondřej Čertík Date: Sat Jul 6 12:16:48 2024 -0600 zstd commit 1c77ac52adbb5603c3e3ad4c876528d302c1de96 Author: Ondřej Čertík Date: Sat Jul 6 12:14:14 2024 -0600 X commit ba8a13ca3ffb5b04d5114f8fba2a38ba3e4471d4 Author: Ondřej Čertík Date: Sat Jul 6 12:12:20 2024 -0600 Fix nlohmann_json commit 81beb7c9e372a2b53d0a87500df6f8837b68d3e0 Author: Ondřej Čertík Date: Sat Jul 6 12:10:20 2024 -0600 Pin packages commit e4fa09a6329cff6430cace0b016f49d7fc71aaa0 Author: Ondřej Čertík Date: Sat Jul 6 11:58:09 2024 -0600 List commit ae166560354fb3b5e395fc471e9422e644f6a441 Author: Ondřej Čertík Date: Sat Jul 6 11:54:47 2024 -0600 Do not upgrade installed packages commit e73253e4e4b17f0c3d874b3b17cc1876a2aa3bc2 Author: Ondřej Čertík Date: Sat Jul 6 11:50:47 2024 -0600 Pin the packages commit d1dc5f0c9102ab882e0df59fd75b19e0859b27ad Author: Ondřej Čertík Date: Sat Jul 6 11:47:13 2024 -0600 Use latest version commit 0c3b95b491a87507eafbe33947a49fb55479a2ab Author: Ondřej Čertík Date: Sat Jul 6 11:39:46 2024 -0600 Fix Windows test commit 34613d7c60e251bce349fb3f52c145f2af929d88 Author: Ondřej Čertík Date: Sat Jul 6 11:24:17 2024 -0600 Use xonsh 0.16.0 commit d1e230a7f27883564f9e5544120612e84f5685a0 Author: Ondřej Čertík Date: Sat Jul 6 11:22:01 2024 -0600 Use the old xeus commit eed485e2c807f610afbc1ebd75d2f1b4a6ab9b51 Author: Ondřej Čertík Date: Sat Jul 6 11:19:56 2024 -0600 Use lp commit e6574c4a9db8a0e4bee5e14a1cc52c865f56b096 Author: Ondřej Čertík Date: Sat Jul 6 11:14:37 2024 -0600 CI: switch to micromamba, update xeus --- .github/workflows/CI.yml | 29 ++++++++++++++--------------- ci/environment.yml | 23 +++++++++++------------ integration_tests/CMakeLists.txt | 2 +- src/lpython/python_evaluator.cpp | 8 ++++---- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ef534c5c6f..2532d4e350 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -35,29 +35,28 @@ jobs: key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/environment.yml') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: mamba-org/setup-micromamba@v1.9.0 with: - miniconda-version: "latest" - auto-update-conda: true environment-file: ci/environment.yml - python-version: ${{ matrix.python-version }} - use-only-tar-bz2: true + create-args: >- + python=${{ matrix.python-version }} + cmake=3.21.1 - name: Install Windows Conda Packages if: contains(matrix.os, 'windows') shell: bash -e -l {0} - run: conda install m2-bison=3.0.4 cmake=3.21.1 + run: micromamba install --freeze-installed m2-bison=3.0.4 m2-filesystem - name: Install Linux / macOS Conda Packages if: contains(matrix.os, 'ubuntu') || contains(matrix.os, 'macos') shell: bash -e -l {0} - run: conda install bison=3.4 nodejs=18 + run: micromamba install --freeze-installed bison=3.4 nodejs=18 - name: Conda info shell: bash -e -l {0} run: | - conda info - conda list + micromamba info + micromamba list - name: Setup Platform (Linux) if: contains(matrix.os, 'ubuntu') @@ -87,9 +86,9 @@ jobs: if: contains(matrix.os, 'windows') shell: cmd run: | - set CONDA_INSTALL_LOCN=C:\\Miniconda3 - call %CONDA_INSTALL_LOCN%\Scripts\activate.bat - call conda activate test + set MAMBA_INSTALL_LOCN=C:\\Users\runneradmin\micromamba + call %MAMBA_INSTALL_LOCN%\Scripts\activate.bat + call micromamba activate lp set LFORTRAN_CMAKE_GENERATOR=Ninja set WIN=1 set MACOS=0 @@ -107,9 +106,9 @@ jobs: if: contains(matrix.os, 'windows') shell: cmd run: | - set CONDA_INSTALL_LOCN=C:\\Miniconda3 - call %CONDA_INSTALL_LOCN%\Scripts\activate.bat - call conda activate test + set MAMBA_INSTALL_LOCN=C:\\Users\runneradmin\micromamba + call %MAMBA_INSTALL_LOCN%\Scripts\activate.bat + call micromamba activate lp set LFORTRAN_CMAKE_GENERATOR=Ninja set WIN=1 set MACOS=0 diff --git a/ci/environment.yml b/ci/environment.yml index 4fcd733601..05e00cbfa9 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -4,20 +4,19 @@ channels: - defaults dependencies: - llvmdev=11.1.0 - - toml=0.10.2 - - pytest=7.2.0 - - jupyter=1.0.0 - - xeus=1.0.1 - - xtl=0.7.4 - - nlohmann_json=3.9.1 - - cppzmq=4.7.1 - - jupyter_kernel_test=0.4.4 + - xeus=5.1.0 + - xeus-zmq=3.0.0 - xonsh=0.13.3 - - re2c=2.2 - - numpy=1.23.4 + - rapidjson + - nlohmann_json + - toml + - pytest + - jupyter + - jupyter_kernel_test + - re2c + - numpy - zlib - zstd - - ninja=1.11.0 - - rapidjson=1.1.0 + - ninja # - bison=3.4 [not win] # - m2-bison=3.4 [win] diff --git a/integration_tests/CMakeLists.txt b/integration_tests/CMakeLists.txt index f6396d0be6..caad1f5c96 100644 --- a/integration_tests/CMakeLists.txt +++ b/integration_tests/CMakeLists.txt @@ -773,7 +773,7 @@ RUN(NAME test_logical_compare LABELS cpython llvm llvm_jit) # TODO: Ad RUN(NAME test_logical_assignment LABELS cpython llvm llvm_jit) # TODO: Add C backend after fixing issue #2708 RUN(NAME vec_01 LABELS cpython llvm llvm_jit c NOFAST) RUN(NAME test_str_comparison LABELS cpython llvm llvm_jit c wasm) -RUN(NAME test_bit_length LABELS cpython llvm llvm_jit c) +RUN(NAME test_bit_length LABELS cpython c) # FIXME: This test fails on llvm & llvm_jit RUN(NAME str_to_list_cast LABELS cpython llvm llvm_jit c) RUN(NAME cast_01 LABELS cpython llvm llvm_jit c) RUN(NAME cast_02 LABELS cpython llvm llvm_jit c) diff --git a/src/lpython/python_evaluator.cpp b/src/lpython/python_evaluator.cpp index 836ddaad22..b8bc2ee3ed 100644 --- a/src/lpython/python_evaluator.cpp +++ b/src/lpython/python_evaluator.cpp @@ -143,11 +143,11 @@ Result PythonCompiler::evaluate( ->m_symtab->get_symbol(run_fn); LCOMPILERS_ASSERT(fn) if (ASRUtils::get_FunctionType(fn)->m_return_var_type->type == ASR::ttypeType::UnsignedInteger) { - uint8_t r = e->execfn(run_fn); + uint8_t r = e->execfn(run_fn); result.type = EvalResult::unsignedInteger1; result.u32 = r; } else { - int8_t r = e->execfn(run_fn); + int8_t r = e->execfn(run_fn); result.type = EvalResult::integer1; result.i32 = r; } @@ -156,11 +156,11 @@ Result PythonCompiler::evaluate( ->m_symtab->get_symbol(run_fn); LCOMPILERS_ASSERT(fn) if (ASRUtils::get_FunctionType(fn)->m_return_var_type->type == ASR::ttypeType::UnsignedInteger) { - uint16_t r = e->execfn(run_fn); + uint16_t r = e->execfn(run_fn); result.type = EvalResult::unsignedInteger2; result.u32 = r; } else { - int16_t r = e->execfn(run_fn); + int16_t r = e->execfn(run_fn); result.type = EvalResult::integer2; result.i32 = r; } From 0856ac8799805b8387873aa31a7509a37a768974 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:08:03 +0530 Subject: [PATCH 2/3] Implement Jupyter kernel --- .gitignore | 1 + CMakeLists.txt | 7 +- doc/src/developers_example.ipynb | 67 +++ examples/example_notebook.ipynb | 134 +++++ share/jupyter/kernels/lpython/kernel.json.in | 10 + src/bin/lpython.cpp | 9 +- src/lpython/CMakeLists.txt | 4 +- src/lpython/python_evaluator.cpp | 132 ++++- src/lpython/python_evaluator.h | 28 +- src/lpython/python_kernel.cpp | 540 +++++++++++++++++++ src/lpython/python_kernel.h | 15 + 11 files changed, 937 insertions(+), 10 deletions(-) create mode 100644 doc/src/developers_example.ipynb create mode 100644 examples/example_notebook.ipynb create mode 100644 share/jupyter/kernels/lpython/kernel.json.in create mode 100644 src/lpython/python_kernel.cpp create mode 100644 src/lpython/python_kernel.h diff --git a/.gitignore b/.gitignore index 9cdcc189be..d18b9b6284 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ src/libasr/wasm_visitor.h src/libasr/pass/intrinsic_function_registry_util.h src/libasr/config.h share/jupyter/kernels/fortran/kernel.json +share/jupyter/kernels/lpython/kernel.json src/runtime/*.o.empty.c python_ast.py python_ast.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1e2ea0b9..8919bbee1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,13 +208,14 @@ endif() # XEUS (Fortran kernel) set(WITH_XEUS no CACHE BOOL "Build with XEUS support") if (WITH_XEUS) - find_package(xeus 0.24.1 REQUIRED) + find_package(xeus 5.1.0 REQUIRED) + find_package(xeus-zmq 3.0.0 REQUIRED) set(HAVE_LFORTRAN_XEUS yes) # Generate kernel.json with correct paths configure_file ( - "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/fortran/kernel.json.in" - "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/fortran/kernel.json" + "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/lpython/kernel.json.in" + "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/lpython/kernel.json" ) # Configuration and data directories for Jupyter and LFortran diff --git a/doc/src/developers_example.ipynb b/doc/src/developers_example.ipynb new file mode 100644 index 0000000000..c63754c8f0 --- /dev/null +++ b/doc/src/developers_example.ipynb @@ -0,0 +1,67 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c86338ac-53ca-4115-8c5a-8bf8a5c7113e", + "metadata": {}, + "outputs": [], + "source": [ + "%%showast\n", + "def add(x: i32, y: i32) -> i32:\n", + " return x + y" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23834b08-2f3f-45e7-a1ce-21a9fd4e5117", + "metadata": {}, + "outputs": [], + "source": [ + "%%showasr\n", + "def add(x: i32, y: i32) -> i32:\n", + " return x + y" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec7426b4-e2e5-416c-bcae-9bb9c8926c9b", + "metadata": {}, + "outputs": [], + "source": [ + "%%showllvm\n", + "def sub(x: i32, y: i32) -> i32:\n", + " return add(x, -y)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "716c56ef-8210-4daf-aa23-96b385801014", + "metadata": {}, + "outputs": [], + "source": [ + "%%showasm\n", + "def mul(x: i32, y: i32) -> i32:\n", + " return x * y" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LPython", + "language": "python", + "name": "lpython" + }, + "language_info": { + "file_extension": ".f90", + "mimetype": "text/x-python", + "name": "python", + "version": "2018" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/example_notebook.ipynb b/examples/example_notebook.ipynb new file mode 100644 index 0000000000..17ce1ba2db --- /dev/null +++ b/examples/example_notebook.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e87300c2-64ed-4636-8448-591f36faba29", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello, LPython\n" + ] + } + ], + "source": [ + "print(\"Hello, LPython\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "dfcac851-7b49-4065-8c64-4a31658249f7", + "metadata": {}, + "outputs": [], + "source": [ + "def add(x: i32, y: i32) -> i32:\n", + " return x + y" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "09213386-84d5-4e7c-83ba-c3b027f765dd", + "metadata": {}, + "outputs": [], + "source": [ + "def sub(x: i32, y: i32) -> i32:\n", + " return x - y" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a4b49fd3-bf17-4287-9d5e-60f14ebc9a0f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "5" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "add(2, 3)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d6f4961f-7f0c-45a6-9bf8-e549e97098b0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-1" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sub(2, 3)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "398fd4be-d7cc-4912-8aa1-880aa58b37ab", + "metadata": {}, + "outputs": [], + "source": [ + "@dataclass\n", + "class MyClass:\n", + " x: i32\n", + " y: f64\n", + " z: str" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "628f0b7d-09a6-49de-a0e6-2f6c664f2ba2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12 2.45000000000000000e+01 LPython\n" + ] + } + ], + "source": [ + "x: MyClass = MyClass(12, 24.5, \"LPython\")\n", + "print(x)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LPython", + "language": "python", + "name": "lpython" + }, + "language_info": { + "file_extension": ".f90", + "mimetype": "text/x-python", + "name": "python", + "version": "2018" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/share/jupyter/kernels/lpython/kernel.json.in b/share/jupyter/kernels/lpython/kernel.json.in new file mode 100644 index 0000000000..e1af020ba4 --- /dev/null +++ b/share/jupyter/kernels/lpython/kernel.json.in @@ -0,0 +1,10 @@ +{ + "display_name": "LPython", + "argv": [ + "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/lpython", + "kernel", + "-f", + "{connection_file}" + ], + "language": "python" +} diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 43649c57b5..25f7ca8119 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -2012,8 +2013,12 @@ int main(int argc, char *argv[]) // } if (kernel) { - std::cerr << "The kernel subcommand is not implemented yet for LPython." << std::endl; - return 1; +#ifdef HAVE_LFORTRAN_XEUS + return LCompilers::LPython::run_kernel(arg_kernel_f); +#else + std::cerr << "The kernel subcommand requires LFortran to be compiled with XEUS support. Recompile with `WITH_XEUS=yes`." << std::endl; + return 1; +#endif } // if (mod) { diff --git a/src/lpython/CMakeLists.txt b/src/lpython/CMakeLists.txt index b8642d43d0..545fb592e9 100644 --- a/src/lpython/CMakeLists.txt +++ b/src/lpython/CMakeLists.txt @@ -22,7 +22,7 @@ endif() if (WITH_XEUS) set(SRC ${SRC} -# fortran_kernel.cpp + python_kernel.cpp ) endif() add_library(lpython_lib ${SRC}) @@ -35,7 +35,7 @@ endif() target_include_directories(lpython_lib BEFORE PUBLIC ${lpython_SOURCE_DIR}/src) target_include_directories(lpython_lib BEFORE PUBLIC ${lpython_BINARY_DIR}/src) if (WITH_XEUS) - target_link_libraries(lpython_lib xeus) + target_link_libraries(lpython_lib xeus xeus-zmq) endif() if (WITH_BFD) target_link_libraries(lpython_lib p::bfd) diff --git a/src/lpython/python_evaluator.cpp b/src/lpython/python_evaluator.cpp index b8bc2ee3ed..3cf6b83f19 100644 --- a/src/lpython/python_evaluator.cpp +++ b/src/lpython/python_evaluator.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #ifdef HAVE_LFORTRAN_LLVM #include @@ -29,12 +30,12 @@ namespace LCompilers { PythonCompiler::PythonCompiler(CompilerOptions compiler_options) : + compiler_options{compiler_options}, al{1024*1024}, #ifdef HAVE_LFORTRAN_LLVM e{std::make_unique()}, #endif eval_count{1}, - compiler_options{compiler_options}, symbol_table{nullptr} { } @@ -234,6 +235,66 @@ Result PythonCompiler::evaluate( #endif } +Result PythonCompiler::get_ast(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics) +{ + Result ast = get_ast2(code, diagnostics); + if (ast.ok) { + if (compiler_options.po.tree) { + return LCompilers::LPython::pickle_tree_python(*ast.result, compiler_options.use_colors); + } else if (compiler_options.po.json || compiler_options.po.visualize) { + return LCompilers::LPython::pickle_json(*ast.result, lm); + } + return LCompilers::LPython::pickle_python(*ast.result, compiler_options.use_colors, + compiler_options.indent); + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return ast.error; + } +} + +Result PythonCompiler::get_asr(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics) +{ + Result asr = get_asr2(code, lm, diagnostics); + if (asr.ok) { + if (compiler_options.po.tree) { + return LCompilers::pickle_tree(*asr.result, compiler_options.use_colors); + } else if (compiler_options.po.json) { + return LCompilers::pickle_json(*asr.result, lm, compiler_options.po.no_loc, false); + } + return LCompilers::pickle(*asr.result, + compiler_options.use_colors, compiler_options.indent); + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return asr.error; + } +} + +Result PythonCompiler::get_asr2( + const std::string &code_orig, LocationManager &lm, + diag::Diagnostics &diagnostics) +{ + // Src -> AST + Result res = get_ast2(code_orig, diagnostics); + LCompilers::LPython::AST::ast_t* ast; + if (res.ok) { + ast = res.result; + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } + + // AST -> ASR + Result res2 = get_asr3(*ast, diagnostics, lm, true); + if (res2.ok) { + return res2.result; + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res2.error; + } +} + Result PythonCompiler::get_ast2( const std::string &code_orig, diag::Diagnostics &diagnostics) { @@ -272,6 +333,48 @@ Result PythonCompiler::get_asr3( return asr; } +Result PythonCompiler::get_llvm( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics + ) +{ + Result> res = get_llvm2(code, lm, pass_manager, diagnostics); + if (res.ok) { +#ifdef HAVE_LFORTRAN_LLVM + return res.result->str(); +#else + throw LCompilersException("LLVM is not enabled"); +#endif + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } +} + +Result> PythonCompiler::get_llvm2( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics) +{ + Result asr = get_asr2(code, lm, diagnostics); + if (!asr.ok) { + return asr.error; + } + Result> res = get_llvm3(*asr.result, pass_manager, + diagnostics, lm.files.back().in_filename); + if (res.ok) { +#ifdef HAVE_LFORTRAN_LLVM + std::unique_ptr m = std::move(res.result); + return m; +#else + throw LCompilersException("LLVM is not enabled"); +#endif + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } +} + + Result> PythonCompiler::get_llvm3( #ifdef HAVE_LFORTRAN_LLVM ASR::TranslationUnit_t &asr, LCompilers::PassManager& lpm, @@ -319,4 +422,31 @@ Result> PythonCompiler::get_llvm3( #endif } +Result PythonCompiler::get_asm( +#ifdef HAVE_LFORTRAN_LLVM + const std::string &code, LocationManager &lm, + LCompilers::PassManager& lpm, + diag::Diagnostics &diagnostics +#else + const std::string &/*code*/, + LocationManager &/*lm*/, + LCompilers::PassManager&/*lpm*/, + diag::Diagnostics &/*diagnostics*/ +#endif + ) +{ +#ifdef HAVE_LFORTRAN_LLVM + Result> res = get_llvm2(code, lm, lpm, diagnostics); + if (res.ok) { + return e->get_asm(*res.result->m_m); + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } +#else + throw LCompilersException("LLVM is not enabled"); +#endif +} + + } // namespace LCompilers::LPython diff --git a/src/lpython/python_evaluator.h b/src/lpython/python_evaluator.h index f5c4d538ec..50958fb840 100644 --- a/src/lpython/python_evaluator.h +++ b/src/lpython/python_evaluator.h @@ -32,6 +32,8 @@ class LLVMEvaluator; class PythonCompiler { public: + CompilerOptions compiler_options; + PythonCompiler(CompilerOptions compiler_options); ~PythonCompiler(); @@ -77,24 +79,46 @@ class PythonCompiler Result evaluate2(const std::string &code); + Result get_ast(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics); + Result get_ast2( const std::string &code_orig, diag::Diagnostics &diagnostics); - + + Result get_asr(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics); + + Result get_asr2( + const std::string &code_orig, LocationManager &lm, + diag::Diagnostics &diagnostics); + Result get_asr3( LCompilers::LPython::AST::ast_t &ast, diag::Diagnostics &diagnostics, LocationManager &lm, bool is_interactive=false); + Result get_llvm( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics); + + Result> get_llvm2( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics); + Result> get_llvm3(ASR::TranslationUnit_t &asr, LCompilers::PassManager& lpm, diag::Diagnostics &diagnostics, const std::string &infile); + Result get_asm(const std::string &code, + LocationManager &lm, + LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics); + private: Allocator al; #ifdef HAVE_LFORTRAN_LLVM std::unique_ptr e; #endif int eval_count; - CompilerOptions compiler_options; SymbolTable *symbol_table; std::string run_fn; }; diff --git a/src/lpython/python_kernel.cpp b/src/lpython/python_kernel.cpp new file mode 100644 index 0000000000..171b043af3 --- /dev/null +++ b/src/lpython/python_kernel.cpp @@ -0,0 +1,540 @@ +#include + +#include +#include + +#ifdef _WIN32 +# include +# define fileno _fileno +# define dup _dup +# define dup2 _dup2 +# define close _close +# include +#else +# include +#endif + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace nl = nlohmann; + +namespace LCompilers::LPython { + + + class RedirectStdout + { + public: + RedirectStdout(std::string &out) : _out{out} { + stdout_fileno = fileno(stdout); + std::cout << std::flush; + fflush(stdout); + saved_stdout = dup(stdout_fileno); +#ifdef _WIN32 + if (_pipe(out_pipe, 65536, O_BINARY) != 0) { +#else + if (pipe(out_pipe) != 0) { +#endif + throw LCompilersException("pipe() failed"); + } + dup2(out_pipe[1], stdout_fileno); + close(out_pipe[1]); + printf("X"); + } + + ~RedirectStdout() { + fflush(stdout); + read(out_pipe[0], buffer, MAX_LEN); + dup2(saved_stdout, stdout_fileno); + _out = std::string(&buffer[1]); + } + private: + std::string &_out; + static const size_t MAX_LEN=1024; + char buffer[MAX_LEN+1] = {0}; + int out_pipe[2]; + int saved_stdout; + int stdout_fileno; + }; + + class custom_interpreter : public xeus::xinterpreter + { + private: + PythonCompiler e; + + public: + custom_interpreter() : e{CompilerOptions()} { + e.compiler_options.interactive = true; + e.compiler_options.po.disable_main = true; + e.compiler_options.emit_debug_line_column = false; + e.compiler_options.generate_object_code = false; + } + virtual ~custom_interpreter() = default; + + private: + + void configure_impl() override; + + void execute_request_impl(send_reply_callback cb, + int execution_counter, + const std::string& code, + //bool silent, + //bool store_history, + xeus::execute_request_config config, + nl::json user_expressions) override; + + nl::json complete_request_impl(const std::string& code, + int cursor_pos) override; + + nl::json inspect_request_impl(const std::string& code, + int cursor_pos, + int detail_level) override; + + nl::json is_complete_request_impl(const std::string& code) override; + + nl::json kernel_info_request_impl() override; + + void shutdown_request_impl() override; + }; + + + void custom_interpreter::execute_request_impl(send_reply_callback cb, + int execution_counter, // Typically the cell number + const std::string& code, // Code to execute + xeus::execute_request_config, //config + nl::json /*user_expressions*/) + { + PythonCompiler::EvalResult r; + std::string std_out; + std::string code0; + CompilerOptions cu; + try { + if (startswith(code, "%%showast")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + } + diag::Diagnostics diagnostics; + Result + res = e.get_ast(code0, lm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + if (startswith(code, "%%showasr")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + } + diag::Diagnostics diagnostics; + Result + res = e.get_asr(code0, lm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + if (startswith(code, "%%showllvm")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + lm.init_simple(code0); + lm.file_ends.push_back(code0.size()); + } + LCompilers::PassManager lpm; + lpm.use_default_passes(); + diag::Diagnostics diagnostics; + Result + res = e.get_llvm(code0, lm, lpm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + if (startswith(code, "%%showasm")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + lm.init_simple(code0); + lm.file_ends.push_back(code0.size()); + } + LCompilers::PassManager lpm; + lpm.use_default_passes(); + diag::Diagnostics diagnostics; + Result + res = e.get_asm(code0, lm, lpm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + // if (startswith(code, "%%showcpp")) { + // code0 = code.substr(code.find("\n")+1); + // LocationManager lm; + // { + // LocationManager::FileLocations fl; + // fl.in_filename = "input"; + // std::ofstream out("input"); + // out << code0; + // lm.files.push_back(fl); + // } + // diag::Diagnostics diagnostics; + // Result + // res = e.get_cpp(code0, lm, diagnostics, 1); + // nl::json result; + // if (res.ok) { + // publish_stream("stdout", res.result); + // result["status"] = "ok"; + // result["payload"] = nl::json::array(); + // result["user_expressions"] = nl::json::object(); + // } else { + // std::string msg = diagnostics.render(lm, cu); + // publish_stream("stderr", msg); + // result["status"] = "error"; + // result["ename"] = "CompilerError"; + // result["evalue"] = msg; + // result["traceback"] = nl::json::array(); + // } + // cb(result); + // return; + // } + // if (startswith(code, "%%showfmt")) { + // code0 = code.substr(code.find("\n")+1); + // LocationManager lm; + // { + // LocationManager::FileLocations fl; + // fl.in_filename = "input"; + // std::ofstream out("input"); + // out << code0; + // lm.files.push_back(fl); + // } + // diag::Diagnostics diagnostics; + // Result + // res = e.get_fmt(code0, lm, diagnostics); + // nl::json result; + // if (res.ok) { + // publish_stream("stdout", res.result); + // result["status"] = "ok"; + // result["payload"] = nl::json::array(); + // result["user_expressions"] = nl::json::object(); + // } else { + // std::string msg = diagnostics.render(lm, cu); + // publish_stream("stderr", msg); + // result["status"] = "error"; + // result["ename"] = "CompilerError"; + // result["evalue"] = msg; + // result["traceback"] = nl::json::array(); + // } + // cb(result); + // return; + // } + + RedirectStdout s(std_out); + code0 = code; + LocationManager lm; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + lm.init_simple(code0); + lm.file_ends.push_back(code0.size()); + } + LCompilers::PassManager lpm; + lpm.use_default_passes(); + diag::Diagnostics diagnostics; + Result + res = e.evaluate(code0, false, lm, lpm, diagnostics); + if (res.ok) { + r = res.result; + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + nl::json result; + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + cb(result); + return; + } + } catch (const LCompilersException &e) { + publish_stream("stderr", "LFortran Exception: " + e.msg()); + nl::json result; + result["status"] = "error"; + result["ename"] = "LCompilersException"; + result["evalue"] = e.msg(); + result["traceback"] = nl::json::array(); + cb(result); + return; + } + + if (std_out.size() > 0) { + publish_stream("stdout", std_out); + } + + switch (r.type) { + case (LCompilers::PythonCompiler::EvalResult::integer4) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.i32); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::integer8) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.i64); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::real4) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.f32); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::real8) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.f64); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::complex4) : { + nl::json pub_data; + pub_data["text/plain"] = "(" + std::to_string(r.c32.re) + ", " + std::to_string(r.c32.im) + ")"; + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::complex8) : { + nl::json pub_data; + pub_data["text/plain"] = "(" + std::to_string(r.c64.re) + ", " + std::to_string(r.c64.im) + ")"; + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::statement) : { + break; + } + case (LCompilers::PythonCompiler::EvalResult::none) : { + break; + } + default : throw LCompilersException("Return type not supported"); + } + + nl::json result; + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + cb(result); + return; + } + + void custom_interpreter::configure_impl() + { + // Perform some operations + } + + nl::json custom_interpreter::complete_request_impl(const std::string& code, + int cursor_pos) + { + nl::json result; + + // Code starts with 'H', it could be the following completion + if (code[0] == 'H') + { + result["status"] = "ok"; + result["matches"] = {"Hello", "Hey", "Howdy"}; + result["cursor_start"] = 5; + result["cursor_end"] = cursor_pos; + } + // No completion result + else + { + result["status"] = "ok"; + result["matches"] = nl::json::array(); + result["cursor_start"] = cursor_pos; + result["cursor_end"] = cursor_pos; + } + + return result; + } + + nl::json custom_interpreter::inspect_request_impl(const std::string& code, + int /*cursor_pos*/, + int /*detail_level*/) + { + nl::json result; + + if (code.compare("print") == 0) + { + result["found"] = true; + result["text/plain"] = "Print objects to the text stream file, [...]"; + } + else + { + result["found"] = false; + } + + result["status"] = "ok"; + return result; + } + + nl::json custom_interpreter::is_complete_request_impl(const std::string& /*code*/) + { + nl::json result; + + // if (is_complete(code)) + // { + result["status"] = "complete"; + // } + // else + // { + // result["status"] = "incomplete"; + // result["indent"] = 4; + //} + + return result; + } + + nl::json custom_interpreter::kernel_info_request_impl() + { + nl::json result; + std::string version = LFORTRAN_VERSION; + std::string banner = "" + "LFortran " + version + "\n" + "Jupyter kernel for Fortran"; + result["banner"] = banner; + result["implementation"] = "LFortran"; + result["implementation_version"] = version; + result["language_info"]["name"] = "python"; + result["language_info"]["version"] = "2018"; + result["language_info"]["mimetype"] = "text/x-python"; + result["language_info"]["file_extension"] = ".f90"; + return result; + } + + void custom_interpreter::shutdown_request_impl() { + std::cout << "Bye!!" << std::endl; + } + + int run_kernel(const std::string &connection_filename) + { + std::unique_ptr context = xeus::make_zmq_context(); + + // Create interpreter instance + using interpreter_ptr = std::unique_ptr; + interpreter_ptr interpreter = interpreter_ptr(new custom_interpreter()); + + using history_manager_ptr = std::unique_ptr; + history_manager_ptr hist = xeus::make_in_memory_history_manager(); + + nl::json debugger_config; + + // Load configuration file + xeus::xconfiguration config = xeus::load_configuration(connection_filename); + + // Create kernel instance and start it + xeus::xkernel kernel(config, + xeus::get_user_name(), + std::move(context), + std::move(interpreter), + xeus::make_xserver_shell_main, + std::move(hist), + xeus::make_console_logger(xeus::xlogger::msg_type, + xeus::make_file_logger(xeus::xlogger::content, "xeus.log")), + xeus::make_null_debugger, + debugger_config); + + std::cout << + "Starting xeus-lpython kernel...\n\n" + "If you want to connect to this kernel from an other client, you can use" + " the " + connection_filename + " file." + << std::endl; + + kernel.start(); + + return 0; + } + +} // namespace LCompilers::LFortran diff --git a/src/lpython/python_kernel.h b/src/lpython/python_kernel.h new file mode 100644 index 0000000000..bf0a5c5173 --- /dev/null +++ b/src/lpython/python_kernel.h @@ -0,0 +1,15 @@ +#ifndef LFORTRAN_PYTHON_KERNEL_H +#define LFORTRAN_PYTHON_KERNEL_H + +#include +#include + +namespace LCompilers::LPython { + +#ifdef HAVE_LFORTRAN_XEUS + int run_kernel(const std::string &connection_filename); +#endif + +} // namespace LCompilers::LFortran + +#endif // LFORTRAN_PYTHON_KERNEL_H From b48e4e963637c9c50f57279f96bc95e8843c90ad Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:15:50 +0530 Subject: [PATCH 3/3] Test Kernel; presently it only builds, we should add tests later --- .github/workflows/CI.yml | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2532d4e350..cfaed873ce 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -496,6 +496,51 @@ jobs: cd integration_tests ./run_tests.py -b cpython c_py + build_jupyter_kernel: + name: Build Jupyter Kernel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ci/environment.yml + create-args: >- + jupyter + nlohmann_json + python=3.10 + bison=3.4 + xeus=5.1.0 + xeus-zmq=3.0.0 + + - uses: hendrikmuhs/ccache-action@main + with: + variant: sccache + key: ${{ github.job }}-${{ matrix.os }} + + - name: Build LPython with Kernel + shell: bash -e -l {0} + run: | + ./build0.sh + export CXXFLAGS="-Werror" + cmake . -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_LLVM=yes \ + -DWITH_XEUS=yes \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" \ + -DCMAKE_INSTALL_PREFIX="$CONDA_PREFIX" + + ninja install + ctest --output-on-failure + jupyter kernelspec list --json + + - name: Test Kernel + shell: bash -e -l {0} + run: | + ctest --output-on-failure + upload_tarball: name: Upload Tarball runs-on: ubuntu-latest