Merge branch 'main' into input

.github/workflows/CI.yml

@@ -430,8 +430,8 @@ jobs:
         shell: bash -e -l {0}
         run: |
             cd integration_tests
-            ./run_tests.py -b c_sym cpython_sym llvm_sym
-            ./run_tests.py -b c_sym cpython_sym llvm_sym -f
+            ./run_tests.py -b c_sym cpython_sym llvm_sym llvm_jit
+            ./run_tests.py -b c_sym cpython_sym llvm_sym llvm_jit -f
 
   integration_tests_cpython:
     name: Run Integration tests with Python ${{ matrix.python-version }}

build_to_wasm.sh

@@ -9,8 +9,9 @@ cp -r src/runtime/lpython src/bin/asset_dir
 
 ./build0.sh
 emcmake cmake \
-    -DCMAKE_BUILD_TYPE=Debug \
+    -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_CXX_FLAGS_DEBUG="-Wall -Wextra -fexceptions" \
+    -DCMAKE_CXX_FLAGS_RELEASE="-Wall -Wextra -fexceptions" \
     -DWITH_LLVM=no \
     -DLPYTHON_BUILD_ALL=yes \
     -DLPYTHON_BUILD_TO_WASM=yes \

integration_tests/run_tests.py

@@ -6,7 +6,7 @@
 
 # Initialization
 DEFAULT_THREADS_TO_USE = 8 # default no of threads is 8
-SUPPORTED_BACKENDS = ['llvm', 'c', 'wasm', 'cpython', 'x86', 'wasm_x86', 'wasm_x64', 'c_py', 'c_sym', 'cpython_sym', 'llvm_sym', 'llvm_py']
+SUPPORTED_BACKENDS = ['llvm', 'c', 'wasm', 'cpython', 'x86', 'wasm_x86', 'wasm_x64', 'c_py', 'c_sym', 'cpython_sym', 'llvm_sym', 'llvm_py', 'llvm_jit']
 BASE_DIR = os.path.dirname(os.path.realpath(__file__))
 LPYTHON_PATH = f"{BASE_DIR}/../src/bin"
 
@@ -62,7 +62,7 @@ def main():
     DEFAULT_THREADS_TO_USE = args.no_of_threads or DEFAULT_THREADS_TO_USE
     fast_tests = "yes" if args.fast else "no"
     for backend in args.backends:
-        python_libs_req = "yes" if backend in ["cpython", "c_py", "c_sym", "llvm_sym", 'llvm_py'] else "no"
+        python_libs_req = "yes" if backend in ["cpython", "c_py", "c_sym", "llvm_sym", 'llvm_py', 'llvm_jit'] else "no"
         test_backend(backend)
 
 

integration_tests/test_list_11.py

@@ -1,5 +1,12 @@
 from lpython import i32
 
+l: list[i32] = [1, 2]
+
+def add_item(i: i32) -> list[i32]:
+    l.append(i)
+    return l
+
+
 def return_empty_list_of_tuples() -> list[i32]:
     return []
 
@@ -19,12 +26,21 @@ def test_iterate_over_string():
         assert s == temp[i]
         i+=1
 
+def test_issue_2639():
+    print(add_item(3))
+
+    assert len(l) == 3
+    assert l[0] == 1
+    assert l[1] == 2
+    assert l[2] == 3
+
 def main0():
     x: list[i32] = return_empty_list_of_tuples()
     print(len(x))
 
     assert len(x) == 0
     test_issue_1882()
     test_iterate_over_string()
+    test_issue_2639()
 
 main0()

src/bin/lpython.cpp

@@ -792,13 +792,17 @@ int emit_llvm(const std::string &infile,
     return 0;
 }
 
-int compile_python_to_object_file(
+/*
+    Compiles python to object file, if `to_jit` is false
+    otherwise execute python code using llvm JIT
+*/
+int compile_python_using_llvm(
         const std::string &infile,
         const std::string &outfile,
         const std::string &runtime_library_dir,
         LCompilers::PassManager& pass_manager,
         CompilerOptions &compiler_options,
-        bool time_report, bool arg_c=false)
+        bool time_report, bool arg_c=false, bool to_jit=false)
 {
     Allocator al(4*1024);
     LCompilers::diag::Diagnostics diagnostics;
@@ -869,7 +873,6 @@ int compile_python_to_object_file(
     }
     LCompilers::PythonCompiler fe(compiler_options);
     LCompilers::LLVMEvaluator e(compiler_options.target);
-    std::unique_ptr<LCompilers::LLVMModule> m;
     auto asr_to_llvm_start = std::chrono::high_resolution_clock::now();
     LCompilers::Result<std::unique_ptr<LCompilers::LLVMModule>>
         res = fe.get_llvm3(*asr, pass_manager, diagnostics, infile);
@@ -882,12 +885,55 @@ int compile_python_to_object_file(
         print_time_report(times, time_report);
         return 3;
     }
-    m = std::move(res.result);
-    auto llvm_start = std::chrono::high_resolution_clock::now();
-    e.save_object_file(*(m->m_m), outfile);
-    auto llvm_end = std::chrono::high_resolution_clock::now();
-    times.push_back(std::make_pair("LLVM to binary", std::chrono::duration<double, std::milli>(llvm_end - llvm_start).count()));
-    print_time_report(times, time_report);
+    std::unique_ptr<LCompilers::LLVMModule> m = std::move(res.result);
+
+    if (to_jit) {
+        LCompilers::LPython::DynamicLibrary cpython_lib;
+        LCompilers::LPython::DynamicLibrary symengine_lib;
+
+        if (compiler_options.enable_cpython) {
+            LCompilers::LPython::open_cpython_library(cpython_lib);
+        }
+        if (compiler_options.enable_symengine) {
+            LCompilers::LPython::open_symengine_library(symengine_lib);
+        }
+
+        auto llvm_start = std::chrono::high_resolution_clock::now();
+
+        bool call_init = false;
+        bool call_stmts = false;
+        if (m->get_return_type("__module___main_____main__global_init") == "void") {
+            call_init = true;
+        }
+        if (m->get_return_type("__module___main_____main__global_stmts") == "void") {
+            call_stmts = true;
+        }
+
+        e.add_module(std::move(m));
+        if (call_init) {
+            e.voidfn("__module___main_____main__global_init");
+        }
+        if (call_stmts) {
+            e.voidfn("__module___main_____main__global_stmts");
+        }
+
+        if (compiler_options.enable_cpython) {
+            LCompilers::LPython::close_cpython_library(cpython_lib);
+        }
+        if (compiler_options.enable_symengine) {
+            LCompilers::LPython::close_symengine_library(symengine_lib);
+        }
+
+        auto llvm_end = std::chrono::high_resolution_clock::now();
+        times.push_back(std::make_pair("LLVM JIT execution", std::chrono::duration<double, std::milli>(llvm_end - llvm_start).count()));
+        print_time_report(times, time_report);
+    } else {
+        auto llvm_start = std::chrono::high_resolution_clock::now();
+        e.save_object_file(*(m->m_m), outfile);
+        auto llvm_end = std::chrono::high_resolution_clock::now();
+        times.push_back(std::make_pair("LLVM to binary", std::chrono::duration<double, std::milli>(llvm_end - llvm_start).count()));
+        print_time_report(times, time_report);
+    }
     return 0;
 }
 
@@ -1560,6 +1606,7 @@ int main(int argc, char *argv[])
         bool print_rtl_header_dir = false;
         bool print_rtl_dir = false;
         bool separate_compilation = false;
+        bool to_jit = false;
 
         std::string arg_fmt_file;
         // int arg_fmt_indent = 4;
@@ -1593,6 +1640,7 @@ int main(int argc, char *argv[])
         app.add_option("-I", compiler_options.import_paths, "Specify the paths"
             "to look for the module")->allow_extra_args(false);
         // app.add_option("-J", arg_J, "Where to save mod files");
+        app.add_flag("--jit", to_jit, "Execute the program using just-in-time (JIT) compiler");
         app.add_flag("-g", compiler_options.emit_debug_info, "Compile with debugging information");
         app.add_flag("--debug-with-line-column", compiler_options.emit_debug_line_column,
             "Convert the linear location info into line + column in the debugging information");
@@ -1894,10 +1942,10 @@ int main(int argc, char *argv[])
             }
         }
 
-        if (arg_c) {
+        if (arg_c && !to_jit) {
             if (backend == Backend::llvm) {
 #ifdef HAVE_LFORTRAN_LLVM
-                return compile_python_to_object_file(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report,
+                return compile_python_using_llvm(arg_file, outfile, runtime_library_dir, lpython_pass_manager, compiler_options, time_report,
                                                      arg_c);
 #else
                 std::cerr << "The -c option requires the LLVM backend to be enabled. Recompile with `WITH_LLVM=yes`." << std::endl;
@@ -1911,6 +1959,23 @@ int main(int argc, char *argv[])
         if (endswith(arg_file, ".py"))
         {
             int err = 0;
+            if (to_jit) {
+#ifdef HAVE_LFORTRAN_LLVM
+                if (backend != Backend::llvm) {
+                    std::cerr << "JIT option is only available with LLVM backend" << std::endl;
+                    return 1;
+                }
+                compiler_options.emit_debug_info = false;
+                compiler_options.emit_debug_line_column = false;
+                compiler_options.generate_object_code = false;
+                return compile_python_using_llvm(arg_file, "", runtime_library_dir,
+                        lpython_pass_manager, compiler_options, time_report, false, true);
+#else
+                std::cerr << "Just-In-Time Compilation of Python files requires the LLVM backend to be enabled."
+                             " Recompile with `WITH_LLVM=yes`." << std::endl;
+                return 1;
+#endif
+        }
             if (backend == Backend::x86) {
                 err = compile_to_binary_x86(arg_file, outfile,
                         runtime_library_dir, compiler_options, time_report);
@@ -1931,7 +1996,7 @@ int main(int argc, char *argv[])
             } else if (backend == Backend::llvm) {
 #ifdef HAVE_LFORTRAN_LLVM
                 std::string tmp_o = outfile + ".tmp.o";
-                err = compile_python_to_object_file(arg_file, tmp_o, runtime_library_dir,
+                err = compile_python_using_llvm(arg_file, tmp_o, runtime_library_dir,
                     lpython_pass_manager, compiler_options, time_report);
                 if (err != 0) return err;
                 err = link_executable({tmp_o}, outfile, runtime_library_dir,

src/libasr/codegen/llvm_utils.cpp

@@ -2120,17 +2120,15 @@ namespace LCompilers {
             throw LCompilersException("list for " + type_code + " not declared yet.");
         }
         int32_t type_size = std::get<1>(typecode2listtype[type_code]);
-        llvm::Value* arg_size = llvm::ConstantInt::get(context,
-                                    llvm::APInt(32, type_size * initial_capacity));
-
-        llvm::Value* list_data = LLVM::lfortran_malloc(context, module, *builder,
-                                                       arg_size);
+        llvm::Value* llvm_type_size = llvm::ConstantInt::get(context, llvm::APInt(32, type_size));
+        llvm::Value* current_capacity = llvm::ConstantInt::get(context, llvm::APInt(32, initial_capacity));
+        llvm::Value* list_data = LLVM::lfortran_calloc(context, module, *builder,
+                                                       current_capacity, llvm_type_size);
         llvm::Type* el_type = std::get<2>(typecode2listtype[type_code]);
         list_data = builder->CreateBitCast(list_data, el_type->getPointerTo());
         llvm::Value* list_data_ptr = get_pointer_to_list_data(list);
         builder->CreateStore(list_data, list_data_ptr);
         llvm::Value* current_end_point = llvm::ConstantInt::get(context, llvm::APInt(32, n));
-        llvm::Value* current_capacity = llvm::ConstantInt::get(context, llvm::APInt(32, initial_capacity));
         builder->CreateStore(current_end_point, get_pointer_to_current_end_point(list));
         builder->CreateStore(current_capacity, get_pointer_to_current_capacity(list));
     }
@@ -2143,8 +2141,8 @@ namespace LCompilers {
         }
         int32_t type_size = std::get<1>(typecode2listtype[type_code]);
         llvm::Value* llvm_type_size = llvm::ConstantInt::get(context, llvm::APInt(32, type_size));
-        llvm::Value* arg_size = builder->CreateMul(llvm_type_size, initial_capacity);
-        llvm::Value* list_data = LLVM::lfortran_malloc(context, module, *builder, arg_size);
+        llvm::Value* list_data = LLVM::lfortran_calloc(context, module, *builder,
+                                                        initial_capacity, llvm_type_size);
 
         llvm::Type* el_type = std::get<2>(typecode2listtype[type_code]);
         list_data = builder->CreateBitCast(list_data, el_type->getPointerTo());
@@ -2288,10 +2286,9 @@ namespace LCompilers {
         builder->CreateStore(src_capacity, dest_capacity_ptr);
         llvm::Value* src_data = LLVM::CreateLoad(*builder, get_pointer_to_list_data(src));
         int32_t type_size = std::get<1>(typecode2listtype[src_type_code]);
-        llvm::Value* arg_size = builder->CreateMul(llvm::ConstantInt::get(context,
-                                                   llvm::APInt(32, type_size)), src_capacity);
-        llvm::Value* copy_data = LLVM::lfortran_malloc(context, *module, *builder,
-                                                       arg_size);
+        llvm::Value* llvm_type_size = llvm::ConstantInt::get(context, llvm::APInt(32, type_size));
+        llvm::Value* copy_data = LLVM::lfortran_calloc(context, *module, *builder,
+                                                       src_capacity, llvm_type_size);
         llvm::Type* el_type = std::get<2>(typecode2listtype[src_type_code]);
         copy_data = builder->CreateBitCast(copy_data, el_type->getPointerTo());
 
@@ -2346,6 +2343,8 @@ namespace LCompilers {
             // end
             llvm_utils->start_new_block(loopend);
         } else {
+            llvm::Value* arg_size = builder->CreateMul(llvm::ConstantInt::get(context,
+                                                   llvm::APInt(32, type_size)), src_capacity);
             builder->CreateMemCpy(copy_data, llvm::MaybeAlign(), src_data,
                                   llvm::MaybeAlign(), arg_size);
             builder->CreateStore(copy_data, get_pointer_to_list_data(dest));

src/libasr/pass/print_list_tuple.cpp

@@ -114,11 +114,23 @@ class PrintListTupleVisitor
                 list_iter_var_name, al, current_scope, int_type);
         }
 
+        std::string list_var_name;
+        ASR::expr_t *list_var;
+        {
+            list_var_name = 
+                current_scope->get_unique_name("__list_var", false);
+            list_var = PassUtils::create_auxiliary_variable(loc,
+                list_var_name, al, current_scope, ASRUtils::expr_type(list_expr));
+        }
+
+        ASR::stmt_t *assign_stmt = ASRUtils::STMT(
+            ASR::make_Assignment_t(al, loc, list_var, list_expr, nullptr));
+
         ASR::expr_t *list_item = ASRUtils::EXPR(
-            ASR::make_ListItem_t(al, loc, list_expr,
+            ASR::make_ListItem_t(al, loc, list_var,
                                     list_iter_var, listC->m_type, nullptr));
         ASR::expr_t *list_len = ASRUtils::EXPR(ASR::make_ListLen_t(
-            al, loc, list_expr, int_type, nullptr));
+            al, loc, list_var, int_type, nullptr));
         ASR::expr_t *constant_one = ASRUtils::EXPR(
             ASR::make_IntegerConstant_t(al, loc, 1, int_type));
         ASR::expr_t *list_len_minus_one =
@@ -199,6 +211,7 @@ class PrintListTupleVisitor
             al, loc, nullptr, loop_head, loop_body.p, loop_body.size(), nullptr, 0));
 
         {
+            print_pass_result_tmp.push_back(al, assign_stmt);
             print_pass_result_tmp.push_back(al, print_open_bracket);
             print_pass_result_tmp.push_back(al, loop);
             print_pass_result_tmp.push_back(al, print_close_bracket);

src/lpython/parser/parser_stype.h

@@ -113,8 +113,8 @@ static_assert(std::is_trivial<YYSTYPE>::value);
 // YYSTYPE must be at least as big, but it should not be bigger, otherwise it
 // would reduce performance.
 // A temporary fix for PowerPC 32-bit, where the following assert fails with (16 == 12).
-#ifndef __ppc__
-static_assert(sizeof(YYSTYPE) == sizeof(Vec<LPython::AST::ast_t*>));
+#if !defined(HAVE_BUILD_TO_WASM) && !defined(__ppc__)
+static_assert(sizeof(YYSTYPE) == sizeof(Vec<AST::ast_t*>));
 #endif
 
 static_assert(std::is_standard_layout<Location>::value);

src/lpython/utils.cpp

@@ -3,6 +3,8 @@
 #define NOMINMAX
 #endif // NOMINMAX
 #include <windows.h>
+#else
+#include <dlfcn.h>
 #endif
 
 #include <fstream>
@@ -126,6 +128,58 @@ bool path_exists(std::string path) {
     }
 }
 
+#ifdef HAVE_LFORTRAN_LLVM
+
+void open_cpython_library(DynamicLibrary &l) {
+    std::string conda_prefix = std::getenv("CONDA_PREFIX");
+#if defined (__linux__)
+    l.l = dlopen((conda_prefix + "/lib/libpython3.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW);
+#elif defined (__APPLE__)
+    l.l = dlopen((conda_prefix + "/lib/libpython3.dylib").c_str(), RTLD_GLOBAL | RTLD_NOW);
+#else
+    l.l = LoadLibrary((conda_prefix + "\\python3.dll").c_str());
+#endif
+
+    if (l.l == nullptr)
+        throw "Could not open CPython library";
+}
+
+void close_cpython_library(DynamicLibrary &l) {
+#if (defined (__linux__)) or (defined (__APPLE__))
+    dlclose(l.l);
+    l.l = nullptr;
+#else
+    FreeLibrary((HMODULE)l.l);
+    l.l = nullptr;
+#endif
+}
+
+void open_symengine_library(DynamicLibrary &l) {
+    std::string conda_prefix = std::getenv("CONDA_PREFIX");
+#if defined (__linux__)
+    l.l = dlopen((conda_prefix + "/lib/libsymengine.so").c_str(), RTLD_DEEPBIND | RTLD_GLOBAL | RTLD_NOW);
+#elif defined (__APPLE__)
+    l.l = dlopen((conda_prefix + "/lib/libsymengine.dylib").c_str(), RTLD_GLOBAL | RTLD_NOW);
+#else
+    l.l = LoadLibrary((conda_prefix + "\\Library\\bin\\symengine-0.11.dll").c_str());
+#endif
+
+    if (l.l == nullptr)
+        throw "Could not open SymEngine library";
+}
+
+void close_symengine_library(DynamicLibrary &l) {
+#if (defined (__linux__)) or (defined (__APPLE__))
+    dlclose(l.l);
+    l.l = nullptr;
+#else
+    FreeLibrary((HMODULE)l.l);
+    l.l = nullptr;
+#endif
+}
+
+#endif
+
 // Decodes the exit status code of the process (in Unix)
 // See `WEXITSTATUS` for more information.
 // https://stackoverflow.com/a/27117435/15913193