feat: semantic API for multiple language SDKs (#989)

Signed-off-by: peefy <[email protected]>

kclvm/Cargo.toml

@@ -37,6 +37,7 @@ kclvm-runner = {path = "./runner"}
 kclvm-parser = {path = "./parser"}
 kclvm-compiler = {path = "./compiler"}
 kclvm-config = {path = "./config"}
+kclvm-loader = {path = "./loader"}
 kclvm-runtime = {path = "./runtime"}
 kclvm-sema = {path = "./sema"}
 kclvm-tools = {path = "./tools"}

kclvm/api/Cargo.toml

@@ -28,6 +28,7 @@ kclvm-config = {path = "../config"}
 kclvm-driver = {path = "../driver"}
 kclvm-error = {path = "../error"}
 kclvm-parser = {path = "../parser"}
+kclvm-loader = {path = "../loader"}
 kclvm-sema = {path = "../sema"}
 kclvm-ast = {path = "../ast"}
 kclvm-ast-pretty = {path = "../ast_pretty"}

kclvm/api/src/service/capi.rs

@@ -107,6 +107,7 @@ pub(crate) fn kclvm_get_service_fn_ptr_by_name(name: &str) -> u64 {
         "KclvmService.Ping" => ping as *const () as u64,
         "KclvmService.ParseFile" => parse_file as *const () as u64,
         "KclvmService.ParseProgram" => parse_program as *const () as u64,
+        "KclvmService.LoadPackage" => load_package as *const () as u64,
         "KclvmService.ExecProgram" => exec_program as *const () as u64,
         "KclvmService.OverrideFile" => override_file as *const () as u64,
         "KclvmService.GetSchemaType" => get_schema_type as *const () as u64,
@@ -170,6 +171,27 @@ pub(crate) fn parse_program(serv: *mut kclvm_service, args: *const c_char) -> *c
     call!(serv, args, ParseProgramArgs, parse_program)
 }
 
+/// load_package provides users with the ability to parse kcl program and sematic model
+/// information including symbols, types, definitions, etc,
+///
+/// # Parameters
+///
+/// `serv`: [*mut kclvm_service]
+///     The pointer of &\[[KclvmServiceImpl]]
+///
+///
+/// `args`: [*const c_char]
+///     the items and compile parameters selected by the user in the KCL CLI
+///     serialized as protobuf byte sequence
+///
+/// # Returns
+///
+/// result: [*const c_char]
+///     Result of the call serialized as protobuf byte sequence
+pub(crate) fn load_package(serv: *mut kclvm_service, args: *const c_char) -> *const c_char {
+    call!(serv, args, LoadPackageArgs, load_package)
+}
+
 /// exec_program provides users with the ability to execute KCL code
 ///
 /// # Parameters

kclvm/api/src/service/into.rs

@@ -1,6 +1,10 @@
-use crate::gpyrpc::{CliConfig, Error, KeyValuePair, LoadSettingsFilesResult, Message, Position};
+use crate::gpyrpc::{
+    CliConfig, Error, KeyValuePair, LoadSettingsFilesResult, Message, Position, Symbol, SymbolIndex,
+};
 use kclvm_config::settings::SettingsFile;
 use kclvm_error::Diagnostic;
+use kclvm_loader::SymbolInfo;
+use kclvm_sema::core::symbol::SymbolRef;
 
 pub(crate) trait IntoLoadSettingsFiles {
     /// Convert self into the LoadSettingsFiles structure.
@@ -11,6 +15,14 @@ pub(crate) trait IntoError {
     fn into_error(self) -> Error;
 }
 
+pub(crate) trait IntoSymbolIndex {
+    fn into_symbol_index(self) -> SymbolIndex;
+}
+
+pub(crate) trait IntoSymbol {
+    fn into_symbol(self) -> Symbol;
+}
+
 impl IntoLoadSettingsFiles for SettingsFile {
     fn into_load_settings_files(self, files: &[String]) -> LoadSettingsFilesResult {
         LoadSettingsFilesResult {
@@ -65,3 +77,27 @@ impl IntoError for Diagnostic {
         }
     }
 }
+
+impl IntoSymbolIndex for SymbolRef {
+    fn into_symbol_index(self) -> SymbolIndex {
+        let (index, generation) = self.get_id().into_raw_parts();
+        SymbolIndex {
+            i: index as u64,
+            g: generation as u64,
+            kind: format!("{:?}", self.get_kind()),
+        }
+    }
+}
+
+impl IntoSymbol for SymbolInfo {
+    fn into_symbol(self) -> Symbol {
+        Symbol {
+            ty: self.ty.ty_str(),
+            name: self.name,
+            owner: self.owner.map(|o| o.into_symbol_index()),
+            def: self.def.map(|d| d.into_symbol_index()),
+            attrs: self.attrs.iter().map(|a| a.into_symbol_index()).collect(),
+            is_global: self.is_global,
+        }
+    }
+}

kclvm/api/src/service/jsonrpc.rs

@@ -83,6 +83,14 @@ fn register_kclvm_service(io: &mut IoHandler) {
         };
         futures::future::ready(catch!(kclvm_service_impl, args, parse_program))
     });
+    io.add_method("KclvmService.LoadPackage", |params: Params| {
+        let kclvm_service_impl = KclvmServiceImpl::default();
+        let args: LoadPackageArgs = match params.parse() {
+            Ok(val) => val,
+            Err(err) => return futures::future::ready(Err(err)),
+        };
+        futures::future::ready(catch!(kclvm_service_impl, args, load_package))
+    });
     io.add_method("KclvmService.ExecProgram", |params: Params| {
         let kclvm_service_impl = KclvmServiceImpl::default();
         let args: ExecProgramArgs = match params.parse() {

kclvm/api/src/service/service_impl.rs

@@ -10,8 +10,10 @@ use anyhow::anyhow;
 use kcl_language_server::rename;
 use kclvm_config::settings::build_settings_pathbuf;
 use kclvm_driver::canonicalize_input_files;
+use kclvm_loader::{load_packages, LoadPackageOptions};
 use kclvm_parser::load_program;
 use kclvm_parser::parse_file;
+use kclvm_parser::KCLModuleCache;
 use kclvm_parser::LoadProgramOptions;
 use kclvm_parser::ParseSession;
 use kclvm_query::get_schema_type;
@@ -30,8 +32,7 @@ use kclvm_tools::vet::validator::LoaderKind;
 use kclvm_tools::vet::validator::ValidateOption;
 use tempfile::NamedTempFile;
 
-use super::into::IntoError;
-use super::into::IntoLoadSettingsFiles;
+use super::into::*;
 use super::ty::kcl_schema_ty_to_pb_ty;
 use super::util::transform_str_para;
 
@@ -75,7 +76,7 @@ impl KclvmServiceImpl {
     /// // File case
     /// let serv = KclvmServiceImpl::default();
     /// let args = &ParseProgramArgs {
-    ///     paths: vec![Path::new(".").join("src").join("testdata").join("test.k").canonicalize().unwrap().display().to_string(),],
+    ///     paths: vec![Path::new(".").join("src").join("testdata").join("test.k").canonicalize().unwrap().display().to_string()],
     ///     ..Default::default()
     /// };
     /// let result = serv.parse_program(args).unwrap();
@@ -95,9 +96,10 @@ impl KclvmServiceImpl {
             Some(LoadProgramOptions {
                 k_code_list: args.sources.clone(),
                 package_maps,
+                load_plugins: true,
                 ..Default::default()
             }),
-            None,
+            Some(KCLModuleCache::default()),
         )?;
         let ast_json = serde_json::to_string(&result.program)?;
 
@@ -146,6 +148,81 @@ impl KclvmServiceImpl {
         })
     }
 
+    /// load_package provides users with the ability to parse kcl program and sematic model
+    /// information including symbols, types, definitions, etc.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use kclvm_api::service::service_impl::KclvmServiceImpl;
+    /// use kclvm_api::gpyrpc::*;
+    /// use std::path::Path;
+    ///
+    /// let serv = KclvmServiceImpl::default();
+    /// let args = &LoadPackageArgs {
+    ///     parse_args: Some(ParseProgramArgs {
+    ///         paths: vec![Path::new(".").join("src").join("testdata").join("parse").join("main.k").canonicalize().unwrap().display().to_string()],
+    ///         ..Default::default()
+    ///     }),
+    ///     resolve_ast: true,
+    ///     ..Default::default()
+    /// };
+    /// let result = serv.load_package(args).unwrap();
+    /// assert_eq!(result.paths.len(), 3);
+    /// assert_eq!(result.parse_errors.len(), 0);
+    /// assert_eq!(result.type_errors.len(), 0);
+    /// assert_eq!(result.node_symbol_map.len(), 159);
+    /// assert_eq!(result.symbols.len(), 12);
+    /// ```
+    pub fn load_package(&self, args: &LoadPackageArgs) -> anyhow::Result<LoadPackageResult> {
+        let mut package_maps = HashMap::new();
+        let parse_args = args.parse_args.clone().unwrap_or_default();
+        for p in &parse_args.external_pkgs {
+            package_maps.insert(p.pkg_name.to_string(), p.pkg_path.to_string());
+        }
+        let packages = load_packages(&LoadPackageOptions {
+            paths: parse_args.paths,
+            load_opts: Some(LoadProgramOptions {
+                k_code_list: parse_args.sources.clone(),
+                package_maps,
+                load_plugins: true,
+                ..Default::default()
+            }),
+            resolve_ast: args.resolve_ast,
+            load_builtin: args.load_builtin,
+        })?;
+        let program_json = serde_json::to_string(&packages.program)?;
+        let mut node_symbol_map = HashMap::new();
+        let mut symbols = HashMap::new();
+        for (k, s) in packages.node_symbol_map {
+            node_symbol_map.insert(k.id.to_string(), s.into_symbol_index());
+        }
+        for (k, s) in packages.symbols {
+            let symbol_index_string = serde_json::to_string(&k)?;
+            symbols.insert(symbol_index_string, s.into_symbol());
+        }
+        Ok(LoadPackageResult {
+            program: program_json,
+            paths: packages
+                .paths
+                .iter()
+                .map(|p| p.to_str().unwrap().to_string())
+                .collect(),
+            node_symbol_map,
+            symbols,
+            parse_errors: packages
+                .parse_errors
+                .into_iter()
+                .map(|e| e.into_error())
+                .collect(),
+            type_errors: packages
+                .type_errors
+                .into_iter()
+                .map(|e| e.into_error())
+                .collect(),
+        })
+    }
+
     /// Execute KCL file with args. **Note that it is not thread safe.**
     ///
     /// # Examples

kclvm/ast/src/ast.rs

@@ -82,11 +82,27 @@ impl Into<Range> for Pos {
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub struct AstIndex(uuid::Uuid);
 
+impl Serialize for AstIndex {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_bytes(self.0.as_bytes())
+    }
+}
+
 impl Default for AstIndex {
     fn default() -> Self {
         Self(uuid::Uuid::new_v4())
     }
 }
+
+impl ToString for AstIndex {
+    fn to_string(&self) -> String {
+        self.0.to_string()
+    }
+}
+
 /// Node is the file, line and column number information
 /// that all AST nodes need to contain.
 /// In fact, column and end_column are the counts of character,

kclvm/loader/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "kclvm-loader"
+version = "0.7.4"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[build-dependencies]
+cc = "1.0"
+
+[dependencies]
+serde_json = "1.0"
+serde = { version = "1", features = ["derive"] }
+glob = "0.3.0"
+walkdir = "2"
+libc = "0.2.112"
+indexmap = "1.0"
+fslock = "0.2.1"
+libloading = "0.7.3"
+threadpool = "1.0"
+chrono = "0.4.19"
+tempfile = "3.5.0"
+anyhow = "1.0"
+once_cell = "1.10"
+cc = "1.0"
+compiler_base_session = {path = "../../compiler_base/session"}
+compiler_base_macros = "0.0.1"
+
+kclvm-ast = {path = "../ast"}
+kclvm-parser = {path = "../parser"}
+kclvm-compiler = {path = "../compiler"}
+kclvm-config = {path = "../config"}
+kclvm-runtime = {path = "../runtime"}
+kclvm-sema = {path = "../sema"}
+kclvm-version = {path = "../version"}
+kclvm-error = {path = "../error"}
+kclvm-query = {path = "../query"}
+kclvm-utils = {path = "../utils"}
+kclvm-driver = {path = "../driver"}