feat: enhance doc parse and add lambda doc hover (#1342)

Signed-off-by: peefy <[email protected]>

kclvm/compiler/src/codegen/llvm/node.rs

@@ -61,7 +61,7 @@ impl<'ctx> TypedResultWalker<'ctx> for LLVMCodeGenContext<'ctx> {
         let mut result = self.ok_result();
         for expr in &expr_stmt.exprs {
             let scalar = self.walk_expr(expr)?;
-            // Only non-call expressions are allowed to emit values bacause of the function void return type.
+            // Only non-call expressions are allowed to emit values because of the function void return type.
             if !matches!(expr.node, ast::Expr::Call(_)) {
                 self.add_scalar(scalar, matches!(expr.node, ast::Expr::Schema(_)));
             }

kclvm/evaluator/src/node.rs

@@ -61,7 +61,7 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> {
         let mut result = self.ok_result();
         for expr in &expr_stmt.exprs {
             let scalar = self.walk_expr(expr)?;
-            // Only non-call expressions are allowed to emit values bacause of the function void return type.
+            // Only non-call expressions are allowed to emit values because of the function void return type.
             if !matches!(expr.node, ast::Expr::Call(_)) {
                 self.add_scalar(scalar.clone(), matches!(expr.node, ast::Expr::Schema(_)));
             }

kclvm/makefile

@@ -60,6 +60,10 @@ build-wasm:
 test:
 	cargo test --workspace -r --features llvm -- --nocapture
 
+# Unit tests without code cov and llvm-features
+test-without-llvm:
+	cargo test --workspace -r -- --nocapture
+
 # Unit tests with code cov (Requires rust 1.60+)
 codecov:
 	rustup component add llvm-tools-preview

kclvm/parser/src/parser/module.rs

@@ -24,17 +24,21 @@ impl<'a> Parser<'a> {
         // doc string
         match self.token.kind {
             TokenKind::Literal(lit) => {
-                if let LitKind::Str { .. } = lit.kind {
-                    let doc_expr = self.parse_str_expr(lit);
-                    self.skip_newlines();
-                    match &doc_expr.node {
-                        Expr::StringLit(str) => {
-                            Some(node_ref!(str.raw_value.clone(), doc_expr.pos()))
+                if let LitKind::Str { is_long_string, .. } = lit.kind {
+                    if is_long_string {
+                        let doc_expr = self.parse_str_expr(lit);
+                        self.skip_newlines();
+                        match &doc_expr.node {
+                            Expr::StringLit(str) => {
+                                Some(node_ref!(str.raw_value.clone(), doc_expr.pos()))
+                            }
+                            Expr::JoinedString(str) => {
+                                Some(node_ref!(str.raw_value.clone(), doc_expr.pos()))
+                            }
+                            _ => None,
                         }
-                        Expr::JoinedString(str) => {
-                            Some(node_ref!(str.raw_value.clone(), doc_expr.pos()))
-                        }
-                        _ => None,
+                    } else {
+                        None
                     }
                 } else {
                     None

kclvm/parser/src/parser/stmt.rs

@@ -1474,26 +1474,7 @@ impl<'a> Parser<'a> {
         self.bump_token(TokenKind::Indent(VALID_SPACES_LENGTH));
 
         // doc string
-        let body_doc = match self.token.kind {
-            TokenKind::Literal(lit) => {
-                if let LitKind::Str { .. } = lit.kind {
-                    let doc_expr = self.parse_str_expr(lit);
-                    self.skip_newlines();
-                    match &doc_expr.node {
-                        Expr::StringLit(str) => {
-                            Some(node_ref!(str.raw_value.clone(), doc_expr.pos()))
-                        }
-                        Expr::JoinedString(str) => {
-                            Some(node_ref!(str.raw_value.clone(), doc_expr.pos()))
-                        }
-                        _ => None,
-                    }
-                } else {
-                    None
-                }
-            }
-            _ => None,
-        };
+        let body_doc = self.parse_doc();
 
         let mut check_expr_list = vec![];
         self.validate_dedent();

kclvm/parser/src/session/mod.rs

@@ -41,6 +41,14 @@ impl ParseSession {
         )
     }
 
+    /// Construct an loc of ont token.
+    pub fn token_loc(&self, tok: Token) -> (Loc, Loc) {
+        (
+            self.lookup_char_pos(tok.span.lo()),
+            self.lookup_char_pos(tok.span.hi()),
+        )
+    }
+
     /// Struct and report an error based on a token and not abort the compiler process.
     #[inline]
     pub fn struct_token_error(&self, expected: &[String], got: Token) {

kclvm/parser/src/tests/snapshots/kclvm_parser__tests__error_recovery__schema_stmt_recovery_34.snap

@@ -1,6 +1,5 @@
 ---
 source: parser/src/tests/error_recovery.rs
-assertion_line: 284
 expression: "crate::tests::parsing_module_string(r#\"\nschema A:\n    \"attr: str\"#)"
 ---
 Module {
@@ -12,16 +11,7 @@ Module {
         Node {
             node: Schema(
                 SchemaStmt {
-                    doc: Some(
-                        Node {
-                            node: "\"attr: str",
-                            filename: "",
-                            line: 3,
-                            column: 4,
-                            end_line: 3,
-                            end_column: 14,
-                        },
-                    ),
+                    doc: None,
                     name: Node {
                         node: "A",
                         filename: "",
@@ -36,7 +26,35 @@ Module {
                     is_protocol: false,
                     args: None,
                     mixins: [],
-                    body: [],
+                    body: [
+                        Node {
+                            node: Expr(
+                                ExprStmt {
+                                    exprs: [
+                                        Node {
+                                            node: StringLit(
+                                                StringLit {
+                                                    is_long_string: false,
+                                                    raw_value: "\"attr: str",
+                                                    value: "attr: str",
+                                                },
+                                            ),
+                                            filename: "",
+                                            line: 3,
+                                            column: 4,
+                                            end_line: 3,
+                                            end_column: 14,
+                                        },
+                                    ],
+                                },
+                            ),
+                            filename: "",
+                            line: 3,
+                            column: 4,
+                            end_line: 3,
+                            end_column: 14,
+                        },
+                    ],
                     decorators: [],
                     checks: [],
                     index_signature: None,
@@ -51,4 +69,3 @@ Module {
     ],
     comments: [],
 }
-

kclvm/query/src/tests.rs

@@ -587,19 +587,19 @@ fn test_list_variable_with_invalid_kcl() {
     let specs = vec!["a".to_string()];
     let result = list_variables(file.clone(), specs).unwrap();
     assert_eq!(result.variables.get("a"), None);
-    assert_eq!(result.parse_errors.len(), 1);
+    assert_eq!(result.parse_errors.len(), 2);
     assert_eq!(result.parse_errors[0].level, Level::Error);
     assert_eq!(
         result.parse_errors[0].code,
         Some(DiagnosticId::Error(ErrorKind::InvalidSyntax))
     );
     assert_eq!(
         result.parse_errors[0].messages[0].message,
-        "unexpected token ':'"
+        "expected one of [\"=\"] got eof",
     );
     assert_eq!(result.parse_errors[0].messages[0].range.0.filename, file);
     assert_eq!(result.parse_errors[0].messages[0].range.0.line, 1);
-    assert_eq!(result.parse_errors[0].messages[0].range.0.column, Some(3));
+    assert_eq!(result.parse_errors[0].messages[0].range.0.column, Some(8));
 }
 
 #[test]
@@ -617,22 +617,22 @@ fn test_overridefile_with_invalid_kcl() {
 
     fs::copy(simple_bk_path.clone(), simple_path.clone()).unwrap();
     assert_eq!(result.result, true);
-    assert_eq!(result.parse_errors.len(), 1);
+    assert_eq!(result.parse_errors.len(), 2);
     assert_eq!(result.parse_errors[0].level, Level::Error);
     assert_eq!(
         result.parse_errors[0].code,
         Some(DiagnosticId::Error(ErrorKind::InvalidSyntax))
     );
     assert_eq!(
         result.parse_errors[0].messages[0].message,
-        "unexpected token ':'"
+        "expected one of [\"=\"] got eof"
     );
     assert_eq!(
         result.parse_errors[0].messages[0].range.0.filename,
         simple_path.display().to_string()
     );
     assert_eq!(result.parse_errors[0].messages[0].range.0.line, 1);
-    assert_eq!(result.parse_errors[0].messages[0].range.0.column, Some(3));
+    assert_eq!(result.parse_errors[0].messages[0].range.0.column, Some(8));
 }
 
 #[test]

kclvm/sema/src/resolver/doc.rs

@@ -1,4 +1,4 @@
-use kclvm_ast::ast::SchemaStmt;
+use kclvm_ast::ast::{self, SchemaStmt};
 use std::collections::{HashMap, HashSet};
 use std::iter::Iterator;
 use std::str;
@@ -157,10 +157,6 @@ impl Reader {
             }
         }
     }
-
-    fn _is_empty(&self) -> bool {
-        return self.data.iter().all(|x| x.trim().len() == 0);
-    }
 }
 
 /// Checks if current line is at the beginning of a section
@@ -240,11 +236,11 @@ fn parse_summary(doc: &mut Reader) -> String {
 }
 
 /// Parse the schema docstring to Doc.
-/// The summary of the schema content will be concatenated to a single line string by whitespaces.
+/// The summary of the schema content will be concatenated to a single line string by whitespace.
 /// The description of each attribute will be returned as separate lines.
-pub fn parse_doc_string(ori: &str) -> Doc {
+pub fn parse_schema_doc_string(ori: &str) -> SchemaDoc {
     if ori.is_empty() {
-        return Doc::new("".to_string(), vec![], HashMap::new());
+        return SchemaDoc::new("".to_string(), vec![], HashMap::new());
     }
     let mut doc = Reader::new(clean_doc(strip_quotes(&ori)));
     doc.reset();
@@ -268,18 +264,18 @@ pub fn parse_doc_string(ori: &str) -> Doc {
             Example::new("".to_string(), "".to_string(), default_example_content),
         );
     }
-    Doc::new(summary, attrs, examples)
+    SchemaDoc::new(summary, attrs, examples)
 }
 
 /// The Doc struct contains a summary of schema and all the attributes described in the the docstring.
 #[derive(Debug, PartialEq, Eq, Clone)]
-pub struct Doc {
+pub struct SchemaDoc {
     pub summary: String,
     pub attrs: Vec<Attribute>,
     pub examples: HashMap<String, Example>,
 }
 
-impl Doc {
+impl SchemaDoc {
     pub fn new(summary: String, attrs: Vec<Attribute>, examples: HashMap<String, Example>) -> Self {
         Self {
             summary,
@@ -357,10 +353,32 @@ impl Example {
     }
 }
 
+/// Extract doc string from the AST body, if the first statement is a long string expression
+/// statement, convert it to a doc string.
+pub fn extract_doc_from_body(stmts: &[Box<ast::Node<ast::Stmt>>]) -> Option<String> {
+    match stmts.first() {
+        Some(stmt) => match &stmt.node {
+            ast::Stmt::Expr(expr_stmt) => match expr_stmt.exprs.first() {
+                Some(expr) => match &expr.node {
+                    ast::Expr::StringLit(str) if str.is_long_string => Some(str.raw_value.clone()),
+                    ast::Expr::JoinedString(str) if str.is_long_string => {
+                        Some(str.raw_value.clone())
+                    }
+                    _ => None,
+                },
+                None => None,
+            },
+            _ => None,
+        },
+        None => None,
+    }
+    .map(|v| clean_doc(strip_quotes(&v)))
+}
+
 #[cfg(test)]
 mod tests {
     use super::{clean_doc, is_at_section, read_to_next_section, strip_quotes, Reader};
-    use crate::resolver::doc::{parse_doc_string, Example};
+    use crate::resolver::doc::{parse_schema_doc_string, Example};
     use std::fs::File;
     use std::io::prelude::*;
     use std::path::PathBuf;
@@ -597,7 +615,7 @@ unindented line
     #[test]
     fn test_parse_doc() {
         let mut content = read_doc_content();
-        let doc = parse_doc_string(&mut content);
+        let doc = parse_schema_doc_string(&mut content);
         assert_eq!(
             doc.summary,
             "Server is the common user interface for long-running services adopting the best practice of Kubernetes."

kclvm/sema/src/resolver/global.rs

@@ -12,7 +12,7 @@ use kclvm_ast::ast;
 use kclvm_ast_pretty::{print_ast_node, print_schema_expr, ASTNode};
 use kclvm_error::*;
 
-use super::doc::parse_doc_string;
+use super::doc::parse_schema_doc_string;
 use super::scope::{ScopeObject, ScopeObjectKind};
 use kclvm_ast::pos::GetPos;
 
@@ -79,7 +79,7 @@ impl<'ctx> Resolver<'ctx> {
                             );
                             continue;
                         }
-                        let parsed_doc = parse_doc_string(&doc);
+                        let parsed_doc = parse_schema_doc_string(&doc);
                         let schema_ty = SchemaType {
                             name: name.to_string(),
                             pkgpath: self.ctx.pkgpath.clone(),
@@ -567,7 +567,7 @@ impl<'ctx> Resolver<'ctx> {
         };
         // Schema attributes
         let mut attr_obj_map: IndexMap<String, SchemaAttr> = IndexMap::default();
-        let parsed_doc = parse_doc_string(
+        let parsed_doc = parse_schema_doc_string(
             &schema_stmt
                 .doc
                 .as_ref()
@@ -900,7 +900,7 @@ impl<'ctx> Resolver<'ctx> {
             &rule_stmt.name.node,
         );
 
-        let parsed_doc = parse_doc_string(
+        let parsed_doc = parse_schema_doc_string(
             &rule_stmt
                 .doc
                 .as_ref()

kclvm/sema/src/resolver/node.rs

@@ -11,6 +11,7 @@ use crate::ty::{
     RESERVED_TYPE_IDENTIFIERS,
 };
 
+use super::doc::extract_doc_from_body;
 use super::format::VALID_FORMAT_SPEC_SET;
 use super::scope::{ScopeKind, ScopeObject, ScopeObjectKind};
 use super::ty::ty_str_replace_pkgpath;
@@ -1071,7 +1072,15 @@ impl<'ctx> MutSelfTypedResultWalker<'ctx> for Resolver<'ctx> {
         if !real_ret_ty.is_any() && ret_ty.is_any() && lambda_expr.return_ty.is_none() {
             ret_ty = real_ret_ty;
         }
-        Arc::new(Type::function(None, ret_ty, &params, "", false, None))
+        let doc = extract_doc_from_body(&lambda_expr.body);
+        Arc::new(Type::function(
+            None,
+            ret_ty,
+            &params,
+            &doc.unwrap_or_default(),
+            false,
+            None,
+        ))
     }
 
     fn walk_keyword(&mut self, keyword: &'ctx ast::Keyword) -> Self::Result {

kclvm/tools/src/LSP/src/completion.rs

@@ -30,7 +30,7 @@ use kclvm_error::Position as KCLPos;
 use kclvm_sema::builtin::{BUILTIN_FUNCTIONS, STANDARD_SYSTEM_MODULES};
 use kclvm_sema::core::package::ModuleInfo;
 use kclvm_sema::core::symbol::SymbolKind;
-use kclvm_sema::resolver::doc::{parse_doc_string, Doc};
+use kclvm_sema::resolver::doc::{parse_schema_doc_string, SchemaDoc};
 use kclvm_sema::ty::{FunctionType, SchemaType, Type, TypeKind};
 use lsp_types::{CompletionItem, CompletionItemKind, InsertTextFormat};
 
@@ -371,10 +371,10 @@ fn completion_newline(
     let mut completions: IndexSet<KCLCompletionItem> = IndexSet::new();
 
     if let Some((doc, schema)) = is_in_docstring(program, pos) {
-        let doc = parse_doc_string(&doc.node);
+        let doc = parse_schema_doc_string(&doc.node);
         if doc.summary.is_empty() && doc.attrs.is_empty() && doc.examples.is_empty() {
             // empty docstring, provide total completion
-            let doc_parsed = Doc::new_from_schema_stmt(&schema);
+            let doc_parsed = SchemaDoc::new_from_schema_stmt(&schema);
             let label = doc_parsed.to_doc_string();
             // generate docstring from doc
             completions.insert(KCLCompletionItem {