Skip to content

Commit

Permalink
feat: support increment parser
Browse files Browse the repository at this point in the history 
Signed-off-by: never <[email protected]>
  • Loading branch information
@NeverRaR
NeverRaR committed Nov 8, 2023
1 parent 350e34a commit f4b55f4
Show file tree
Hide file tree
Showing 13 changed files with 257 additions and 74 deletions.
75 changes: 65 additions & 10 deletions kclvm/parser/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pub use crate::session::ParseSession;
use compiler_base_macros::bug;
use compiler_base_session::Session;
use compiler_base_span::span::new_byte_pos;
use indexmap::IndexMap;
use kclvm_ast::ast;
use kclvm_config::modfile::{get_vendor_home, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE};
use kclvm_error::diagnostic::Range;
Expand All @@ -26,6 +27,7 @@ use kclvm_utils::pkgpath::rm_external_pkg_name;

use lexer::parse_token_streams;
use parser::Parser;
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
Expand Down Expand Up @@ -277,33 +279,62 @@ impl Default for LoadProgramOptions {
}
}

/// Load the KCL program by paths and options,
/// "module_cache" is used to cache parsed asts to support incremental parse,
/// if it is None, module caching will be disabled
///
/// # Examples
///
/// ```
/// use kclvm_parser::{load_program, ParseSession};
/// use kclvm_parser::KCLModuleCache;
/// use kclvm_ast::ast::Program;
/// use std::sync::Arc;
///
/// // Create sessions
/// let sess = Arc::new(ParseSession::default());
/// // Create module cache
/// let module_cache = KCLModuleCache::default();
/// // Get default args
/// let args = ExecProgramArgs::default();
/// let opts = args.get_load_program_options();
///
/// // Parse kcl file
/// let kcl_path = "./src/test_data/import-01.k";
/// let prog = load_program(sess.clone(), &[kcl_path], Some(opts), Some(module_cache.clone())).unwrap();
///
/// ```
pub fn load_program(
sess: Arc<ParseSession>,
paths: &[&str],
opts: Option<LoadProgramOptions>,
module_cache: Option<KCLModuleCache>,
) -> Result<ast::Program, String> {
// todo: support cache
if let Some(opts) = opts {
Loader::new(sess, paths, Some(opts)).load_main()
} else {
Loader::new(sess, paths, None).load_main()
}
Loader::new(sess, paths, opts, module_cache).load_main()
}

pub type KCLModuleCache = Arc<RefCell<IndexMap<String, ast::Module>>>;
struct Loader {
sess: Arc<ParseSession>,
paths: Vec<String>,
opts: LoadProgramOptions,
missing_pkgs: Vec<String>,
module_cache: Option<KCLModuleCache>,
}

impl Loader {
fn new(sess: Arc<ParseSession>, paths: &[&str], opts: Option<LoadProgramOptions>) -> Self {
fn new(
sess: Arc<ParseSession>,
paths: &[&str],
opts: Option<LoadProgramOptions>,
module_cache: Option<Arc<RefCell<IndexMap<String, ast::Module>>>>,
) -> Self {
Self {
sess,
paths: paths.iter().map(|s| s.to_string()).collect(),
opts: opts.unwrap_or_default(),
missing_pkgs: Default::default(),
module_cache,
}
}

Expand All @@ -329,8 +360,18 @@ impl Loader {
// load module

for (i, filename) in k_files.iter().enumerate() {
let mut m =
parse_file_with_session(self.sess.clone(), filename, maybe_k_codes[i].clone())?;
let mut m = if let Some(module_cache) = self.module_cache.as_ref() {
let m = parse_file_with_session(
self.sess.clone(),
filename,
maybe_k_codes[i].clone(),
)?;
let mut module_cache_ref = module_cache.borrow_mut();
module_cache_ref.insert(filename.clone(), m.clone());
m
} else {
parse_file_with_session(self.sess.clone(), filename, maybe_k_codes[i].clone())?
};
self.fix_rel_import_path(entry.path(), &mut m);
pkg_files.push(m);
}
Expand Down Expand Up @@ -532,7 +573,21 @@ impl Loader {
let mut pkg_files = Vec::new();
let k_files = pkg_info.k_files.clone();
for filename in k_files {
let mut m = parse_file_with_session(self.sess.clone(), filename.as_str(), None)?;
let mut m = if let Some(module_cache) = self.module_cache.as_ref() {
let module_cache_ref = module_cache.borrow();
if let Some(module) = module_cache_ref.get(&filename) {
module.clone()
} else {
let m = parse_file_with_session(self.sess.clone(), &filename, None)?;
drop(module_cache_ref);
let mut module_cache_ref = module_cache.borrow_mut();
module_cache_ref.insert(filename.clone(), m.clone());
m
}
} else {
parse_file_with_session(self.sess.clone(), &filename, None)?
};
//let mut m = parse_file_with_session(self.sess.clone(), filename.as_str(), None)?;

m.pkg = pkg_info.pkg_path.clone();
m.name = "".to_string();
Expand Down
199 changes: 149 additions & 50 deletions kclvm/parser/src/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ fn test_in_order() {
}

pub fn test_import_vendor() {
let module_cache = KCLModuleCache::default();
let vendor = set_vendor_home();
let sm = SourceMap::new(FilePathMapping::empty());
let sess = Arc::new(ParseSession::with_source_map(Arc::new(sm)));
Expand Down Expand Up @@ -285,27 +286,36 @@ pub fn test_import_vendor() {
.canonicalize()
.unwrap();

test_cases.into_iter().for_each(|(test_case_name, pkgs)| {
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None).unwrap();
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
println!("{:?} - {:?}", test_case_name, name);
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
let test_fn =
|test_case_name: &&str, pkgs: &Vec<&str>, module_cache: Option<KCLModuleCache>| {
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None, module_cache).unwrap();
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
println!("{:?} - {:?}", test_case_name, name);
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
}
break;
}
break;
}
}
});
});
};

test_cases
.iter()
.for_each(|(test_case_name, pkgs)| test_fn(test_case_name, pkgs, None));

test_cases.iter().for_each(|(test_case_name, pkgs)| {
test_fn(test_case_name, pkgs, Some(module_cache.clone()))
});
}

Expand All @@ -323,7 +333,7 @@ pub fn test_import_vendor_without_kclmod() {

test_cases.into_iter().for_each(|(test_case_name, pkgs)| {
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None).unwrap();
let m = load_program(sess.clone(), &[&test_case_path], None, None).unwrap();
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
assert!(pkgs.contains(&name.as_str()));
Expand Down Expand Up @@ -354,7 +364,29 @@ pub fn test_import_vendor_without_vendor_home() {
.canonicalize()
.unwrap();
let test_case_path = dir.join("assign.k").display().to_string();
match load_program(sess.clone(), &[&test_case_path], None) {
match load_program(sess.clone(), &[&test_case_path], None, None) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
"pkgpath assign not found in the program",
"pkgpath assign.assign not found in the program",
];
assert_eq!(errors.len(), msgs.len());
for (diag, m) in errors.iter().zip(msgs.iter()) {
assert_eq!(diag.messages[0].message, m.to_string());
}
}
Err(_) => {
panic!("Unreachable code.")
}
}

match load_program(
sess.clone(),
&[&test_case_path],
None,
Some(KCLModuleCache::default()),
) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
Expand Down Expand Up @@ -382,7 +414,27 @@ fn test_import_vendor_with_same_internal_pkg() {
.canonicalize()
.unwrap();
let test_case_path = dir.join("same_name.k").display().to_string();
match load_program(sess.clone(), &[&test_case_path], None) {
match load_program(sess.clone(), &[&test_case_path], None, None) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
"the `same_vendor` is found multiple times in the current package and vendor package"
];
assert_eq!(errors.len(), msgs.len());
for (diag, m) in errors.iter().zip(msgs.iter()) {
assert_eq!(diag.messages[0].message, m.to_string());
}
}
Err(_) => {
panic!("Unreachable code.")
}
}
match load_program(
sess.clone(),
&[&test_case_path],
None,
Some(KCLModuleCache::default()),
) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
Expand All @@ -409,7 +461,28 @@ fn test_import_vendor_without_kclmod_and_same_name() {
.canonicalize()
.unwrap();
let test_case_path = dir.join("assign.k").display().to_string();
match load_program(sess.clone(), &[&test_case_path], None) {
match load_program(sess.clone(), &[&test_case_path], None, None) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
"the `assign` is found multiple times in the current package and vendor package",
];
assert_eq!(errors.len(), msgs.len());
for (diag, m) in errors.iter().zip(msgs.iter()) {
assert_eq!(diag.messages[0].message, m.to_string());
}
}
Err(_) => {
panic!("Unreachable code.")
}
}

match load_program(
sess.clone(),
&[&test_case_path],
None,
Some(KCLModuleCache::default()),
) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
Expand All @@ -430,7 +503,7 @@ fn test_import_vendor_by_external_arguments() {
let vendor = set_vendor_home();
let sm = SourceMap::new(FilePathMapping::empty());
let sess = Arc::new(ParseSession::with_source_map(Arc::new(sm)));

let module_cache = KCLModuleCache::default();
let external_dir = &PathBuf::from(".")
.join("testdata")
.join("test_vendor")
Expand Down Expand Up @@ -480,34 +553,45 @@ fn test_import_vendor_by_external_arguments() {
.canonicalize()
.unwrap();

test_cases
.into_iter()
.for_each(|(test_case_name, dep_name, pkgs)| {
let mut opts = LoadProgramOptions::default();
opts.package_maps.insert(
dep_name.to_string(),
external_dir.join(dep_name).display().to_string(),
);
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None).unwrap();
let test_fn = |test_case_name: &&str,
dep_name: &&str,
pkgs: &Vec<&str>,
module_cache: Option<KCLModuleCache>| {
let mut opts = LoadProgramOptions::default();
opts.package_maps.insert(
dep_name.to_string(),
external_dir.join(dep_name).display().to_string(),
);
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None, module_cache).unwrap();

assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
}
break;
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
}
break;
}
});
}
});
};

test_cases
.iter()
.for_each(|(test_case_name, dep_name, pkgs)| test_fn(test_case_name, dep_name, pkgs, None));

test_cases
.iter()
.for_each(|(test_case_name, dep_name, pkgs)| {
test_fn(test_case_name, dep_name, pkgs, Some(module_cache.clone()))
});
}

Expand Down Expand Up @@ -595,7 +679,22 @@ fn test_dir_with_k_code_list() {
let mut opts = LoadProgramOptions::default();
opts.k_code_list = vec!["test_code = 1".to_string()];

match load_program(sess.clone(), &[&testpath.display().to_string()], Some(opts)) {
match load_program(
sess.clone(),
&[&testpath.display().to_string()],
Some(opts.clone()),
None,
) {
Ok(_) => panic!("unreachable code"),
Err(err) => assert_eq!(err, "Invalid code list"),
}

match load_program(
sess.clone(),
&[&testpath.display().to_string()],
Some(opts),
Some(KCLModuleCache::default()),
) {
Ok(_) => panic!("unreachable code"),
Err(err) => assert_eq!(err, "Invalid code list"),
}
Expand Down
Loading

0 comments on commit f4b55f4

Please sign in to comment.