Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support increment parser #857

Merged
merged 1 commit into from
Nov 8, 2023
Merged

feat: support increment parser #857

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 61 additions & 10 deletions kclvm/parser/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pub use crate::session::ParseSession;
use compiler_base_macros::bug;
use compiler_base_session::Session;
use compiler_base_span::span::new_byte_pos;
use indexmap::IndexMap;
use kclvm_ast::ast;
use kclvm_config::modfile::{get_vendor_home, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE};
use kclvm_error::diagnostic::Range;
Expand All @@ -26,6 +27,7 @@ use kclvm_utils::pkgpath::rm_external_pkg_name;

use lexer::parse_token_streams;
use parser::Parser;
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
Expand Down Expand Up @@ -277,33 +279,59 @@ impl Default for LoadProgramOptions {
}
}

/// Load the KCL program by paths and options,
/// "module_cache" is used to cache parsed asts to support incremental parse,
/// if it is None, module caching will be disabled
///
/// # Examples
///
/// ```
/// use kclvm_parser::{load_program, ParseSession};
/// use kclvm_parser::KCLModuleCache;
/// use kclvm_ast::ast::Program;
/// use std::sync::Arc;
///
/// // Create sessions
/// let sess = Arc::new(ParseSession::default());
/// // Create module cache
/// let module_cache = KCLModuleCache::default();
///
/// // Parse kcl file
/// let kcl_path = "./testdata/import-01.k";
/// let prog = load_program(sess.clone(), &[kcl_path], None, Some(module_cache.clone())).unwrap();
///
/// ```
pub fn load_program(
sess: Arc<ParseSession>,
paths: &[&str],
opts: Option<LoadProgramOptions>,
module_cache: Option<KCLModuleCache>,
) -> Result<ast::Program, String> {
// todo: support cache
if let Some(opts) = opts {
Loader::new(sess, paths, Some(opts)).load_main()
} else {
Loader::new(sess, paths, None).load_main()
}
Loader::new(sess, paths, opts, module_cache).load_main()
}

pub type KCLModuleCache = Arc<RefCell<IndexMap<String, ast::Module>>>;
struct Loader {
sess: Arc<ParseSession>,
paths: Vec<String>,
opts: LoadProgramOptions,
missing_pkgs: Vec<String>,
module_cache: Option<KCLModuleCache>,
}

impl Loader {
fn new(sess: Arc<ParseSession>, paths: &[&str], opts: Option<LoadProgramOptions>) -> Self {
fn new(
sess: Arc<ParseSession>,
paths: &[&str],
opts: Option<LoadProgramOptions>,
module_cache: Option<Arc<RefCell<IndexMap<String, ast::Module>>>>,
) -> Self {
Self {
sess,
paths: paths.iter().map(|s| s.to_string()).collect(),
opts: opts.unwrap_or_default(),
missing_pkgs: Default::default(),
module_cache,
}
}

Expand All @@ -329,8 +357,18 @@ impl Loader {
// load module

for (i, filename) in k_files.iter().enumerate() {
let mut m =
parse_file_with_session(self.sess.clone(), filename, maybe_k_codes[i].clone())?;
let mut m = if let Some(module_cache) = self.module_cache.as_ref() {
let m = parse_file_with_session(
self.sess.clone(),
filename,
maybe_k_codes[i].clone(),
)?;
let mut module_cache_ref = module_cache.borrow_mut();
module_cache_ref.insert(filename.clone(), m.clone());
m
} else {
parse_file_with_session(self.sess.clone(), filename, maybe_k_codes[i].clone())?
};
self.fix_rel_import_path(entry.path(), &mut m);
pkg_files.push(m);
}
Expand Down Expand Up @@ -532,7 +570,20 @@ impl Loader {
let mut pkg_files = Vec::new();
let k_files = pkg_info.k_files.clone();
for filename in k_files {
let mut m = parse_file_with_session(self.sess.clone(), filename.as_str(), None)?;
let mut m = if let Some(module_cache) = self.module_cache.as_ref() {
let module_cache_ref = module_cache.borrow();
if let Some(module) = module_cache_ref.get(&filename) {
module.clone()
} else {
let m = parse_file_with_session(self.sess.clone(), &filename, None)?;
drop(module_cache_ref);
let mut module_cache_ref = module_cache.borrow_mut();
module_cache_ref.insert(filename.clone(), m.clone());
m
}
} else {
parse_file_with_session(self.sess.clone(), &filename, None)?
};

m.pkg = pkg_info.pkg_path.clone();
m.name = "".to_string();
Expand Down
199 changes: 149 additions & 50 deletions kclvm/parser/src/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ fn test_in_order() {
}

pub fn test_import_vendor() {
let module_cache = KCLModuleCache::default();
let vendor = set_vendor_home();
let sm = SourceMap::new(FilePathMapping::empty());
let sess = Arc::new(ParseSession::with_source_map(Arc::new(sm)));
Expand Down Expand Up @@ -285,27 +286,36 @@ pub fn test_import_vendor() {
.canonicalize()
.unwrap();

test_cases.into_iter().for_each(|(test_case_name, pkgs)| {
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None).unwrap();
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
println!("{:?} - {:?}", test_case_name, name);
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
let test_fn =
|test_case_name: &&str, pkgs: &Vec<&str>, module_cache: Option<KCLModuleCache>| {
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None, module_cache).unwrap();
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
println!("{:?} - {:?}", test_case_name, name);
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
}
break;
}
break;
}
}
});
});
};

test_cases
.iter()
.for_each(|(test_case_name, pkgs)| test_fn(test_case_name, pkgs, None));

test_cases.iter().for_each(|(test_case_name, pkgs)| {
test_fn(test_case_name, pkgs, Some(module_cache.clone()))
});
}

Expand All @@ -323,7 +333,7 @@ pub fn test_import_vendor_without_kclmod() {

test_cases.into_iter().for_each(|(test_case_name, pkgs)| {
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None).unwrap();
let m = load_program(sess.clone(), &[&test_case_path], None, None).unwrap();
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
assert!(pkgs.contains(&name.as_str()));
Expand Down Expand Up @@ -354,7 +364,29 @@ pub fn test_import_vendor_without_vendor_home() {
.canonicalize()
.unwrap();
let test_case_path = dir.join("assign.k").display().to_string();
match load_program(sess.clone(), &[&test_case_path], None) {
match load_program(sess.clone(), &[&test_case_path], None, None) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
"pkgpath assign not found in the program",
"pkgpath assign.assign not found in the program",
];
assert_eq!(errors.len(), msgs.len());
for (diag, m) in errors.iter().zip(msgs.iter()) {
assert_eq!(diag.messages[0].message, m.to_string());
}
}
Err(_) => {
panic!("Unreachable code.")
}
}

match load_program(
sess.clone(),
&[&test_case_path],
None,
Some(KCLModuleCache::default()),
) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
Expand Down Expand Up @@ -382,7 +414,27 @@ fn test_import_vendor_with_same_internal_pkg() {
.canonicalize()
.unwrap();
let test_case_path = dir.join("same_name.k").display().to_string();
match load_program(sess.clone(), &[&test_case_path], None) {
match load_program(sess.clone(), &[&test_case_path], None, None) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
"the `same_vendor` is found multiple times in the current package and vendor package"
];
assert_eq!(errors.len(), msgs.len());
for (diag, m) in errors.iter().zip(msgs.iter()) {
assert_eq!(diag.messages[0].message, m.to_string());
}
}
Err(_) => {
panic!("Unreachable code.")
}
}
match load_program(
sess.clone(),
&[&test_case_path],
None,
Some(KCLModuleCache::default()),
) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
Expand All @@ -409,7 +461,28 @@ fn test_import_vendor_without_kclmod_and_same_name() {
.canonicalize()
.unwrap();
let test_case_path = dir.join("assign.k").display().to_string();
match load_program(sess.clone(), &[&test_case_path], None) {
match load_program(sess.clone(), &[&test_case_path], None, None) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
"the `assign` is found multiple times in the current package and vendor package",
];
assert_eq!(errors.len(), msgs.len());
for (diag, m) in errors.iter().zip(msgs.iter()) {
assert_eq!(diag.messages[0].message, m.to_string());
}
}
Err(_) => {
panic!("Unreachable code.")
}
}

match load_program(
sess.clone(),
&[&test_case_path],
None,
Some(KCLModuleCache::default()),
) {
Ok(_) => {
let errors = sess.classification().0;
let msgs = [
Expand All @@ -430,7 +503,7 @@ fn test_import_vendor_by_external_arguments() {
let vendor = set_vendor_home();
let sm = SourceMap::new(FilePathMapping::empty());
let sess = Arc::new(ParseSession::with_source_map(Arc::new(sm)));

let module_cache = KCLModuleCache::default();
let external_dir = &PathBuf::from(".")
.join("testdata")
.join("test_vendor")
Expand Down Expand Up @@ -480,34 +553,45 @@ fn test_import_vendor_by_external_arguments() {
.canonicalize()
.unwrap();

test_cases
.into_iter()
.for_each(|(test_case_name, dep_name, pkgs)| {
let mut opts = LoadProgramOptions::default();
opts.package_maps.insert(
dep_name.to_string(),
external_dir.join(dep_name).display().to_string(),
);
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None).unwrap();
let test_fn = |test_case_name: &&str,
dep_name: &&str,
pkgs: &Vec<&str>,
module_cache: Option<KCLModuleCache>| {
let mut opts = LoadProgramOptions::default();
opts.package_maps.insert(
dep_name.to_string(),
external_dir.join(dep_name).display().to_string(),
);
let test_case_path = dir.join(test_case_name).display().to_string();
let m = load_program(sess.clone(), &[&test_case_path], None, module_cache).unwrap();

assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
}
break;
assert_eq!(m.pkgs.len(), pkgs.len());
m.pkgs.into_iter().for_each(|(name, modules)| {
assert!(pkgs.contains(&name.as_str()));
for pkg in pkgs.clone() {
if name == pkg {
if name == "__main__" {
assert_eq!(modules.len(), 1);
assert_eq!(modules.get(0).unwrap().filename, test_case_path);
} else {
modules.into_iter().for_each(|module| {
assert!(module.filename.contains(&vendor));
});
}
break;
}
});
}
});
};

test_cases
.iter()
.for_each(|(test_case_name, dep_name, pkgs)| test_fn(test_case_name, dep_name, pkgs, None));

test_cases
.iter()
.for_each(|(test_case_name, dep_name, pkgs)| {
test_fn(test_case_name, dep_name, pkgs, Some(module_cache.clone()))
});
}

Expand Down Expand Up @@ -595,7 +679,22 @@ fn test_dir_with_k_code_list() {
let mut opts = LoadProgramOptions::default();
opts.k_code_list = vec!["test_code = 1".to_string()];

match load_program(sess.clone(), &[&testpath.display().to_string()], Some(opts)) {
match load_program(
sess.clone(),
&[&testpath.display().to_string()],
Some(opts.clone()),
None,
) {
Ok(_) => panic!("unreachable code"),
Err(err) => assert_eq!(err, "Invalid code list"),
}

match load_program(
sess.clone(),
&[&testpath.display().to_string()],
Some(opts),
Some(KCLModuleCache::default()),
) {
Ok(_) => panic!("unreachable code"),
Err(err) => assert_eq!(err, "Invalid code list"),
}
Expand Down
Loading
Loading