diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 40371e3cc..c748225db 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" dependencies = [ "backtrace", ] @@ -1636,6 +1636,7 @@ dependencies = [ "kclvm-runtime", "kclvm-tools", "kclvm-version", + "kclvm-vfs", ] [[package]] @@ -1774,6 +1775,7 @@ dependencies = [ "kclvm-sema", "kclvm-span", "kclvm-utils", + "kclvm-vfs", "num-bigint", "petgraph", "regex", @@ -1825,6 +1827,7 @@ dependencies = [ "kclvm-sema", "kclvm-utils", "kclvm-version", + "kclvm-vfs", "libc", "libloading", "once_cell", @@ -1957,6 +1960,23 @@ dependencies = [ "vergen", ] +[[package]] +name = "kclvm-vfs" +version = "0.1.0" +dependencies = [ + "anyhow", + "compiler_base_span 0.0.2", + "kclvm-config", + "kclvm-span", + "kclvm-utils", + "lazy_static", + "parking_lot 0.12.1", + "path-slash", + "pathdiff", + "pcre2", + "regex", +] + [[package]] name = "kclvm_runtime_internal_macros" version = "0.5.0" @@ -2129,9 +2149,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -2366,6 +2386,12 @@ dependencies = [ "windows-targets 0.48.1", ] +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "pathdiff" version = "0.2.1" @@ -2374,21 +2400,20 @@ checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" [[package]] name = "pcre2" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486aca7e74edb8cab09a48d461177f450a5cca3b55e61d139f7552190e2bbcf5" +checksum = "4c9d53a8ea5fc3d3568d3de4bebc12606fd0eb8234c602576f1f1ee4880488a7" dependencies = [ "libc", "log", "pcre2-sys", - "thread_local", ] [[package]] name = "pcre2-sys" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae234f441970dbd52d4e29bee70f3b56ca83040081cb2b55b7df772b16e0b06e" +checksum = "25b8a7b5253a4465b873a21ee7e8d6ec561a57eed5d319621bec36bea35c86ae" dependencies = [ "cc", "libc", @@ -2862,13 +2887,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.3", + "regex-automata 0.4.6", "regex-syntax", ] @@ -2880,9 +2905,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-automata" -version = "0.3.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -2891,9 +2916,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "ron" diff --git a/kclvm/Cargo.toml b/kclvm/Cargo.toml index 295f13512..7a6d7fc02 100644 --- a/kclvm/Cargo.toml +++ b/kclvm/Cargo.toml @@ -75,5 +75,6 @@ members = [ "version", "query", "utils", - "tools/src/LSP" + "tools/src/LSP", + "vfs" ] diff --git a/kclvm/cmd/Cargo.toml b/kclvm/cmd/Cargo.toml index c4f07acaf..28f309020 100644 --- a/kclvm/cmd/Cargo.toml +++ b/kclvm/cmd/Cargo.toml @@ -17,3 +17,5 @@ kclvm-runtime = {path = "../runtime"} kclvm-tools = {path = "../tools"} kclvm-error = {path = "../error"} kclvm-version = {path = "../version"} + +kclvm-vfs ={ path = "../vfs" } diff --git a/kclvm/config/src/vfs.rs b/kclvm/config/src/vfs.rs index 25018222a..9a783d0b2 100644 --- a/kclvm/config/src/vfs.rs +++ b/kclvm/config/src/vfs.rs @@ -27,6 +27,7 @@ pub fn is_rel_pkgpath(pkgpath: &str) -> bool { pkgpath.starts_with('.') } +#[deprecated] pub fn fix_import_path(root: &str, filepath: &str, import_path: &str) -> String { // relpath: import .sub // FixImportPath(root, "path/to/app/file.k", ".sub") => path.to.app.sub @@ -95,6 +96,7 @@ pub fn fix_import_path(root: &str, filepath: &str, import_path: &str) -> String } #[test] +#[deprecated] fn test_fix_import_path() { #[cfg(not(target_os = "windows"))] let root = "/home/konfig"; diff --git a/kclvm/parser/Cargo.toml b/kclvm/parser/Cargo.toml index 56b9a8b5e..f9cfb3925 100644 --- a/kclvm/parser/Cargo.toml +++ b/kclvm/parser/Cargo.toml @@ -25,6 +25,7 @@ regex = "1.7.0" anyhow = "1.0" indexmap = "1.0" +kclvm-vfs ={ path = "../vfs"} kclvm-lexer = {path = "../lexer"} kclvm-ast = {path = "../ast"} kclvm-span = {path = "../span"} diff --git a/kclvm/parser/src/lib.rs b/kclvm/parser/src/lib.rs index 32fddf76e..217442cd1 100644 --- a/kclvm/parser/src/lib.rs +++ b/kclvm/parser/src/lib.rs @@ -1,6 +1,5 @@ //! Copyright The KCL Authors. All rights reserved. -pub mod entry; pub mod file_graph; mod lexer; mod parser; @@ -10,12 +9,13 @@ mod session; mod tests; extern crate kclvm_error; - -use crate::entry::get_compile_entries_from_paths; pub use crate::session::{ParseSession, ParseSessionRef}; use compiler_base_macros::bug; use compiler_base_session::Session; use compiler_base_span::span::new_byte_pos; +use kclvm_vfs::entry::Entries; +use kclvm_vfs::vfs::VFSPath; +use kclvm_vfs::vfs::PkgPath; use file_graph::FileGraph; use indexmap::{IndexMap, IndexSet}; use kclvm_ast::ast; @@ -27,6 +27,8 @@ use kclvm_utils::pkgpath::parse_external_pkg_name; use kclvm_utils::pkgpath::rm_external_pkg_name; use anyhow::Result; +use kclvm_vfs::sourcemap::SourceMapVfs; +use kclvm_vfs::vfs::VFS; use lexer::parse_token_streams; use parser::Parser; use std::collections::HashMap; @@ -153,6 +155,39 @@ pub fn parse_file_force_errors(filename: &str, code: Option) -> Result, + filename: String, +) -> Result { + // Code source. + let filepath = PathBuf::from(filename.clone()); + let src = String::from_utf8(vfs.read(filename)?)?; + + // TODO: Replace the sf and src_from_sf with the vfs + + // Build a source map to store file sources. + let sf = sess.0.sm.new_source_file(filepath.clone().into(), src); + + let src_from_sf = match sf.src.as_ref() { + Some(src) => src, + None => { + return Err(anyhow::anyhow!("Internal Bug: Failed to load KCL file.")); + } + }; + + // Lexer + let stream = lexer::parse_token_streams(&sess, src_from_sf.as_str(), sf.start_pos); + // Parser + let mut p = parser::Parser::new(&sess, stream); + let mut m = p.parse_module(); + m.filename = filepath.display().to_string(); + m.pkg = kclvm_ast::MAIN_PKG.to_string(); + m.name = kclvm_ast::MAIN_PKG.to_string(); + + Ok(m) +} + /// Parse a KCL file to the AST module with the parse session . pub fn parse_file_with_session( sess: ParseSessionRef, @@ -307,30 +342,67 @@ pub fn load_program( Loader::new(sess, paths, opts, module_cache).load_main() } +pub fn load_program_vfs( + sess: ParseSessionRef, + entries: Entries, + opts: Option, + module_cache: Option, +) -> Result { + Loader::new_vfs( + sess, + entries, + opts, + module_cache, + Box::new(SourceMapVfs::new()), + ) + .load_main() +} + pub type KCLModuleCache = Arc>>; struct Loader { sess: ParseSessionRef, - paths: Vec, opts: LoadProgramOptions, missing_pkgs: Vec, module_cache: Option, file_graph: FileGraph, + compile_entries: Entries, + vfs: Box, } impl Loader { + fn new_vfs( + sess: ParseSessionRef, + entries: Entries, + opts: Option, + module_cache: Option>>>, + vfs: Box, + ) -> Self { + Self { + sess, + opts: opts.clone().unwrap_or_default(), + module_cache, + missing_pkgs: Default::default(), + file_graph: FileGraph::default(), + compile_entries: entries, + vfs, + } + } + fn new( sess: ParseSessionRef, paths: &[&str], opts: Option, module_cache: Option>>>, ) -> Self { + let paths: Vec = paths.iter().map(|s| s.to_string()).collect(); Self { sess, - paths: paths.iter().map(|s| s.to_string()).collect(), - opts: opts.unwrap_or_default(), + opts: opts.clone().unwrap_or_default(), module_cache, missing_pkgs: Default::default(), file_graph: FileGraph::default(), + compile_entries: Default::default(), + vfs: Box::new(SourceMapVfs::new()), } } @@ -340,37 +412,38 @@ impl Loader { } fn _load_main(&mut self) -> Result { - let compile_entries = get_compile_entries_from_paths(&self.paths, &self.opts)?; - let workdir = compile_entries.get_root_path().to_string(); + let workdir = self.compile_entries.get_root_path().clone(); + let entries = self.compile_entries.clone(); let mut pkgs = HashMap::new(); let mut pkg_files = Vec::new(); - for entry in compile_entries.iter() { - // Get files from options with root. - // let k_files = self.get_main_files_from_pkg(entry.path(), entry.name())?; - let k_files = entry.get_k_files(); - let maybe_k_codes = entry.get_k_codes(); + for entry in entries.iter() { + let k_files = entry.files.clone(); // Load main package. - for (i, filename) in k_files.iter().enumerate() { + for (i, filepath) in k_files.iter().enumerate() { let mut m = if let Some(module_cache) = self.module_cache.as_ref() { - let m = parse_file_with_session( + let m = parse_file_with_session_vfs( self.sess.clone(), - filename, - maybe_k_codes[i].clone(), + &mut self.vfs, + filepath.to_string(), )?; let mut module_cache_ref = module_cache.write().unwrap(); - module_cache_ref.insert(filename.clone(), m.clone()); + module_cache_ref.insert(filepath.clone(), m.clone()); m } else { - parse_file_with_session(self.sess.clone(), filename, maybe_k_codes[i].clone())? + parse_file_with_session_vfs( + self.sess.clone(), + &mut self.vfs, + filepath.to_string(), + )? }; - fix_rel_import_path(entry.path(), &mut m); + fix_rel_import_path(&entry.modpath, &mut m); pkg_files.push(m); } // Insert an empty vec to determine whether there is a circular import. pkgs.insert(kclvm_ast::MAIN_PKG.to_string(), vec![]); self.load_import_package( - entry.path(), + &entry.modpath, entry.name().to_string(), &mut pkg_files, &mut pkgs, @@ -508,11 +581,9 @@ impl Loader { for stmt in &mut m.body { let pos = stmt.pos().clone(); if let ast::Stmt::Import(ref mut import_spec) = &mut stmt.node { - import_spec.path.node = kclvm_config::vfs::fix_import_path( - pkgroot, - &m.filename, - import_spec.path.node.as_str(), - ); + let pkgpath = PathBuf::from(import_spec.path.node.as_str()); + import_spec.path.node = + pkgpath.abs(Some(pkgroot.to_string()), Some(m.filename.to_string())); import_spec.pkg_name = pkg_name.to_string(); // Load the import package source code and compile. let pkg_info = self.load_package( @@ -809,18 +880,32 @@ impl Loader { fn pkg_exists_in_path(&self, path: String, pkgpath: &str) -> bool { let mut pathbuf = PathBuf::from(path); pkgpath.split('.').for_each(|s| pathbuf.push(s)); - pathbuf.exists() || pathbuf.with_extension(KCL_FILE_EXTENSION).exists() + + let vfs = self.vfs.as_ref(); + + let dir_exist = if !pathbuf.exists_in(vfs) { + pathbuf.write(vfs, None).is_ok() + } else { + true + }; + + let k_file_path = pathbuf.with_extension(KCL_FILE_EXTENSION); + let k_file_exist = if !k_file_path.exists_in(vfs) { + k_file_path.write(vfs, None).is_ok() + } else { + true + }; + + return k_file_exist || dir_exist; } } fn fix_rel_import_path(pkgroot: &str, m: &mut ast::Module) { for stmt in &mut m.body { if let ast::Stmt::Import(ref mut import_spec) = &mut stmt.node { - import_spec.path.node = kclvm_config::vfs::fix_import_path( - pkgroot, - &m.filename, - import_spec.path.node.as_str(), - ); + let pkgpath = PathBuf::from(import_spec.path.node.as_str()); + import_spec.path.node = + pkgpath.abs(Some(pkgroot.to_string()), Some(m.filename.to_string())); } } } diff --git a/kclvm/parser/src/tests.rs b/kclvm/parser/src/tests.rs index 5dfb903e0..7104e0a08 100644 --- a/kclvm/parser/src/tests.rs +++ b/kclvm/parser/src/tests.rs @@ -608,6 +608,7 @@ fn test_import_vendor_by_external_arguments() { } #[test] +#[deprecated] fn test_get_compile_entries_from_paths() { let testpath = PathBuf::from("./src/testdata/multimods") .canonicalize() diff --git a/kclvm/runner/Cargo.toml b/kclvm/runner/Cargo.toml index b46b87edb..426519453 100644 --- a/kclvm/runner/Cargo.toml +++ b/kclvm/runner/Cargo.toml @@ -26,6 +26,7 @@ cc = "1.0" compiler_base_session = {path = "../../compiler_base/session"} compiler_base_macros = "0.0.1" +kclvm-vfs = {path = "../vfs"} kclvm-ast = {path = "../ast"} kclvm-parser = {path = "../parser"} kclvm-compiler = {path = "../compiler"} diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index a297e91ac..1fcebe5f1 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -7,7 +7,7 @@ use kclvm_ast::{ MAIN_PKG, }; use kclvm_driver::{canonicalize_input_files, expand_input_files}; -use kclvm_parser::{load_program, KCLModuleCache, ParseSessionRef}; +use kclvm_parser::{load_program, load_program_vfs, KCLModuleCache, ParseSessionRef}; use kclvm_query::apply_overrides; use kclvm_sema::resolver::{ resolve_program, resolve_program_with_opts, scope::ProgramScope, Options, @@ -16,6 +16,7 @@ use linker::Command; pub use runner::{Artifact, ExecProgramArgs, ExecProgramResult, MapErrorResult}; use runner::{KclLibRunner, KclLibRunnerOptions, ProgramRunner}; use tempfile::tempdir; +use vfs::vfs::KCLVfs; pub mod assembler; pub mod linker; @@ -73,21 +74,31 @@ pub fn exec_program(sess: ParseSessionRef, args: &ExecProgramArgs) -> Result>(); + + vfs::load_local_paths(kcl_paths.clone())?; + let compile_entries = vfs::entry::get_compile_entries_from_paths( + &kcl_paths, + &opts.clone().k_code_list, + opts.clone().package_maps, + opts.clone().work_dir.clone(), + )?; + let module_cache = KCLModuleCache::default(); - let mut program = load_program( + let mut program = load_program_vfs( sess.clone(), - kcl_paths_str.as_slice(), + compile_entries, Some(opts), Some(module_cache), )? .program; + apply_overrides( &mut program, &args.overrides, &[], args.print_override_ast || args.debug > 0, )?; + execute(sess, program, args) } diff --git a/kclvm/vfs/Cargo.toml b/kclvm/vfs/Cargo.toml new file mode 100644 index 000000000..402a5bca0 --- /dev/null +++ b/kclvm/vfs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "kclvm-vfs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +compiler_base_span ={ path = "../../compiler_base/span" } +anyhow = "1.0.81" +parking_lot = "0.12.1" +path-slash = "0.2.1" +regex = "1.10.3" +pathdiff = "0.2.1" +pcre2 = "0.2.6" +lazy_static = "1.4.0" + +kclvm-config = { path = "../config" } +kclvm-utils ={ path = "../utils" } +kclvm-span = {path = "../span"} diff --git a/kclvm/parser/src/entry.rs b/kclvm/vfs/src/entry.rs similarity index 67% rename from kclvm/parser/src/entry.rs rename to kclvm/vfs/src/entry.rs index c607e8ae9..3f8da27b5 100644 --- a/kclvm/parser/src/entry.rs +++ b/kclvm/vfs/src/entry.rs @@ -1,58 +1,31 @@ -use anyhow::Result; -use kclvm_config::modfile::get_pkg_root; +use anyhow::{Ok, Result}; +use std::result::Result as StdResult; + use kclvm_config::modfile::KCL_FILE_SUFFIX; use kclvm_config::path::ModRelativePath; -use kclvm_utils::path::PathPrefix; use kclvm_utils::path::{is_absolute, is_dir, path_exist}; -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::fs; use std::path::Path; -use crate::LoadProgramOptions; +use crate::vfs::{VFSPath, VFS}; + +// use crate::{file_id, vfs, PathPrefix}; +// use crate::vfs::FileId; +// use crate::read_vfs; /// [`Entries`] is a map of package name to package root path for one compilation /// # note /// /// The [`entries`] in [`Entries`] is ordered, and the order of Entrys may affect the result of the compilation. /// The reason why the [`Entries`] is not an [`IndexMap`] is that the [`entries`] is duplicable and ordered. -#[derive(Default, Debug)] +#[derive(Default, Debug, Clone)] pub struct Entries { root_path: String, entries: VecDeque, } impl Entries { - /// [`get_unique_normal_paths_by_name`] will return all the unique normal paths of [`Entry`] with the given name in [`Entries`]. - pub fn get_unique_normal_paths_by_name(&self, name: &str) -> Vec { - let paths = self - .get_unique_paths_by_name(name) - .iter() - .filter(|path| { - // All the paths contains the normal paths and the mod relative paths start with ${KCL_MOD}. - // If the number of 'kcl.mod' paths is 0, except for the mod relative paths start with ${KCL_MOD}, - // then use empty path "" as the default. - !ModRelativePath::new(path.to_string()) - .is_relative_path() - .unwrap_or(false) - }) - .map(|entry| entry.to_string()) - .collect::>(); - paths - } - - /// [`get_unique_paths_by_name`] will return all the unique paths of [`Entry`] with the given name in [`Entries`]. - pub fn get_unique_paths_by_name(&self, name: &str) -> Vec { - let mut paths = self - .entries - .iter() - .filter(|entry| entry.name() == name) - .map(|entry| entry.path().to_string()) - .collect::>(); - paths.sort(); - paths.dedup(); - paths - } - /// [`push_entry`] will push a new [`Entry`] into [`Entries`]. pub fn push_entry(&mut self, entry: Entry) { self.entries.push_back(entry); @@ -120,22 +93,20 @@ impl Entries { } /// [`Entry`] is a package name and package root path pair. -#[derive(Default, Debug)] +#[derive(Default, Debug, Clone)] pub struct Entry { name: String, - path: String, - k_files: Vec, - k_code_lists: Vec>, + pub modpath: String, + pub files: Vec, } impl Entry { /// [`new`] will create a new [`Entry`] with the given name and path. - pub fn new(name: String, path: String) -> Self { + pub fn new(name: String, modpath: String) -> Self { Self { name, - path, - k_files: vec![], - k_code_lists: vec![], + modpath, + files: Vec::new(), } } @@ -145,46 +116,50 @@ impl Entry { } /// [`path`] will return the path of [`Entry`]. - pub fn path(&self) -> &String { - &self.path + pub fn modpath(&self) -> &String { + &self.modpath } /// [`set_path`] will set the path of [`Entry`] to the given path. - pub fn set_path(&mut self, path: String) { - self.path = path; + pub fn set_modpath(&mut self, modpath: String) { + self.modpath = modpath; } /// [`extend_k_files`] will extend the k files of [`Entry`] to the given k file. pub fn extend_k_files(&mut self, k_files: Vec) { - self.k_files.extend(k_files); - } - - /// [`extend_k_files_and_codes`] will extend the k files and k codes of [`Entry`] to the given k file and k code. - pub fn extend_k_files_and_codes( - &mut self, - k_files: Vec, - k_codes: &mut VecDeque, - ) { - for k_file in k_files.iter() { - self.k_code_lists.push(k_codes.pop_front()); - self.k_files.push(k_file.to_string()); - } - } - - /// [`push_k_code`] will push the k code of [`Entry`] to the given k code. - pub fn push_k_code(&mut self, k_code: Option) { - self.k_code_lists.push(k_code); - } - - /// [`get_k_files`] will return the k files of [`Entry`]. - pub fn get_k_files(&self) -> &Vec { - &self.k_files + self.files.extend(k_files); } - /// [`get_k_codes`] will return the k codes of [`Entry`]. - pub fn get_k_codes(&self) -> &Vec> { - &self.k_code_lists - } + // pub fn push_k_file(&mut self, k_file: String) { + // self. + // } + + // [`extend_k_files_and_codes`] will extend the k files and k codes of [`Entry`] to the given k file and k code. + // pub fn extend_k_files_and_codes( + // &mut self, + // k_files: Vec, + // k_codes: &mut VecDeque, + // ) { + // for k_file in k_files.iter() { + // let path = Path::new(k_file); + // // 如果这个文件路径已经带有code了，就不再写入 + // if let Some(code) = k_codes.pop_front().map(|code| code.as_bytes().to_vec()) { + // path.write(Some(code)).unwrap(); + // } else { + // // 如果没有code，就从本地读取并更新 vfs + // if path.exists_and_update_vfs() { + // // 将文件路径写入 vfs + // self.vfiles.push(file_id(k_file.to_string()).unwrap()); + // } + // } + // } + // } + + // pub fn push_k_file_and_code(&mut self, k_file: String, k_code: String) { + // let path = Path::new(&k_file); + // path.write(Some(k_code.as_bytes().to_vec())); + // self.vfiles.push(file_id(k_file.to_string()).unwrap()); + // } } /// [`get_compile_entries_from_paths`] returns all the [`Entries`] for compilation from the given [`file_paths`]. @@ -277,20 +252,48 @@ impl Entry { /// ``` pub fn get_compile_entries_from_paths( file_paths: &[String], - opts: &LoadProgramOptions, + k_code_list: &[String], + external_pkg_maps: HashMap, + work_dir: String, + vfs: &dyn VFS, ) -> Result { if file_paths.is_empty() { return Err(anyhow::anyhow!("No input KCL files or paths")); } let mut result = Entries::default(); - let mut k_code_queue = VecDeque::from(opts.k_code_list.clone()); + let mut k_code_queue = VecDeque::from(k_code_list.to_vec()); + + // 1. find the package root path by `kcl.mod` file, work_dir, or the path of the *.k file. + let mut tmpmodpaths = vec![]; + let mut modpath = work_dir.clone(); + for filepath in file_paths { + let path = ModRelativePath::from(filepath.to_string()); + if !path.is_relative_path()? { + if let Some(root) = get_pkg_root(filepath) { + tmpmodpaths.push(root) + } + } + } + + if tmpmodpaths.len() == 0 { + modpath = "".to_string(); + } + + if tmpmodpaths.len() == 1 && work_dir.is_empty() { + modpath = tmpmodpaths[0].clone(); + } else if !work_dir.is_empty() { + modpath = work_dir.clone(); + } + + result.root_path = modpath.clone(); + for s in file_paths { let path = ModRelativePath::from(s.to_string()); // If the path is a [`ModRelativePath`] with prefix '${:KCL_MOD}', // calculate the real path and the package name. if let Some((pkg_name, pkg_path)) = path.get_root_pkg_name()?.and_then(|name| { - opts.package_maps + external_pkg_maps .get(&name) .map(|pkg_path: &String| (name, pkg_path)) }) { @@ -298,84 +301,63 @@ pub fn get_compile_entries_from_paths( let s = path.canonicalize_by_root_path(pkg_path)?; if let Some(root) = get_pkg_root(&s) { let mut entry: Entry = Entry::new(pkg_name.clone(), root.clone()); - entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(&s, &root, &pkg_name, opts)?, - &mut k_code_queue, - ); + for k_file in + get_main_files_from_pkg_path(&s, &root, &pkg_name, k_code_list)?.iter() + { + let path = Path::new(k_file); + let res = path.write( + vfs, + k_code_queue + .pop_front() + .map(|code| code.as_bytes().to_vec()), + ); + + if res.is_ok() { + entry.files.push(k_file.to_string()); + } + } + result.push_entry(entry); continue; } // If the [`ModRelativePath`] with prefix '${KCL_MOD}' } else if path.is_relative_path()? && path.get_root_pkg_name()?.is_none() { // Push it into `result`, and deal it later. - let mut entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), path.get_path()); - entry.push_k_code(k_code_queue.pop_front()); + let s = path.canonicalize_by_root_path(&modpath)?; + let mut entry: Entry = Entry::new("__main__".to_string(), path.get_path()); + let path = Path::new(&s); + let res = path.write( + vfs, + k_code_queue + .pop_front() + .map(|code| code.as_bytes().to_vec()), + ); + + if res.is_ok() { + entry.files.push(s.to_string()); + } result.push_entry(entry); continue; } else if let Some(root) = get_pkg_root(s) { // If the path is a normal path. - let mut entry: Entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), root.clone()); - entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(s, &root, kclvm_ast::MAIN_PKG, opts)?, - &mut k_code_queue, - ); - result.push_entry(entry); - } - } + let mut entry: Entry = Entry::new("__main__".to_string(), root.clone()); + for k_file in get_main_files_from_pkg_path(&s, &root, "__main__", k_code_list)?.iter() { + let path = Path::new(k_file); + let res = path.write( + vfs, + k_code_queue + .pop_front() + .map(|code| code.as_bytes().to_vec()), + ); - // The main 'kcl.mod' can not be found, the empty path "" will be took by default. - if result - .get_unique_normal_paths_by_name(kclvm_ast::MAIN_PKG) - .is_empty() - { - let mut entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), "".to_string()); - for s in file_paths { - entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(s, "", kclvm_ast::MAIN_PKG, opts)?, - &mut k_code_queue, - ); + if res.is_ok() { + entry.files.push(k_file.to_string()); + } + } + result.push_entry(entry); } - result.push_entry(entry); } - let pkg_root = if result - .get_unique_normal_paths_by_name(kclvm_ast::MAIN_PKG) - .len() - == 1 - && opts.work_dir.is_empty() - { - // If the 'kcl.mod' can be found only once, the package root path will be the path of the 'kcl.mod'. - result - .get_unique_normal_paths_by_name(kclvm_ast::MAIN_PKG) - .get(0) - .unwrap() - .to_string() - } else if !opts.work_dir.is_empty() { - // If the 'kcl.mod' can be found more than once, the package root path will be the 'work_dir'. - if let Some(root_work_dir) = get_pkg_root(&opts.work_dir) { - root_work_dir - } else { - "".to_string() - } - } else { - "".to_string() - }; - result.root_path = pkg_root.clone(); - // Replace the '${KCL_MOD}' of all the paths with package name '__main__'. - result.apply_to_all_entries(|entry| { - let path = ModRelativePath::from(entry.path().to_string()); - if entry.name() == kclvm_ast::MAIN_PKG && path.is_relative_path()? { - entry.set_path(pkg_root.to_string()); - entry.extend_k_files(get_main_files_from_pkg_path( - &path.canonicalize_by_root_path(&pkg_root)?, - &pkg_root, - kclvm_ast::MAIN_PKG, - opts, - )?); - } - Ok(()) - })?; - Ok(result) } @@ -384,7 +366,7 @@ fn get_main_files_from_pkg_path( pkg_path: &str, root: &str, pkg_name: &str, - opts: &LoadProgramOptions, + k_code_list: &[String], ) -> Result> { // fix path let mut path_list = Vec::new(); @@ -401,12 +383,6 @@ fn get_main_files_from_pkg_path( return Err(anyhow::anyhow!("Can not find {} in the path: {}", s, root)); } } - if !root.is_empty() && !is_absolute(s.as_str()) { - let p = std::path::Path::new(s.as_str()); - if let Ok(x) = std::fs::canonicalize(p) { - s = x.adjust_canonicalization(); - } - } path_list.push(s); @@ -416,7 +392,7 @@ fn get_main_files_from_pkg_path( for (i, path) in path_list.iter().enumerate() { // read dir/*.k if is_dir(path) { - if opts.k_code_list.len() > i { + if k_code_list.len() > i { return Err(anyhow::anyhow!("Invalid code list")); } // k_code_list @@ -435,7 +411,7 @@ fn get_main_files_from_pkg_path( // check all file exists for (i, filename) in k_files.iter().enumerate() { - if i < opts.k_code_list.len() { + if i < k_code_list.len() { continue; } @@ -463,9 +439,9 @@ pub fn get_dir_files(dir: &str, is_recursive: bool) -> Result> { let path = Path::new(&path); if path.is_dir() { match fs::read_dir(path) { - Ok(entries) => { + StdResult::Ok(entries) => { for entry in entries { - if let Ok(entry) = entry { + if let StdResult::Ok(entry) = entry { let path = entry.path(); if path.is_dir() && is_recursive { queue.push_back(path.to_string_lossy().to_string()); @@ -502,3 +478,32 @@ fn is_ignored_file(filename: &str) -> bool { || (filename.ends_with("_test.k")) || (filename.starts_with('_')) } + +pub fn get_pkg_root(k_file_path: &str) -> Option { + if k_file_path.is_empty() { + return None; + } + // # search by kcl.mod file + if let StdResult::Ok(module_path) = std::path::Path::new(k_file_path).canonicalize() { + let mut module_path = module_path; + while module_path.exists() { + let kcl_mod_path = module_path.join("kcl.mod"); + if kcl_mod_path.exists() && kcl_mod_path.is_file() { + return Some(module_path.display().to_string()); + } + if let Some(path) = module_path.parent() { + module_path = path.to_path_buf(); + } else { + break; + } + } + } + if k_file_path.ends_with(".k") { + if let StdResult::Ok(path) = std::path::Path::new(k_file_path).canonicalize() { + if let Some(path) = path.parent() { + return Some(path.display().to_string()); + } + } + } + None +} diff --git a/kclvm/vfs/src/lib.rs b/kclvm/vfs/src/lib.rs new file mode 100644 index 000000000..63387479d --- /dev/null +++ b/kclvm/vfs/src/lib.rs @@ -0,0 +1,3 @@ +pub mod entry; +pub mod sourcemap; +pub mod vfs; diff --git a/kclvm/vfs/src/sourcemap.rs b/kclvm/vfs/src/sourcemap.rs new file mode 100644 index 000000000..aafcfaddf --- /dev/null +++ b/kclvm/vfs/src/sourcemap.rs @@ -0,0 +1,62 @@ +use std::{fs, path::PathBuf, sync::Arc}; + +use anyhow::Ok; +use kclvm_span::FilePathMapping; +use parking_lot::RwLock; + +use crate::vfs::VFS; + +pub struct SourceFile { + sf_inner: Arc>, +} + +pub struct SourceMapVfs { + sm_inner: Arc>, +} + +impl SourceMapVfs { + pub fn new() -> Self { + SourceMapVfs { + sm_inner: Arc::new(RwLock::new(kclvm_span::SourceMap::new( + FilePathMapping::empty(), + ))), + } + } +} + +impl VFS for SourceMapVfs { + fn write(&self, path: String, contents: Option>) -> anyhow::Result<()> { + let contents = if let Some(contents) = contents { + contents + } else { + fs::read(&path)? + }; + + let _ = self + .sm_inner + .write() + .new_source_file(PathBuf::from(&path).into(), String::from_utf8(contents)?); + Ok(()) + } + + fn read(&self, path: String) -> anyhow::Result> { + let sf = self + .sm_inner + .read() + .get_source_file(&PathBuf::from(&path).into()); + + if sf.is_none() { + return Err(anyhow::anyhow!("Source file {} not found", path)); + } + + let binding = sf.unwrap(); + let src_from_sf = match binding.src.as_ref() { + Some(src) => src, + None => { + return Err(anyhow::anyhow!("Source file {} not found", path)); + } + }; + + Ok(src_from_sf.to_string().into_bytes()) + } +} diff --git a/kclvm/vfs/src/vfs.rs b/kclvm/vfs/src/vfs.rs new file mode 100644 index 000000000..a172437b8 --- /dev/null +++ b/kclvm/vfs/src/vfs.rs @@ -0,0 +1,143 @@ +use anyhow::anyhow; +use anyhow::Ok; +use anyhow::Result; +use parking_lot::RwLock; + +use std::fs; +use std::panic; +use std::panic::AssertUnwindSafe; +use std::{ + collections::HashMap, + hash::Hash, + path::{Path, PathBuf}, + sync::Arc, +}; + +use crate::sourcemap::SourceMapVfs; +use lazy_static::lazy_static; +use std::sync::Mutex; + +pub trait VFS { + // TODO: More actions to be get SourceFile directly + fn write(&self, path: String, contents: Option>) -> Result<()>; + fn read(&self, path: String) -> Result>; +} + +pub trait VFSPath { + fn write(&self, vfs: &dyn VFS, contents: Option>) -> Result<()>; + fn read(&self, vfs: &dyn VFS) -> Result>; + fn exists_in(&self, vfs: &dyn VFS) -> bool; + fn standardized(&self) -> String; +} + +impl

VFSPath for P +where + P: AsRef, +{ + fn write(&self, vfs: &dyn VFS, contents: Option>) -> Result<()> { + vfs.write(self.as_ref().display().to_string(), contents) + } + + fn read(&self, vfs: &dyn VFS) -> Result> { + vfs.read(self.as_ref().display().to_string()) + } + + fn exists_in(&self, vfs: &dyn VFS) -> bool { + vfs.read(self.as_ref().display().to_string()).is_ok() + } + + fn standardized(&self) -> String { + use regex::Regex; + let re = Regex::new(r"([a-zA-Z]):\\").unwrap(); + let path = re + .replace_all(&self.as_ref().display().to_string(), "$1/") + .to_string(); + path.replace("\\", "/").replace("//", "/") + } +} + +pub trait PkgPath { + fn abs(&self, root_path: Option, start_path: Option) -> String; + fn to_pkgpath(&self) -> String; +} + +impl

PkgPath for P +where + P: AsRef, +{ + fn abs(&self, root_path: Option, start_path: Option) -> String { + // relpath: import .sub + // FixImportPath(root, "path/to/app/file.k", ".sub") => path.to.app.sub + // FixImportPath(root, "path/to/app/file.k", "..sub") => path.to.sub + // FixImportPath(root, "path/to/app/file.k", "...sub") => path.sub + // FixImportPath(root, "path/to/app/file.k", "....sub") => sub + // FixImportPath(root, "path/to/app/file.k", ".....sub") => "" + // + // abspath: import path.to.sub + // FixImportPath(root, "path/to/app/file.k", "path.to.sub") => path.to.sub + + let import_path = self.as_ref().display().to_string(); + let root = root_path.unwrap_or("".to_string()); + let filepath = start_path.unwrap_or("".to_string()); + + if !import_path.starts_with('.') { + return import_path.to_string(); + } + + // Filepath to pkgpath + let pkgpath = { + let base = Path::new(&root); + let dirpath = std::path::Path::new(&filepath).parent().unwrap(); + + let pkgpath = if let Some(x) = pathdiff::diff_paths(dirpath, base) { + x.to_str().unwrap().to_string() + } else { + dirpath.to_str().unwrap().to_string() + }; + + let pkgpath = pkgpath.replace(['/', '\\'], "."); + pkgpath.trim_end_matches('.').to_string() + }; + + let mut leading_dot_count = import_path.len(); + for (i, c) in import_path.chars().enumerate() { + if c != '.' { + leading_dot_count = i; + break; + } + } + + // The pkgpath is the current root path + if pkgpath.is_empty() { + if leading_dot_count <= 1 { + return import_path.trim_matches('.').to_string(); + } else { + return "".to_string(); + } + } + + if leading_dot_count == 1 { + return pkgpath + &import_path; + } + + let ss = pkgpath.split('.').collect::>(); + + if (leading_dot_count - 1) < ss.len() { + let prefix = ss[..(ss.len() - leading_dot_count + 1)].join("."); + let suffix = import_path[leading_dot_count..].to_string(); + + return format!("{}.{}", prefix, suffix); + } + + if leading_dot_count - 1 == ss.len() { + return import_path[leading_dot_count..].to_string(); + } + + "".to_string() + } + + fn to_pkgpath(&self) -> String { + let std_path = self.standardized(); + return std_path.replace("/", "."); + } +} diff --git a/kclvm/vfs/test.txt b/kclvm/vfs/test.txt new file mode 100644 index 000000000..7c4a013e5 --- /dev/null +++ b/kclvm/vfs/test.txt @@ -0,0 +1 @@ +aaa \ No newline at end of file diff --git a/main.k b/main.k new file mode 100644 index 000000000..9f5d7a3c1 --- /dev/null +++ b/main.k @@ -0,0 +1,3 @@ +import sub.main + +a = main.aaa \ No newline at end of file diff --git a/sub/main.k b/sub/main.k new file mode 100644 index 000000000..560e55c65 --- /dev/null +++ b/sub/main.k @@ -0,0 +1 @@ +aaa = "aaa" \ No newline at end of file