From 017610bcf0e67c7a3130b919905971e5d49a0cc5 Mon Sep 17 00:00:00 2001 From: Peefy Date: Wed, 22 May 2024 18:32:30 +0800 Subject: [PATCH] feat: treat kcl.mod file as a compile unit (#1348) Signed-off-by: peefy --- kclvm/config/src/cache.rs | 2 - kclvm/config/src/modfile.rs | 225 +++++++++++++++++++++--------- kclvm/config/src/testdata/kcl.mod | 19 ++- kclvm/config/src/vfs.rs | 12 +- kclvm/driver/src/lib.rs | 102 +++++++++----- 5 files changed, 247 insertions(+), 113 deletions(-) diff --git a/kclvm/config/src/cache.rs b/kclvm/config/src/cache.rs index 6da1f8731..adf1b5bc5 100644 --- a/kclvm/config/src/cache.rs +++ b/kclvm/config/src/cache.rs @@ -23,7 +23,6 @@ pub const KCL_CACHE_PATH_ENV_VAR: &str = "KCL_CACHE_PATH"; pub type CacheInfo = Vec; pub type Cache = HashMap; -#[allow(dead_code)] pub struct CacheOption { cache_dir: String, } @@ -150,7 +149,6 @@ fn get_cache_dir(root: &str, cache_dir: Option<&str>) -> String { } #[inline] -#[allow(dead_code)] fn get_cache_filename(root: &str, target: &str, pkgpath: &str, cache_dir: Option<&str>) -> String { let cache_dir = cache_dir.unwrap_or(DEFAULT_CACHE_DIR); let root = std::env::var(KCL_CACHE_PATH_ENV_VAR).unwrap_or(root.to_string()); diff --git a/kclvm/config/src/modfile.rs b/kclvm/config/src/modfile.rs index 2f3a13d84..3d764fd2e 100644 --- a/kclvm/config/src/modfile.rs +++ b/kclvm/config/src/modfile.rs @@ -2,13 +2,19 @@ use anyhow::Result; use kclvm_utils::path::PathPrefix; -use serde::Deserialize; -use std::{env, fs, io::Read, path::PathBuf}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + env, fs, + io::Read, + path::{Path, PathBuf}, +}; use toml; use crate::path::ModRelativePath; pub const KCL_MOD_FILE: &str = "kcl.mod"; +pub const KCL_MOD_LOCK_FILE: &str = "kcl.mod.lock"; pub const KCL_FILE_SUFFIX: &str = ".k"; pub const KCL_FILE_EXTENSION: &str = "k"; pub const KCL_MOD_PATH_ENV: &str = "${KCL_MOD}"; @@ -16,6 +22,113 @@ pub const KCL_PKG_PATH: &str = "KCL_PKG_PATH"; pub const DEFAULT_KCL_HOME: &str = ".kcl"; pub const DEFAULT_KPM_SUBDIR: &str = "kpm"; +/// ModFile is kcl package file 'kcl.mod'. +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct ModFile { + pub package: Option, + pub profile: Option, + pub dependencies: Option, +} + +/// Package is the kcl package section of 'kcl.mod'. +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct Package { + /// The name of the package. + pub name: Option, + /// The kcl compiler version + pub edition: Option, + /// The version of the package. + pub version: Option, + /// Description denotes the description of the package. + pub description: Option, + /// Exclude denote the files to include when publishing. + pub include: Option>, + /// Exclude denote the files to exclude when publishing. + pub exclude: Option>, +} + +/// Profile is the profile section of 'kcl.mod'. +/// It is used to specify the compilation options of the current package. +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct Profile { + /// A list of entry-point files. + pub entries: Option>, + /// Flag that, when true, disables the emission of the special 'none' value in the output. + pub disable_none: Option, + /// Flag that, when true, ensures keys in maps are sorted. + pub sort_keys: Option, + /// A list of attribute selectors for conditional compilation. + pub selectors: Option>, + /// A list of override paths. + pub overrides: Option>, + /// A list of additional options for the KCL compiler. + pub options: Option>, +} + +/// A map of package names to their respective dependency specifications. +pub type Dependencies = HashMap; + +/// Dependency represents a single dependency for a package, which may come in different forms +/// such as version, Git repository, OCI repository, or a local path. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(untagged)] +pub enum Dependency { + /// Specifies a version dependency, e.g., "1.0.0". + Version(String), + /// Specifies a Git source dependency. + Git(GitSource), + /// Specifies an OCI (Open Container Initiative) image source dependency. + Oci(OciSource), + /// Specifies a local path dependency. + Local(LocalSource), +} + +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct GitSource { + /// The URL of the Git repository. + pub git: String, + /// An optional branch name within the Git repository. + pub branch: Option, + /// An optional commit hash to check out from the Git repository. + pub commit: Option, + /// An optional tag name to check out from the Git repository. + pub tag: Option, + /// An optional version specification associated with Git source. + pub version: Option, +} + +/// Defines an OCI package as a source for a dependency. +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct OciSource { + // The URI of the OCI repository. + pub oci: String, + /// An optional tag of the OCI package in the registry. + pub tag: Option, +} + +/// Defines a local filesystem path as a source for a dependency. +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct LocalSource { + /// The path to the local directory or file. + pub path: String, +} + +impl ModFile { + #[inline] + pub fn get_entries(&self) -> Option> { + self.profile.as_ref().map(|p| p.entries.clone()).flatten() + } +} + +/// Load kcl mod file from path +pub fn load_mod_file>(path: P) -> Result { + let file_path = path.as_ref().join(KCL_MOD_FILE); + let mut file = std::fs::File::open(file_path)?; + let mut buffer: Vec = vec![]; + file.read_to_end(&mut buffer)?; + toml::from_slice(buffer.as_slice()).map_err(|e| anyhow::anyhow!(e)) +} + /// Get the path holding the external kcl package. /// From the environment variable KCL_PKG_PATH. /// If `KCL_PKG_PATH` is not present, then the user root string is returned. @@ -51,39 +164,16 @@ pub fn create_default_vendor_home() -> Option { match kpm_home.canonicalize() { Ok(path) => return Some(path.display().to_string()), Err(_) => match fs::create_dir_all(kpm_home.clone()) { - Ok(_) => return Some(kpm_home.canonicalize().unwrap().display().to_string()), + Ok(_) => match kpm_home.canonicalize() { + Ok(p) => Some(p.display().to_string()), + Err(_) => None, + }, Err(_) => None, }, } } -#[allow(dead_code)] -#[derive(Default, Deserialize)] -pub struct KCLModFile { - pub root: Option, - pub root_pkg: Option, - pub build: Option, - pub expected: Option, -} - -#[allow(dead_code)] -#[derive(Default, Deserialize)] -pub struct KCLModFileBuildSection { - pub enable_pkg_cache: Option, - pub cached_pkg_prefix: Option, - pub target: Option, -} - -#[allow(dead_code)] -#[derive(Default, Deserialize)] -pub struct KCLModFileExpectedSection { - pub min_build_time: Option, - pub max_build_time: Option, - pub kclvm_version: Option, - pub kcl_plugin_version: Option, - pub global_version: Option, -} - +/// Get package root path from input file paths and workdir. pub fn get_pkg_root_from_paths(file_paths: &[String], workdir: String) -> Result { if file_paths.is_empty() { return Err("No input KCL files or paths".to_string()); @@ -114,6 +204,7 @@ pub fn get_pkg_root_from_paths(file_paths: &[String], workdir: String) -> Result } } +/// Get package root path from the single input file path. pub fn get_pkg_root(k_file_path: &str) -> Option { if k_file_path.is_empty() { return None; @@ -143,17 +234,6 @@ pub fn get_pkg_root(k_file_path: &str) -> Option { None } -pub fn load_mod_file(root: &str) -> KCLModFile { - let k_mod_file_path = std::path::Path::new(root).join(KCL_MOD_FILE); - if !k_mod_file_path.exists() { - return KCLModFile::default(); - } - let mut file = std::fs::File::open(k_mod_file_path.to_str().unwrap()).unwrap(); - let mut buffer: Vec = vec![]; - file.read_to_end(&mut buffer).unwrap(); - toml::from_slice(buffer.as_slice()).unwrap() -} - #[cfg(test)] mod modfile_test { use crate::modfile::*; @@ -190,37 +270,48 @@ mod modfile_test { #[test] fn test_load_mod_file() { - let kcl_mod = load_mod_file(TEST_ROOT); - assert!(kcl_mod.build.as_ref().unwrap().enable_pkg_cache.unwrap()); + let kcl_mod = load_mod_file(TEST_ROOT).unwrap(); + assert_eq!( + kcl_mod.package.as_ref().unwrap().name.as_ref().unwrap(), + "test_add_deps" + ); + assert_eq!( + kcl_mod.package.as_ref().unwrap().version.as_ref().unwrap(), + "0.0.1" + ); + assert_eq!( + kcl_mod.package.as_ref().unwrap().edition.as_ref().unwrap(), + "0.0.1" + ); + assert_eq!( + kcl_mod.profile.as_ref().unwrap().entries.as_ref().unwrap(), + &vec!["main.k".to_string()] + ); + assert_eq!( + kcl_mod.dependencies.as_ref().unwrap().get("pkg0"), + Some(&Dependency::Git(GitSource { + git: "test_url".to_string(), + tag: Some("test_tag".to_string()), + ..Default::default() + })) + ); assert_eq!( - kcl_mod - .build - .as_ref() - .unwrap() - .cached_pkg_prefix - .as_ref() - .unwrap(), - "pkg.path" + kcl_mod.dependencies.as_ref().unwrap().get("pkg1"), + Some(&Dependency::Version("oci_tag1".to_string())) ); assert_eq!( - kcl_mod - .expected - .as_ref() - .unwrap() - .kclvm_version - .as_ref() - .unwrap(), - "v0.3.0" + kcl_mod.dependencies.as_ref().unwrap().get("pkg2"), + Some(&Dependency::Oci(OciSource { + oci: "oci://ghcr.io/kcl-lang/helloworld".to_string(), + tag: Some("0.1.1".to_string()), + ..Default::default() + })) ); assert_eq!( - kcl_mod - .expected - .as_ref() - .unwrap() - .kcl_plugin_version - .as_ref() - .unwrap(), - "v0.2.0" + kcl_mod.dependencies.as_ref().unwrap().get("pkg3"), + Some(&Dependency::Local(LocalSource { + path: "../pkg".to_string(), + })) ); } } diff --git a/kclvm/config/src/testdata/kcl.mod b/kclvm/config/src/testdata/kcl.mod index 3be7f7e6d..156f9a3ce 100644 --- a/kclvm/config/src/testdata/kcl.mod +++ b/kclvm/config/src/testdata/kcl.mod @@ -1,6 +1,13 @@ -[build] -enable_pkg_cache=true -cached_pkg_prefix="pkg.path" -[expected] -kclvm_version="v0.3.0" -kcl_plugin_version="v0.2.0" +[package] +name = "test_add_deps" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +pkg0 = { git = "test_url", tag = "test_tag" } +pkg1 = "oci_tag1" +pkg2 = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.1" } +pkg3 = { path = "../pkg"} + +[profile] +entries = ["main.k"] diff --git a/kclvm/config/src/vfs.rs b/kclvm/config/src/vfs.rs index 25018222a..769a1a158 100644 --- a/kclvm/config/src/vfs.rs +++ b/kclvm/config/src/vfs.rs @@ -29,14 +29,14 @@ pub fn is_rel_pkgpath(pkgpath: &str) -> bool { pub fn fix_import_path(root: &str, filepath: &str, import_path: &str) -> String { // relpath: import .sub - // FixImportPath(root, "path/to/app/file.k", ".sub") => path.to.app.sub - // FixImportPath(root, "path/to/app/file.k", "..sub") => path.to.sub - // FixImportPath(root, "path/to/app/file.k", "...sub") => path.sub - // FixImportPath(root, "path/to/app/file.k", "....sub") => sub - // FixImportPath(root, "path/to/app/file.k", ".....sub") => "" + // fix_import_path(root, "path/to/app/file.k", ".sub") => path.to.app.sub + // fix_import_path(root, "path/to/app/file.k", "..sub") => path.to.sub + // fix_import_path(root, "path/to/app/file.k", "...sub") => path.sub + // fix_import_path(root, "path/to/app/file.k", "....sub") => sub + // fix_import_path(root, "path/to/app/file.k", ".....sub") => "" // // abspath: import path.to.sub - // FixImportPath(root, "path/to/app/file.k", "path.to.sub") => path.to.sub + // fix_import_path(root, "path/to/app/file.k", "path.to.sub") => path.to.sub if !import_path.starts_with('.') { return import_path.to_string(); diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index 26d96719d..d8bee4063 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -8,7 +8,10 @@ mod tests; use glob::glob; use kclvm_config::{ - modfile::{get_pkg_root, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_PATH_ENV}, + modfile::{ + get_pkg_root, load_mod_file, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE, + KCL_MOD_PATH_ENV, + }, path::ModRelativePath, settings::{build_settings_pathbuf, DEFAULT_SETTING_FILE}, }; @@ -25,7 +28,7 @@ use std::{ }; use walkdir::WalkDir; -/// Expand the file pattern to a list of files. +/// Expand the single file pattern to a list of files. pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?; let mut matched_files = vec![]; @@ -37,6 +40,7 @@ pub fn expand_if_file_pattern(file_pattern: String) -> Result, Strin Ok(matched_files) } +/// Expand input kcl files with the file patterns. pub fn expand_input_files(k_files: &[String]) -> Vec { let mut res = vec![]; for file in k_files { @@ -124,34 +128,28 @@ pub fn canonicalize_input_files( Ok(kcl_paths) } -/// Get compile uint(files and options) from a single file +/// Get compile uint(files and options) from a single file input. +/// 1. Lookup entry files in kcl.yaml +/// 2. Lookup entry files in kcl.mod +/// 3. If not found, consider the path or folder where the file is +/// located as the compilation entry point pub fn lookup_compile_unit( file: &str, load_pkg: bool, ) -> (Vec, Option) { - let compiled_file: String = file.to_string(); match lookup_compile_unit_path(file) { - Ok(dir) => { + Ok(CompileUnit::SettingFile(dir)) => { let settings_files = lookup_setting_files(&dir); let files = if settings_files.is_empty() { vec![file] } else { vec![] }; - let settings_files = settings_files.iter().map(|f| f.to_str().unwrap()).collect(); match build_settings_pathbuf(&files, Some(settings_files), None) { Ok(setting_buf) => { let setting = setting_buf.settings(); - let files = if let Some(cli_configs) = setting.clone().kcl_cli_configs { - let mut k_filename_list = cli_configs.files.unwrap_or_default(); - if k_filename_list.is_empty() { - k_filename_list = cli_configs.file.unwrap_or_default(); - } - k_filename_list - } else { - vec![] - }; + let files = setting.input(); let work_dir = setting_buf .path() @@ -166,7 +164,7 @@ pub fn lookup_compile_unit( match canonicalize_input_files(&files, work_dir, true) { Ok(kcl_paths) => { // 1. find the kcl.mod path - let _ = fill_pkg_maps_for_k_file(compiled_file.into(), &mut load_opt); + let _ = fill_pkg_maps_for_k_file(file.into(), &mut load_opt); (kcl_paths, Some(load_opt)) } Err(_) => (vec![file.to_string()], None), @@ -175,9 +173,41 @@ pub fn lookup_compile_unit( Err(_) => (vec![file.to_string()], None), } } - Err(_) => { + Ok(CompileUnit::ModFile(dir)) => match load_mod_file(&dir) { + Ok(mod_file) => { + let mut load_opt = kclvm_parser::LoadProgramOptions::default(); + let _ = fill_pkg_maps_for_k_file(file.into(), &mut load_opt); + if let Some(files) = mod_file.get_entries() { + let work_dir = dir.to_string_lossy().to_string(); + load_opt.work_dir = work_dir.clone(); + match canonicalize_input_files(&files, work_dir, true) { + Ok(kcl_paths) => { + let _ = fill_pkg_maps_for_k_file(file.into(), &mut load_opt); + (kcl_paths, Some(load_opt)) + } + Err(_) => (vec![file.to_string()], None), + } + } else { + if load_pkg { + let path = Path::new(file); + if let Some(ext) = path.extension() { + if ext == KCL_FILE_EXTENSION && path.is_file() { + if let Some(parent) = path.parent() { + if let Ok(files) = get_kcl_files(parent, false) { + return (files, Some(load_opt)); + } + } + } + } + } + (vec![file.to_string()], Some(load_opt)) + } + } + Err(_) => (vec![file.to_string()], None), + }, + Ok(CompileUnit::NotFound) | Err(_) => { let mut load_opt = kclvm_parser::LoadProgramOptions::default(); - let _ = fill_pkg_maps_for_k_file(compiled_file.into(), &mut load_opt); + let _ = fill_pkg_maps_for_k_file(file.into(), &mut load_opt); if load_pkg { let path = Path::new(file); @@ -196,6 +226,7 @@ pub fn lookup_compile_unit( } } +/// Lookup default setting files e.g. kcl.yaml pub fn lookup_setting_files(dir: &Path) -> Vec { let mut settings = vec![]; if let Ok(p) = lookup_kcl_yaml(dir) { @@ -204,7 +235,7 @@ pub fn lookup_setting_files(dir: &Path) -> Vec { settings } -pub fn lookup_kcl_yaml(dir: &Path) -> io::Result { +fn lookup_kcl_yaml(dir: &Path) -> io::Result { let mut path = dir.to_path_buf(); path.push(DEFAULT_SETTING_FILE); if path.is_file() { @@ -217,6 +248,15 @@ pub fn lookup_kcl_yaml(dir: &Path) -> io::Result { } } +/// CompileUnit is the kcl program default entries that are defined +/// in the config files. +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum CompileUnit { + SettingFile(PathBuf), + ModFile(PathBuf), + NotFound, +} + /// For the KCL project, some definitions may be introduced through multi-file /// compilation (kcl.yaml). This function is used to start from a single file and try /// to find a `compile unit` that contains all definitions @@ -231,31 +271,29 @@ pub fn lookup_kcl_yaml(dir: &Path) -> io::Result { /// | +-- prod /// | | +-- main.k /// | | +-- kcl.yaml -/// | | +-- stack.yaml /// | +-- test /// | | +-- main.k /// | | +-- kcl.yaml -/// | | +-- stack.yaml -/// | +-- project.yaml +/// | +-- kcl.mod /// /// If the input file is project/prod/main.k or project/test/main.k, it will return /// Path("project/prod") or Path("project/test") -pub fn lookup_compile_unit_path(file: &str) -> io::Result { +pub fn lookup_compile_unit_path(file: &str) -> io::Result { let path = PathBuf::from(file); let current_dir_path = path.as_path().parent().unwrap(); - let entrys = read_dir(current_dir_path)?; - for entry in entrys { + let entries = read_dir(current_dir_path)?; + for entry in entries { let entry = entry?; + // The entry priority of `kcl.yaml`` is higher than that of `kcl.mod`. if entry.file_name() == *DEFAULT_SETTING_FILE { - // If find "kcl.yaml", the input file is in a stack, return the - // path of this stack - return Ok(PathBuf::from(current_dir_path)); + // If find "kcl.yaml", the input file is in a compile stack, return the + // path of this compile stack + return Ok(CompileUnit::SettingFile(PathBuf::from(current_dir_path))); + } else if entry.file_name() == *KCL_MOD_FILE { + return Ok(CompileUnit::ModFile(PathBuf::from(current_dir_path))); } } - Err(io::Error::new( - ErrorKind::NotFound, - "Ran out of places to find kcl.yaml", - )) + Ok(CompileUnit::NotFound) } /// Get kcl files from path.