diff --git a/kclvm/api/src/capi_test.rs b/kclvm/api/src/capi_test.rs index 0eba6d6c3..be5af64c3 100644 --- a/kclvm/api/src/capi_test.rs +++ b/kclvm/api/src/capi_test.rs @@ -161,6 +161,20 @@ fn test_c_api_rename_code() { ); } +#[test] +fn test_c_api_testing() { + test_c_api::( + "KclvmService.Test", + "test.json", + "test.response.json", + |r| { + for i in &mut r.info { + i.duration = 0; + } + }, + ); +} + fn test_c_api_without_wrapper(svc_name: &str, input: &str, output: &str) where A: Message + DeserializeOwned, @@ -190,6 +204,7 @@ where let mut result = R::decode(result.to_bytes()).unwrap(); let result_json = serde_json::to_string(&result).unwrap(); + let except_result_path = Path::new(TEST_DATA_PATH).join(output); let except_result_json = fs::read_to_string(&except_result_path).unwrap_or_else(|_| { panic!( diff --git a/kclvm/api/src/service/capi.rs b/kclvm/api/src/service/capi.rs index 66975c76d..fba6dce72 100644 --- a/kclvm/api/src/service/capi.rs +++ b/kclvm/api/src/service/capi.rs @@ -117,6 +117,7 @@ pub(crate) fn kclvm_get_service_fn_ptr_by_name(name: &str) -> u64 { "KclvmService.LoadSettingsFiles" => load_settings_files as *const () as u64, "KclvmService.Rename" => rename as *const () as u64, "KclvmService.RenameCode" => rename_code as *const () as u64, + "KclvmService.Test" => test as *const () as u64, _ => panic!("unknown method name : {name}"), } } @@ -239,3 +240,8 @@ pub(crate) fn rename(serv: *mut kclvm_service, args: *const c_char) -> *const c_ pub(crate) fn rename_code(serv: *mut kclvm_service, args: *const c_char) -> *const c_char { call!(serv, args, RenameCodeArgs, rename_code) } + +/// Service for the testing tool. +pub(crate) fn test(serv: *mut kclvm_service, args: *const c_char) -> *const c_char { + call!(serv, args, TestArgs, test) +} diff --git a/kclvm/api/src/service/service_impl.rs b/kclvm/api/src/service/service_impl.rs index b08755ab1..85bcc8056 100644 --- a/kclvm/api/src/service/service_impl.rs +++ b/kclvm/api/src/service/service_impl.rs @@ -14,6 +14,8 @@ use kclvm_query::override_file; use kclvm_runner::exec_program; use kclvm_tools::format::{format, format_source, FormatOptions}; use kclvm_tools::lint::lint_files; +use kclvm_tools::testing; +use kclvm_tools::testing::TestRun; use kclvm_tools::vet::validator::validate; use kclvm_tools::vet::validator::LoaderKind; use kclvm_tools::vet::validator::ValidateOption; @@ -515,4 +517,58 @@ impl KclvmServiceImpl { changed_codes: source_codes, }) } + + /// Service for the testing tool. + /// + /// # Examples + /// + /// ``` + /// use kclvm_api::service::service_impl::KclvmServiceImpl; + /// use kclvm_api::gpyrpc::*; + /// + /// let serv = KclvmServiceImpl::default(); + /// let result = serv.test(&TestArgs { + /// pkg_list: vec!["./src/testdata/testing/module/...".to_string()], + /// ..TestArgs::default() + /// }).unwrap(); + /// assert_eq!(result.info.len(), 2); + /// // Passed case + /// assert!(result.info[0].error.is_empty()); + /// // Failed case + /// assert!(result.info[1].error.is_empty()); + /// ``` + pub fn test(&self, args: &TestArgs) -> anyhow::Result { + let mut result = TestResult::default(); + let exec_args = match &args.exec_args { + Some(exec_args) => { + let args_json = serde_json::to_string(exec_args)?; + kclvm_runner::ExecProgramArgs::from_str(args_json.as_str()) + } + None => kclvm_runner::ExecProgramArgs::default(), + }; + let opts = testing::TestOptions { + exec_args, + run_regexp: args.run_regexp.clone(), + fail_fast: args.fail_fast, + }; + for pkg in &args.pkg_list { + let suites = testing::load_test_suites(pkg, &opts)?; + for suite in &suites { + let suite_result = suite.run(&opts)?; + for (name, info) in &suite_result.info { + result.info.push(TestCaseInfo { + name: name.clone(), + error: info + .error + .as_ref() + .map(|e| e.to_string()) + .unwrap_or_default(), + duration: info.duration.as_micros() as u64, + log_message: info.log_message.clone(), + }) + } + } + } + Ok(result) + } } diff --git a/kclvm/api/src/testdata/test.json b/kclvm/api/src/testdata/test.json new file mode 100644 index 000000000..8ab1bd206 --- /dev/null +++ b/kclvm/api/src/testdata/test.json @@ -0,0 +1,3 @@ +{ + "pkg_list": ["./src/testdata/testing/module/..."] +} diff --git a/kclvm/api/src/testdata/test.response.json b/kclvm/api/src/testdata/test.response.json new file mode 100644 index 000000000..969db3e7f --- /dev/null +++ b/kclvm/api/src/testdata/test.response.json @@ -0,0 +1,14 @@ +{ + "info": [ + { + "name": "test_func_0", + "error": "", + "log_message": "" + }, + { + "name": "test_func_1", + "error": "", + "log_message": "" + } + ] +} \ No newline at end of file diff --git a/kclvm/api/src/testdata/testing/module/kcl.mod b/kclvm/api/src/testdata/testing/module/kcl.mod new file mode 100644 index 000000000..35d888aa7 --- /dev/null +++ b/kclvm/api/src/testdata/testing/module/kcl.mod @@ -0,0 +1,3 @@ +[package] +name = "test_data" + diff --git a/kclvm/api/src/testdata/testing/module/pkg/func.k b/kclvm/api/src/testdata/testing/module/pkg/func.k new file mode 100644 index 000000000..26df9cf5a --- /dev/null +++ b/kclvm/api/src/testdata/testing/module/pkg/func.k @@ -0,0 +1,3 @@ +func = lambda x { + x +} diff --git a/kclvm/api/src/testdata/testing/module/pkg/func_test.k b/kclvm/api/src/testdata/testing/module/pkg/func_test.k new file mode 100644 index 000000000..b02295046 --- /dev/null +++ b/kclvm/api/src/testdata/testing/module/pkg/func_test.k @@ -0,0 +1,7 @@ +test_func_0 = lambda { + assert func("a") == "a" +} + +test_func_1 = lambda { + assert func("b") == "b" +} diff --git a/kclvm/spec/gpyrpc/gpyrpc.proto b/kclvm/spec/gpyrpc/gpyrpc.proto index 1733de9d6..56c41996a 100644 --- a/kclvm/spec/gpyrpc/gpyrpc.proto +++ b/kclvm/spec/gpyrpc/gpyrpc.proto @@ -87,6 +87,8 @@ service KclvmService { rpc Rename(Rename_Args) returns(Rename_Result); rpc RenameCode(RenameCode_Args) returns(RenameCode_Result); + + rpc Test(Test_Args) returns (Test_Result); } message Ping_Args { @@ -324,6 +326,29 @@ message RenameCode_Result { map changed_codes = 1; // the changed code. a : map } +// --------------------------------------------------------------------------------- +// Test API +// Test KCL packages with test arguments +// --------------------------------------------------------------------------------- + +message Test_Args { + ExecProgram_Args exec_args = 1; // This field stores the execution program arguments. + repeated string pkg_list = 2; // The package path list to be tested e.g., "./...", "/path/to/package/", "/path/to/package/..." + string run_regexp = 3; // This field stores a regular expression for filtering tests to run. + bool fail_fast = 4; // This field determines whether the test run should stop on the first failure. +} + +message Test_Result { + repeated TestCaseInfo info = 2; +} + +message TestCaseInfo { + string name = 1; // Test case name + string error = 2; + uint64 duration = 3; // Number of whole microseconds in the duration. + string log_message = 4; +} + // ---------------------------------------------------------------------------- // KCL Type Structure // ---------------------------------------------------------------------------- diff --git a/kclvm/tools/src/testing/mod.rs b/kclvm/tools/src/testing/mod.rs index 82e5a329e..88aa328fa 100644 --- a/kclvm/tools/src/testing/mod.rs +++ b/kclvm/tools/src/testing/mod.rs @@ -31,8 +31,6 @@ pub trait TestRun { /// Represents the result of a test. #[derive(Debug, Default)] pub struct TestResult { - /// This field stores the log message of the test. - pub log_message: String, /// This field stores test case information in an [IndexMap], where the key is a [String] and the value is a [TestCaseInfo] struct. pub info: IndexMap, } @@ -40,6 +38,8 @@ pub struct TestResult { /// Represents information about a test case. #[derive(Debug, Default)] pub struct TestCaseInfo { + /// This field stores the log message of the test. + pub log_message: String, /// This field stores the error associated with the test case, if any. pub error: Option, /// This field stores the duration of the test case. diff --git a/kclvm/tools/src/testing/suite.rs b/kclvm/tools/src/testing/suite.rs index e763b1309..c56229e5c 100644 --- a/kclvm/tools/src/testing/suite.rs +++ b/kclvm/tools/src/testing/suite.rs @@ -80,12 +80,11 @@ impl TestRun for TestSuite { result.info.insert( name.clone(), TestCaseInfo { + log_message: exec_result.log_message.clone(), duration: Instant::now() - start, error, }, ); - // Add the log message to the result. - result.log_message += &exec_result.log_message; if fail_fast { break; }