From ec3042905a42e1af881641e06cb3cabbdc78f4d8 Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 7 Feb 2024 17:34:25 +0800 Subject: [PATCH] feat: add build-in method 'file.read' to load file Signed-off-by: zongz --- kclvm/runtime/src/_kclvm.bc | Bin 13592 -> 13628 bytes kclvm/runtime/src/_kclvm.h | 4 +- kclvm/runtime/src/_kclvm.ll | 4 +- kclvm/runtime/src/_kclvm.rs | 3 +- kclvm/runtime/src/_kclvm_addr.rs | 3 +- kclvm/runtime/src/_kclvm_api_spec.rs | 6 ++- kclvm/runtime/src/file/mod.rs | 28 ++++++++++++++ kclvm/runtime/src/lib.rs | 3 ++ kclvm/sema/src/builtin/system_module.rs | 36 +++++++++++++++++- test/grammar/builtins/file/load_json/main.k | 8 ++++ .../builtins/file/load_json/stdout.golden | 13 +++++++ .../grammar/builtins/file/load_json/test.json | 5 +++ test/grammar/builtins/file/load_txt/main.k | 3 ++ .../builtins/file/load_txt/stdout.golden | 1 + test/grammar/builtins/file/load_txt/test.txt | 1 + test/grammar/builtins/file/load_yaml/main.k | 8 ++++ .../builtins/file/load_yaml/stdout.golden | 13 +++++++ .../grammar/builtins/file/load_yaml/test.yaml | 3 ++ 18 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 kclvm/runtime/src/file/mod.rs create mode 100644 test/grammar/builtins/file/load_json/main.k create mode 100644 test/grammar/builtins/file/load_json/stdout.golden create mode 100644 test/grammar/builtins/file/load_json/test.json create mode 100644 test/grammar/builtins/file/load_txt/main.k create mode 100644 test/grammar/builtins/file/load_txt/stdout.golden create mode 100644 test/grammar/builtins/file/load_txt/test.txt create mode 100644 test/grammar/builtins/file/load_yaml/main.k create mode 100644 test/grammar/builtins/file/load_yaml/stdout.golden create mode 100644 test/grammar/builtins/file/load_yaml/test.yaml diff --git a/kclvm/runtime/src/_kclvm.bc b/kclvm/runtime/src/_kclvm.bc index 1ad2df9c51857067eaa902a2a2664eee9da4d82f..2fb0f1688892e5655bfd05fa4fb71b712387f408 100644 GIT binary patch delta 3681 zcmZWse@s(n6n$-JwbT@(K!sE(j#hjF24nshYEwZnsM};}Hyl&xCPrUK##ES5=WDt# z1=Llf%G_e`XRt9K_+xIOLnk`fn818uBs0SW%^F2b7LAHw-)l?zdLDm3a(lVox#ygF zzpwR78{f8Y$-}NwnAn zA#!TW9~bKYT=l-ntr3uB?R6YRaJN!}?hAuzpQO!f@m2HpPs+JxYTboU&UI9^GXdK5+>l>U zW4l9+Oi8F@%xyU%4F$(tRb5YkYAkcFa#bvp>v-Z=q%sGarFdyK+GatT^~J?qP_BAd zj(&0tizykeYxGm%>z`+M`b@JDZ1Q_qncpTpEW{-}1L^3JtPTmfq`XFkF7fEbATsM5 zb%6%da@Xn_c4ey#?0P)8Yw+(R;3T|jARjo9fA1wl= zS>G=*UCh%9^ccM$#uBe@A1+3QJD=J5?SeG25p$o)DzMzgFE3a^%8_bSBNSGv9TDx< zX^t?jUZef<+n-TXA$2ivQKCJUl_WH<*O-U%L^I9o99{jsHVCqBUy&fPR`#vQC~A4F z587z^1ewREL}c6&5i~j+nM@!81dcRU7sxl7ay5#eoXKM^r!fnuHWEBfZX$_?mWQ9uu9R64c9Eo zhcwSNWeO|+E#1$5kr&$t`N7BAGiZ&g$49bzTexq9nz21K(uN1VP zM~rTkXc6q6e-dre^kWm9UCLugphb!2E765`Lqv(|jAkM9KyR4ht_yJhYfrRxYPu~z z9euA#bop04nK!5~d;Q@1#XEGPW35ath>A;@g%XN zNn11@B&a^3dT6i*4xsq&28O=eXpLDABbJPRAgxjA1u@dYg9=d(%tJ4`LI06*C3kYC z!}Hd2r6!}^E~v(z>XBobrLP8i<=V7;p}pqNuGgAiH?WIs%NJ{4pWi=!+z1`CY#xs) zcwQbr6$~S}_zGgew$tF;733*S$*g9;xVQM1Nd3n1RUR^Z(ha z6mUiYYR8c|ehwb}!nRHt15$QIy|6iAm jqo{NRMMcT#)}AQUwJMb>SC^J7UsGaRad3h$LPGry96mr{ delta 3659 zcmZWse^Aq982@f;;24du$xlIn7}y-inHq`MM0U}nLfc@`sZ^-AtklkgOZzTz5#vY{ zFu*V+=x`^>4^0u}1jREvJ?WfY>E&@|;X2F1^E?%#_scf7@8|vg!`*ui&-3|wp3n2V z?@pTQ%vA{jS%*q4WyMRUS{UyaQIK`}m@qL=>pfzEQkpwg?2i}6`keVkEN?rUQ!kYJ zocRSE(3;NY2&QYo}bL_5LSUb=T_m^cJ5Z0?v3T z<7@zgb*}AOj^H|uY&eSG&gHkRL~v|zvZ0s}d5fH8me{X`r7U8E-m?NroUueIpT-Eh zXDO$v?7iZd0thRYz9r;mQ%8k=Z-Q*KbyA~dGMk$}L((}fSip^mVHZYnoSE}`?5R}3;J4^Zwpn7em+GF@}&2zR5zYgh)PfDvn z67v`(iDOJ^07>l1=zSuWOI-Xpo4lNwiZ&VCQ0KOZ3o~)a@Gyfe$ruozORN=NA+o{f z0Blxmx-3Tt(A3q@d#f{KDs}@_{^j<)1kfb#BIQrr9;&i_VObK|WUFi$77mFy>DGiJ z%QBoM1q+%NL!8|Zb3PW+^~m0@f=)PB2WTM7@tf=&$fmv=^RrLY$)Y`J1roW(mmC%{ z*MDuPLOp8lRbeMahvTpl56=13AhI20Bj`-=g)=ogXR>7cqy83LM`zx>pMq64-J!Y= zdUXj_)gLN*0rW7g3tkR-*so7-n$*L?pX1un|YLwetjKN!)?4_K&ol|092iBecpbVUr;2G*k%$#yRWl^jD2UG9vr_$Wy>tD`|QHCXF&M$~n_5pQ#=>*h~%>rM)} zi1I$I-si-de52iUeqiW2w$vj%#%UozARuh=A)Ki3#(O>*q?xs{qb+ zw)`&P;KQGo<-sjcu0qRrKVp^>pY~H1qLY%ud{siNoo=O34^{OyB}FQWH$TIYsx6~u zsmp(`q-m31tWO~4(Dc4;0qhgQ53RTW#uqKdF>7EmaLSCOR&L6?{JEEjzjPKf%s4G9 zA9v^5niF`J{Anu0OZ{=j2qL>vxF3d54Sh^#2z zgviE@cz5t5f1>S>c_%_MUu!~Ss!ko6+`N`TX1ozb-8VzY?TD=B-7M<<{rwnq|BinW zhsbQhR6FOL6VHd2p?QaNy^#NGb!fuFXYizDPgm(1E82LTB$hP(YJG@<5cN;IO#+r2 z+Wh5!2rOypp*TGHj8gZ+fAM=P-22D#k*eZb0}u*0 zsnE!%Jdbl%a>TgbBI$Eo4-z*~?p8fmWI*fo4^c1Wp&vJ1^jU%A8JD$QK2yDsVdNEp zuFS|OeskBTW5+Cwfm1wUZ;T~Tq|h9hJbrtNuuSGIx*~{SH2w^u@T)28vP5@Es`C<` K>v}WsuIzs%n>5S- diff --git a/kclvm/runtime/src/_kclvm.h b/kclvm/runtime/src/_kclvm.h index 0cdfe40eb..dc2c004f0 100644 --- a/kclvm/runtime/src/_kclvm.h +++ b/kclvm/runtime/src/_kclvm.h @@ -1,4 +1,4 @@ -// Copyright 2021 The KCL Authors. All rights reserved. +// Copyright The KCL Authors. All rights reserved. // Auto generated, DONOT EDIT!!! @@ -300,6 +300,8 @@ void kclvm_dict_update_key_value(kclvm_value_ref_t* p, kclvm_value_ref_t* key, k kclvm_value_ref_t* kclvm_dict_values(kclvm_context_t* ctx, kclvm_value_ref_t* p); +kclvm_value_ref_t* kclvm_file_read(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs); + kclvm_value_ref_t* kclvm_iterator_cur_key(kclvm_iterator_t* p); kclvm_value_ref_t* kclvm_iterator_cur_value(kclvm_iterator_t* p); diff --git a/kclvm/runtime/src/_kclvm.ll b/kclvm/runtime/src/_kclvm.ll index 509be3a9c..2269d92bb 100644 --- a/kclvm/runtime/src/_kclvm.ll +++ b/kclvm/runtime/src/_kclvm.ll @@ -1,4 +1,4 @@ -; Copyright 2021 The KCL Authors. All rights reserved. +; Copyright The KCL Authors. All rights reserved. ; Auto generated, DONOT EDIT!!! @@ -248,6 +248,8 @@ declare void @kclvm_dict_update_key_value(%kclvm_value_ref_t* %p, %kclvm_value_r declare %kclvm_value_ref_t* @kclvm_dict_values(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %p); +declare %kclvm_value_ref_t* @kclvm_file_read(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs); + declare %kclvm_value_ref_t* @kclvm_iterator_cur_key(%kclvm_iterator_t* %p); declare %kclvm_value_ref_t* @kclvm_iterator_cur_value(%kclvm_iterator_t* %p); diff --git a/kclvm/runtime/src/_kclvm.rs b/kclvm/runtime/src/_kclvm.rs index 2b46c3eb0..f3ae033af 100644 --- a/kclvm/runtime/src/_kclvm.rs +++ b/kclvm/runtime/src/_kclvm.rs @@ -1,4 +1,4 @@ -//! Copyright The KCL Authors. All rights reserved. +// Copyright The KCL Authors. All rights reserved. // Auto generated, DONOT EDIT!!! @@ -136,6 +136,7 @@ pub enum ApiFunc { kclvm_dict_update, kclvm_dict_update_key_value, kclvm_dict_values, + kclvm_file_read, kclvm_iterator_cur_key, kclvm_iterator_cur_value, kclvm_iterator_delete, diff --git a/kclvm/runtime/src/_kclvm_addr.rs b/kclvm/runtime/src/_kclvm_addr.rs index 450bd44c5..813f83af8 100644 --- a/kclvm/runtime/src/_kclvm_addr.rs +++ b/kclvm/runtime/src/_kclvm_addr.rs @@ -1,4 +1,4 @@ -//! Copyright The KCL Authors. All rights reserved. +// Copyright The KCL Authors. All rights reserved. // Auto generated, DONOT EDIT!!! @@ -143,6 +143,7 @@ pub fn _kclvm_get_fn_ptr_by_name(name: &str) -> u64 { "kclvm_dict_update" => crate::kclvm_dict_update as *const () as u64, "kclvm_dict_update_key_value" => crate::kclvm_dict_update_key_value as *const () as u64, "kclvm_dict_values" => crate::kclvm_dict_values as *const () as u64, + "kclvm_file_read" => crate::kclvm_file_read as *const () as u64, "kclvm_iterator_cur_key" => crate::kclvm_iterator_cur_key as *const () as u64, "kclvm_iterator_cur_value" => crate::kclvm_iterator_cur_value as *const () as u64, "kclvm_iterator_delete" => crate::kclvm_iterator_delete as *const () as u64, diff --git a/kclvm/runtime/src/_kclvm_api_spec.rs b/kclvm/runtime/src/_kclvm_api_spec.rs index 9d4397eab..3fb06f91c 100644 --- a/kclvm/runtime/src/_kclvm_api_spec.rs +++ b/kclvm/runtime/src/_kclvm_api_spec.rs @@ -1,4 +1,4 @@ -//! Copyright The KCL Authors. All rights reserved. +// Copyright 2023 The KCL Authors. All rights reserved. // Auto generated by command, DONOT EDIT!!! @@ -1174,3 +1174,7 @@ // api-spec(c): kclvm_value_ref_t* kclvm_yaml_validate(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs); // api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_validate(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs); +// api-spec: kclvm_file_read +// api-spec(c): kclvm_value_ref_t* kclvm_file_read(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs); +// api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_file_read(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs); + diff --git a/kclvm/runtime/src/file/mod.rs b/kclvm/runtime/src/file/mod.rs new file mode 100644 index 000000000..a678ddfe4 --- /dev/null +++ b/kclvm/runtime/src/file/mod.rs @@ -0,0 +1,28 @@ +use std::fs; + +use crate::*; + +// https://docs.python.org/3/library/math.html +// https://doc.rust-lang.org/std/primitive.f64.html +// https://github.com/RustPython/RustPython/blob/main/stdlib/src/math.rs + +#[no_mangle] +#[runtime_fn] +pub extern "C" fn kclvm_file_read( + ctx: *mut kclvm_context_t, + args: *const kclvm_value_ref_t, + _kwargs: *const kclvm_value_ref_t, +) -> *const kclvm_value_ref_t { + let args = ptr_as_ref(args); + let ctx = mut_ptr_as_ref(ctx); + + if let Some(x) = args.arg_i_str(0, None) { + let contents = + fs::read_to_string(&x).expect(&format!("failed to access the file in {}", x)); + + let s = ValueRef::str(contents.as_ref()); + return s.into_raw(ctx); + } + + panic!("read() takes exactly one argument (0 given)"); +} diff --git a/kclvm/runtime/src/lib.rs b/kclvm/runtime/src/lib.rs index 80681407b..6b13596d0 100644 --- a/kclvm/runtime/src/lib.rs +++ b/kclvm/runtime/src/lib.rs @@ -104,6 +104,9 @@ pub use self::units::*; pub mod yaml; pub use self::yaml::*; +pub mod file; +pub use self::file::*; + pub mod _kcl_run; pub use self::_kcl_run::*; diff --git a/kclvm/sema/src/builtin/system_module.rs b/kclvm/sema/src/builtin/system_module.rs index cd5ca61d8..50e538264 100644 --- a/kclvm/sema/src/builtin/system_module.rs +++ b/kclvm/sema/src/builtin/system_module.rs @@ -1549,8 +1549,40 @@ register_collection_member! { ) } +// ------------------------------ +// file system package +// ------------------------------ + +pub const FILE: &str = "file"; +pub const FILE_FUNCTION_NAMES: &[&str] = &["read"]; +macro_rules! register_file_member { + ($($name:ident => $ty:expr)*) => ( + pub const FILE_FUNCTION_TYPES: Lazy> = Lazy::new(|| { + let mut builtin_mapping = IndexMap::default(); + $( builtin_mapping.insert(stringify!($name).to_string(), $ty); )* + builtin_mapping + }); + ) +} +register_file_member! { + union_all => Type::function( + None, + Type::any_ref(), + &[ + Parameter { + name: "filepath".to_string(), + ty: Type::str_ref(), + has_default: false, + }, + ], + r#"Read the file content from path"#, + false, + None, + ) +} + pub const STANDARD_SYSTEM_MODULES: &[&str] = &[ - COLLECTION, NET, MANIFESTS, MATH, DATETIME, REGEX, YAML, JSON, CRYPTO, BASE64, UNITS, + COLLECTION, NET, MANIFESTS, MATH, DATETIME, REGEX, YAML, JSON, CRYPTO, BASE64, UNITS, FILE, ]; pub const STANDARD_SYSTEM_MODULE_NAMES_WITH_AT: &[&str] = &[ @@ -1565,6 +1597,7 @@ pub const STANDARD_SYSTEM_MODULE_NAMES_WITH_AT: &[&str] = &[ "@crypto", "@base64", "@units", + "@file", ]; /// Get the system module members @@ -1585,6 +1618,7 @@ pub fn get_system_module_members(name: &str) -> Vec<&str> { members } COLLECTION => COLLECTION_FUNCTION_NAMES.to_vec(), + FILE => FILE_FUNCTION_NAMES.to_vec(), _ => bug!("invalid system module name '{}'", name), } } diff --git a/test/grammar/builtins/file/load_json/main.k b/test/grammar/builtins/file/load_json/main.k new file mode 100644 index 000000000..1bda19025 --- /dev/null +++ b/test/grammar/builtins/file/load_json/main.k @@ -0,0 +1,8 @@ +import json +import file + +data_string = json.decode(file.read("test.json")) + +key1 = data_string.key1 +key2 = data_string.key2 +data = data_string.data \ No newline at end of file diff --git a/test/grammar/builtins/file/load_json/stdout.golden b/test/grammar/builtins/file/load_json/stdout.golden new file mode 100644 index 000000000..3c268cc99 --- /dev/null +++ b/test/grammar/builtins/file/load_json/stdout.golden @@ -0,0 +1,13 @@ +data_string: + key1: value1 + key2: value2 + data: + - 1 + - 2 + - 3 +key1: value1 +key2: value2 +data: +- 1 +- 2 +- 3 diff --git a/test/grammar/builtins/file/load_json/test.json b/test/grammar/builtins/file/load_json/test.json new file mode 100644 index 000000000..e8772183f --- /dev/null +++ b/test/grammar/builtins/file/load_json/test.json @@ -0,0 +1,5 @@ +{ + "key1": "value1", + "key2": "value2", + "data": [1, 2, 3] +} \ No newline at end of file diff --git a/test/grammar/builtins/file/load_txt/main.k b/test/grammar/builtins/file/load_txt/main.k new file mode 100644 index 000000000..741febc4f --- /dev/null +++ b/test/grammar/builtins/file/load_txt/main.k @@ -0,0 +1,3 @@ +import file + +a = file.read("test.txt") \ No newline at end of file diff --git a/test/grammar/builtins/file/load_txt/stdout.golden b/test/grammar/builtins/file/load_txt/stdout.golden new file mode 100644 index 000000000..623ab06c1 --- /dev/null +++ b/test/grammar/builtins/file/load_txt/stdout.golden @@ -0,0 +1 @@ +a: Helloworld \ No newline at end of file diff --git a/test/grammar/builtins/file/load_txt/test.txt b/test/grammar/builtins/file/load_txt/test.txt new file mode 100644 index 000000000..0cf453276 --- /dev/null +++ b/test/grammar/builtins/file/load_txt/test.txt @@ -0,0 +1 @@ +Helloworld \ No newline at end of file diff --git a/test/grammar/builtins/file/load_yaml/main.k b/test/grammar/builtins/file/load_yaml/main.k new file mode 100644 index 000000000..01a780149 --- /dev/null +++ b/test/grammar/builtins/file/load_yaml/main.k @@ -0,0 +1,8 @@ +import yaml +import file + +data_string = yaml.decode(file.read("test.yaml")) + +key1 = data_string.key1 +key2 = data_string.key2 +data = data_string.data \ No newline at end of file diff --git a/test/grammar/builtins/file/load_yaml/stdout.golden b/test/grammar/builtins/file/load_yaml/stdout.golden new file mode 100644 index 000000000..3c268cc99 --- /dev/null +++ b/test/grammar/builtins/file/load_yaml/stdout.golden @@ -0,0 +1,13 @@ +data_string: + key1: value1 + key2: value2 + data: + - 1 + - 2 + - 3 +key1: value1 +key2: value2 +data: +- 1 +- 2 +- 3 diff --git a/test/grammar/builtins/file/load_yaml/test.yaml b/test/grammar/builtins/file/load_yaml/test.yaml new file mode 100644 index 000000000..0ffe95c1b --- /dev/null +++ b/test/grammar/builtins/file/load_yaml/test.yaml @@ -0,0 +1,3 @@ +key1: "value1" +key2: "value2" +data: [1, 2, 3]