From 4260414216bd68e751df6c193f57fce9e5d2d1b8 Mon Sep 17 00:00:00 2001 From: zhaishuang <1531939060@qq.com> Date: Fri, 10 Nov 2023 15:50:27 +0800 Subject: [PATCH 1/3] rust\lakesoul-datafusion\test\upsert_tests.rs add some test cases Signed-off-by: zhaishuang <1531939060@qq.com> --- rust/Cargo.lock | 8 + rust/lakesoul-datafusion/Cargo.toml | 1 + .../src/test/upsert_tests.rs | 282 +++++++++++++++++- 3 files changed, 290 insertions(+), 1 deletion(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e3f0a53f1..0150ebaec 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -580,6 +580,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "pure-rust-locales", "serde", "wasm-bindgen", "windows-targets", @@ -1534,6 +1535,7 @@ name = "lakesoul-datafusion" version = "0.1.0" dependencies = [ "async-trait", + "chrono", "futures", "lakesoul-io", "lakesoul-metadata", @@ -2295,6 +2297,12 @@ dependencies = [ "autotools", ] +[[package]] +name = "pure-rust-locales" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed02a829e62dc2715ceb8afb4f80e298148e1345749ceb369540fe0eb3368432" + [[package]] name = "quick-xml" version = "0.28.2" diff --git a/rust/lakesoul-datafusion/Cargo.toml b/rust/lakesoul-datafusion/Cargo.toml index 8e0aa7dd9..3a5d91c62 100644 --- a/rust/lakesoul-datafusion/Cargo.toml +++ b/rust/lakesoul-datafusion/Cargo.toml @@ -17,3 +17,4 @@ prost = "0.11" async-trait = "0.1" futures = "0.3" uuid = { version = "1.4.0", features = ["v4", "fast-rng", "macro-diagnostics"]} +chrono = { version = "0.4", features = ["unstable-locales"] } \ No newline at end of file diff --git a/rust/lakesoul-datafusion/src/test/upsert_tests.rs b/rust/lakesoul-datafusion/src/test/upsert_tests.rs index f7fa0a8a7..43b59f551 100644 --- a/rust/lakesoul-datafusion/src/test/upsert_tests.rs +++ b/rust/lakesoul-datafusion/src/test/upsert_tests.rs @@ -7,7 +7,8 @@ mod upsert_with_io_config_tests { use std::env; use std::path::PathBuf; use std::time::SystemTime; - + use chrono::naive::NaiveDate; + use lakesoul_io::arrow::record_batch::RecordBatch; use lakesoul_io::arrow::util::pretty::print_batches; use lakesoul_io::datafusion::assert_batches_eq; @@ -20,6 +21,7 @@ mod upsert_with_io_config_tests { use lakesoul_io::arrow::datatypes::{Schema, SchemaRef, Field}; use lakesoul_io::lakesoul_io_config::LakeSoulIOConfigBuilder; use lakesoul_io::lakesoul_writer::SyncSendableMutableLakeSoulWriter; + use lakesoul_io::arrow::array::Int64Array; fn init_table(batch: RecordBatch, table_name: &str, pks:Vec) -> LakeSoulIOConfigBuilder { @@ -79,6 +81,34 @@ mod upsert_with_io_config_tests { RecordBatch::try_from_iter_with_nullable(iter).unwrap() } + fn create_batch_i64(names: Vec<&str>, values: Vec<&[i64]>) -> RecordBatch { + let values = values + .into_iter() + .map(|vec| Arc::new(Int64Array::from(Vec::from(vec))) as ArrayRef) + .collect::>(); + let iter = names.into_iter().zip(values).map(|(name, array)| (name, array, true)).collect::>(); + RecordBatch::try_from_iter_with_nullable(iter).unwrap() + } + + fn check_upsert_i64(batch: RecordBatch, table_name: &str, selected_cols: Vec<&str>, filters: Option, builder: LakeSoulIOConfigBuilder, expected: &[&str]) -> LakeSoulIOConfigBuilder { + let builder = execute_upsert(batch, table_name, builder.clone()); + let builder = builder + .with_schema(SchemaRef::new(Schema::new( + selected_cols.iter().map(|col| Field::new(*col, arrow::datatypes::DataType::Int64, true)).collect::>() + ))); + let builder = if let Some(filters) = filters { + builder.with_filter_str(filters) + } else { + builder + }; + let config = builder.clone().build(); + + let mut reader = SyncSendableMutableLakeSoulReader::new(LakeSoulReader::new(config).unwrap(), Builder::new_current_thread().build().unwrap()); + let _ = reader.start_blocked(); + let result = reader.next_rb_blocked(); + assert_batches_eq!(expected, &[result.unwrap().unwrap()]); + builder + } #[test] fn test_merge_same_column_i32() { @@ -518,6 +548,256 @@ mod upsert_with_io_config_tests { ); } + #[test] + fn test_derange_hash_key_and_data_schema_order_int_type_upsert_1_times_i32(){ + let table_name = "derange_hash_key_and_data_schema_order_int_type_upsert_1_times_i32"; + let builder = init_table( + create_batch_i32(vec!["range", "hash1", "hash2", "value", "name", "age"], vec![&[20201101, 20201101], &[1, 2], &[1, 2], &[1, 2], &[1, 2],&[1, 2]]), + table_name, + vec!["range".to_string(), "hash1".to_string(),"hash2".to_string()]); + check_upsert( + create_batch_i32(vec!["range", "hash1", "hash2", "value"], vec![&[20201102, 20201102, 20201102], &[1, 3, 4], &[12, 32, 42], &[1, 3, 4]]), + table_name, + vec!["range", "hash1", "hash2", "value", "name", "age"], + Some("and(noteq(range, null), eq(range, 20201102))".to_string()), + builder.clone(), + &[ + "+----------+-------+-------+-------+------+-----+", + "| range | hash1 | hash2 | value | name | age |", + "+----------+-------+-------+-------+------+-----+", + "| 20201102 | 1 | 12 | 1 | | |", + "| 20201102 | 3 | 32 | 3 | | |", + "| 20201102 | 4 | 42 | 4 | | |", + "+----------+-------+-------+-------+------+-----+", + ] + ); + } + + #[test] + fn test_derange_hash_key_and_data_schema_order_int_type_upsert_2_times_i32(){ + let table_name = "derange_hash_key_and_data_schema_order_int_type_upsert_2_times_i32"; + let builder = init_table( + create_batch_i32(vec!["range", "hash1", "hash2", "value", "name", "age"], vec![&[20201101, 20201101], &[1, 2], &[1, 2], &[1, 2], &[1, 2],&[1, 2]]), + table_name, + vec!["range".to_string(), "hash1".to_string(),"hash2".to_string()]); + let builder = execute_upsert( + create_batch_i32(vec!["range", "hash1", "hash2", "value"], vec![&[20201102, 20201102, 20201102], &[1, 3, 4], &[12, 32, 42], &[1, 3, 4]]), + table_name, + builder); + check_upsert( + create_batch_i32(vec!["range", "hash2", "name", "hash1"], vec![&[20201102, 20201102, 20201102], &[12, 22, 32], &[11, 22, 33], &[1, 2, 3]]), + table_name, + vec!["range", "hash1", "hash2", "value", "name", "age"], + Some("and(noteq(range, null), eq(range, 20201102))".to_string()), + builder.clone(), + &[ + "+----------+-------+-------+-------+------+-----+", + "| range | hash1 | hash2 | value | name | age |", + "+----------+-------+-------+-------+------+-----+", + "| 20201102 | 1 | 12 | 1 | 11 | |", + "| 20201102 | 2 | 22 | | 22 | |", + "| 20201102 | 3 | 32 | 3 | 33 | |", + "| 20201102 | 4 | 42 | 4 | | |", + "+----------+-------+-------+-------+------+-----+", + ] + ); + } + + #[test] + fn test_derange_hash_key_and_data_schema_order_int_type_upsert_3_times_i32(){ + let table_name = "derange_hash_key_and_data_schema_order_int_type_upsert_3_times_i32"; + let builder = init_table( + create_batch_i32(vec!["range", "hash1", "hash2", "value", "name", "age"], vec![&[20201101, 20201101], &[1, 2], &[1, 2], &[1, 2], &[1, 2],&[1, 2]]), + table_name, + vec!["range".to_string(), "hash1".to_string(),"hash2".to_string()]); + let builder = execute_upsert( + create_batch_i32(vec!["range", "hash1", "hash2", "value"], vec![&[20201102, 20201102, 20201102], &[1, 3, 4], &[12, 32, 42], &[1, 3, 4]]), + table_name, + builder); + let builder = execute_upsert( + create_batch_i32(vec!["range", "hash2", "name", "hash1"], vec![&[20201102, 20201102, 20201102], &[12, 22, 32], &[11, 22, 33], &[1, 2, 3]]), + table_name, + builder); + check_upsert( + create_batch_i32(vec!["range", "age", "hash2", "name", "hash1"], vec![&[20201102, 20201102, 20201102], &[4567, 2345, 3456], &[42, 22, 32], &[456, 234, 345], &[4, 2, 3]]), + table_name, + vec!["range", "hash1", "hash2", "value", "name", "age"], + Some("and(and(noteq(range, null), eq(range, 20201102)), noteq(value, null))".to_string()), + builder.clone(), + &[ + "+----------+-------+-------+-------+------+------+", + "| range | hash1 | hash2 | value | name | age |", + "+----------+-------+-------+-------+------+------+", + "| 20201102 | 1 | 12 | 1 | 11 | |", + "| 20201102 | 3 | 32 | 3 | 345 | 3456 |", + "| 20201102 | 4 | 42 | 4 | 456 | 4567 |", + "+----------+-------+-------+-------+------+------+", + ] + ); + } + + #[test] + fn test_derange_hash_key_and_data_schema_order_string_type_upsert_1_times_i32(){ + let table_name = "derange_hash_key_and_data_schema_order_string_type_upsert_1_times_i32"; + let builder = init_table( + create_batch_i32(vec!["range", "hash1", "hash2", "value", "name", "age"], vec![&[20201101, 20201101], &[1, 2], &[1, 2], &[1, 2], &[1, 2],&[1, 2]]), + table_name, + vec!["range".to_string(), "hash1".to_string(),"hash2".to_string()]); + check_upsert( + create_batch_i32(vec!["range", "hash1", "hash2", "value"], vec![&[20201102, 20201102, 20201102], &[1, 3, 4], &[12, 32, 42], &[1, 3, 4]]), + table_name, + vec!["range", "hash1", "hash2", "value", "name", "age"], + Some("and(noteq(range, null), eq(range, 20201102))".to_string()), + builder.clone(), + &[ + "+----------+-------+-------+-------+------+-----+", + "| range | hash1 | hash2 | value | name | age |", + "+----------+-------+-------+-------+------+-----+", + "| 20201102 | 1 | 12 | 1 | | |", + "| 20201102 | 3 | 32 | 3 | | |", + "| 20201102 | 4 | 42 | 4 | | |", + "+----------+-------+-------+-------+------+-----+", + ] + ); + } + + #[test] + fn test_derange_hash_key_and_data_schema_order_string_type_upsert_2_times_i32(){ + let table_name = "derange_hash_key_and_data_schema_order_string_type_upsert_2_times_i32"; + let builder = init_table( + create_batch_i32(vec!["range", "hash1", "hash2", "value", "name", "age"], vec![&[20201101, 20201101], &[1, 2], &[1, 2], &[1, 2], &[1, 2],&[1, 2]]), + table_name, + vec!["range".to_string(), "hash1".to_string(),"hash2".to_string()]); + let builder = execute_upsert( + create_batch_i32(vec!["range", "hash1", "hash2", "value"], vec![&[20201102, 20201102, 20201102], &[1, 3, 4], &[12, 32, 42], &[1, 3, 4]]), + table_name, + builder); + check_upsert( + create_batch_i32(vec!["range", "hash2", "name", "hash1"], vec![&[20201102, 20201102, 20201102], &[12, 22, 32], &[11, 22, 33], &[1, 2, 3]]), + table_name, + vec!["range", "hash1", "hash2", "value", "name", "age"], + Some("and(noteq(range, null), eq(range, 20201102))".to_string()), + builder.clone(), + &[ + "+----------+-------+-------+-------+------+-----+", + "| range | hash1 | hash2 | value | name | age |", + "+----------+-------+-------+-------+------+-----+", + "| 20201102 | 1 | 12 | 1 | 11 | |", + "| 20201102 | 2 | 22 | | 22 | |", + "| 20201102 | 3 | 32 | 3 | 33 | |", + "| 20201102 | 4 | 42 | 4 | | |", + "+----------+-------+-------+-------+------+-----+", + ] + ); + } + + #[test] + fn test_derange_hash_key_and_data_schema_order_string_type_upsert_3_times_i32(){ + let table_name = "derange_hash_key_and_data_schema_order_string_type_upsert_3_times_i32"; + let builder = init_table( + create_batch_i32(vec!["range", "hash1", "hash2", "value", "name", "age"], vec![&[20201101, 20201101], &[1, 2], &[1, 2], &[1, 2], &[1, 2],&[1, 2]]), + table_name, + vec!["range".to_string(), "hash1".to_string(),"hash2".to_string()]); + let builder = execute_upsert( + create_batch_i32(vec!["range", "hash1", "hash2", "value"], vec![&[20201102, 20201102, 20201102], &[1, 3, 4], &[12, 32, 42], &[1, 3, 4]]), + table_name, + builder); + let builder = execute_upsert( + create_batch_i32(vec!["range", "hash2", "name", "hash1"], vec![&[20201102, 20201102, 20201102], &[12, 22, 32], &[11, 22, 33], &[1, 2, 3]]), + table_name, + builder); + check_upsert( + create_batch_i32(vec!["range", "age", "hash2", "name", "hash1"], vec![&[20201102, 20201102, 20201102], &[4567, 2345, 3456], &[42, 22, 32], &[456, 234, 345], &[4, 2, 3]]), + table_name, + vec!["range", "hash1", "hash2", "value", "name", "age"], + Some("and(and(noteq(range, null), eq(range, 20201102)), noteq(value, null))".to_string()), + builder.clone(), + &[ + "+----------+-------+-------+-------+------+------+", + "| range | hash1 | hash2 | value | name | age |", + "+----------+-------+-------+-------+------+------+", + "| 20201102 | 1 | 12 | 1 | 11 | |", + "| 20201102 | 3 | 32 | 3 | 345 | 3456 |", + "| 20201102 | 4 | 42 | 4 | 456 | 4567 |", + "+----------+-------+-------+-------+------+------+", + ] + ); + } + + #[test] + fn test_merge_same_column_with_timestamp_type_i64_time(){ + let dt1=NaiveDate::from_ymd_opt(1000, 6, 14).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + let dt2=NaiveDate::from_ymd_opt(1582, 6, 15).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + let dt3=NaiveDate::from_ymd_opt(1900, 6, 16).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + let dt4=NaiveDate::from_ymd_opt(2018, 6, 17).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + + let val1=dt1.timestamp_micros(); + let val2=dt2.timestamp_micros(); + let val3=dt3.timestamp_micros(); + let val4=dt4.timestamp_micros(); + + let table_name = "test_merge_same_column_with_timestamp_type_i64_time"; + let builder = init_table( + create_batch_i64(vec!["range", "hash", "value", "timestamp"], vec![&[20201101, 20201101, 20201101, 20201102], &[1, 2, 3, 4], &[1, 2, 3, 4], &[val1, val2, val3, val4]]), + table_name, + vec!["range".to_string(), "hash".to_string()]); + check_upsert_i64( + create_batch_i64(vec!["range", "hash", "value"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44]]), + table_name, + vec!["range", "hash", "value", "timestamp"], + None, + builder.clone(), + &[ + "+----------+------+-------+--------------------+", + "| range | hash | value | timestamp |", + "+----------+------+-------+--------------------+", + "| 20201101 | 1 | 11 | -30596023866876544 |", + "| 20201101 | 2 | 2 | -12229803066876544 |", + "| 20201101 | 3 | 33 | -2194615866876544 |", + "| 20201101 | 4 | 44 | |", + "| 20201102 | 4 | 4 | 1529224133123456 |", + "+----------+------+-------+--------------------+", + ] + ); + } + + #[test] + fn test_merge_different_columns_with_timestamp_type_i64_time(){ + let dt1=NaiveDate::from_ymd_opt(1000, 6, 14).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + let _dt2=NaiveDate::from_ymd_opt(1582, 6, 15).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + let dt3=NaiveDate::from_ymd_opt(1900, 6, 16).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + let dt4=NaiveDate::from_ymd_opt(2018, 6, 17).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); + + let val1=dt1.timestamp_micros(); + let _val2=_dt2.timestamp_micros(); + let val3=dt3.timestamp_micros(); + let val4=dt4.timestamp_micros(); + + let table_name = "merge_different_columns_with_timestamp_type_i32_time"; + let builder = init_table( + create_batch_i32(vec!["range", "hash", "value"], vec![&[20201101, 20201101, 20201101, 20201102], &[1, 2, 3, 4], &[1, 2, 3, 4]]), + table_name, + vec!["range".to_string(), "hash".to_string()]); + check_upsert_i64( + create_batch_i64(vec!["range", "hash", "name", "timestamp"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44], &[val1, val3, val4]]), + table_name, + vec!["range", "hash", "value", "name", "timestamp"], + None, + builder.clone(), + &[ + "+----------+------+-------+------+--------------------+", + "| range | hash | value | name | timestamp |", + "+----------+------+-------+------+--------------------+", + "| 20201101 | 1 | 1 | 11 | -30596023866876544 |", + "| 20201101 | 2 | 2 | | |", + "| 20201101 | 3 | 3 | 33 | -2194615866876544 |", + "| 20201101 | 4 | | 44 | 1529224133123456 |", + "| 20201102 | 4 | 4 | | |", + "+----------+------+-------+------+--------------------+", + ] + ); + } + } From 5d406227693dc55af915c2c3ae4991dc1200c7d1 Mon Sep 17 00:00:00 2001 From: zhaishuang <1531939060@qq.com> Date: Tue, 14 Nov 2023 10:47:24 +0800 Subject: [PATCH 2/3] modify column timestamp's type from i64 to timestamp type Signed-off-by: zhaishuang <1531939060@qq.com> --- .../src/test/upsert_tests.rs | 66 +++++++++++-------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/rust/lakesoul-datafusion/src/test/upsert_tests.rs b/rust/lakesoul-datafusion/src/test/upsert_tests.rs index 43b59f551..9e377e29b 100644 --- a/rust/lakesoul-datafusion/src/test/upsert_tests.rs +++ b/rust/lakesoul-datafusion/src/test/upsert_tests.rs @@ -17,8 +17,8 @@ mod upsert_with_io_config_tests { use lakesoul_io::lakesoul_reader::{LakeSoulReader, SyncSendableMutableLakeSoulReader}; use lakesoul_io::tokio::runtime::Builder; use lakesoul_io::arrow; - use lakesoul_io::arrow::array::{ArrayRef, Int32Array}; - use lakesoul_io::arrow::datatypes::{Schema, SchemaRef, Field}; + use lakesoul_io::arrow::array::{ArrayRef, Int32Array,TimestampMicrosecondArray}; + use lakesoul_io::arrow::datatypes::{Schema, SchemaRef, Field,TimestampMicrosecondType, TimeUnit}; use lakesoul_io::lakesoul_io_config::LakeSoulIOConfigBuilder; use lakesoul_io::lakesoul_writer::SyncSendableMutableLakeSoulWriter; use lakesoul_io::arrow::array::Int64Array; @@ -81,20 +81,28 @@ mod upsert_with_io_config_tests { RecordBatch::try_from_iter_with_nullable(iter).unwrap() } - fn create_batch_i64(names: Vec<&str>, values: Vec<&[i64]>) -> RecordBatch { - let values = values + fn create_batch_i32_and_timestamp(names: Vec<&str>, values: Vec<&[i32]>, timestamp:Vec) -> RecordBatch { + let mut values = values .into_iter() - .map(|vec| Arc::new(Int64Array::from(Vec::from(vec))) as ArrayRef) + .map(|vec| Arc::new(Int32Array::from(Vec::from(vec))) as ArrayRef) .collect::>(); + let timestamp = Arc::new(TimestampMicrosecondArray::from(timestamp)) as ArrayRef; + values.push(timestamp); let iter = names.into_iter().zip(values).map(|(name, array)| (name, array, true)).collect::>(); RecordBatch::try_from_iter_with_nullable(iter).unwrap() } - fn check_upsert_i64(batch: RecordBatch, table_name: &str, selected_cols: Vec<&str>, filters: Option, builder: LakeSoulIOConfigBuilder, expected: &[&str]) -> LakeSoulIOConfigBuilder { + fn check_upsert_i32_and_timestamp(batch: RecordBatch, table_name: &str, selected_cols: Vec<&str>, filters: Option, builder: LakeSoulIOConfigBuilder, expected: &[&str]) -> LakeSoulIOConfigBuilder { let builder = execute_upsert(batch, table_name, builder.clone()); let builder = builder .with_schema(SchemaRef::new(Schema::new( - selected_cols.iter().map(|col| Field::new(*col, arrow::datatypes::DataType::Int64, true)).collect::>() + selected_cols.iter().map(|col| + if *col=="timestamp"{ + Field::new(*col, arrow::datatypes::DataType::Timestamp(TimeUnit::Microsecond, None), true) + }else{ + Field::new(*col, arrow::datatypes::DataType::Int32, true) + } + ).collect::>() ))); let builder = if let Some(filters) = filters { builder.with_filter_str(filters) @@ -738,25 +746,25 @@ mod upsert_with_io_config_tests { let table_name = "test_merge_same_column_with_timestamp_type_i64_time"; let builder = init_table( - create_batch_i64(vec!["range", "hash", "value", "timestamp"], vec![&[20201101, 20201101, 20201101, 20201102], &[1, 2, 3, 4], &[1, 2, 3, 4], &[val1, val2, val3, val4]]), + create_batch_i32_and_timestamp(vec!["range", "hash", "value", "timestamp"], vec![&[20201101, 20201101, 20201101, 20201102], &[1, 2, 3, 4], &[1, 2, 3, 4]], vec![val1, val2, val3, val4]), table_name, vec!["range".to_string(), "hash".to_string()]); - check_upsert_i64( + check_upsert_i32_and_timestamp( create_batch_i64(vec!["range", "hash", "value"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44]]), table_name, vec!["range", "hash", "value", "timestamp"], None, builder.clone(), &[ - "+----------+------+-------+--------------------+", - "| range | hash | value | timestamp |", - "+----------+------+-------+--------------------+", - "| 20201101 | 1 | 11 | -30596023866876544 |", - "| 20201101 | 2 | 2 | -12229803066876544 |", - "| 20201101 | 3 | 33 | -2194615866876544 |", - "| 20201101 | 4 | 44 | |", - "| 20201102 | 4 | 4 | 1529224133123456 |", - "+----------+------+-------+--------------------+", + "+----------+------+-------+----------------------------+", + "| range | hash | value | timestamp |", + "+----------+------+-------+----------------------------+", + "| 20201101 | 1 | 11 | 1000-06-14T08:28:53.123456 |", + "| 20201101 | 2 | 2 | 1582-06-15T08:28:53.123456 |", + "| 20201101 | 3 | 33 | 1900-06-16T08:28:53.123456 |", + "| 20201101 | 4 | 44 | |", + "| 20201102 | 4 | 4 | 2018-06-17T08:28:53.123456 |", + "+----------+------+-------+----------------------------+", ] ); } @@ -778,22 +786,22 @@ mod upsert_with_io_config_tests { create_batch_i32(vec!["range", "hash", "value"], vec![&[20201101, 20201101, 20201101, 20201102], &[1, 2, 3, 4], &[1, 2, 3, 4]]), table_name, vec!["range".to_string(), "hash".to_string()]); - check_upsert_i64( - create_batch_i64(vec!["range", "hash", "name", "timestamp"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44], &[val1, val3, val4]]), + check_upsert_i32_and_timestamp( + create_batch_i32_and_timestamp(vec!["range", "hash", "name", "timestamp"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44]],vec![val1, val3, val4]), table_name, vec!["range", "hash", "value", "name", "timestamp"], None, builder.clone(), &[ - "+----------+------+-------+------+--------------------+", - "| range | hash | value | name | timestamp |", - "+----------+------+-------+------+--------------------+", - "| 20201101 | 1 | 1 | 11 | -30596023866876544 |", - "| 20201101 | 2 | 2 | | |", - "| 20201101 | 3 | 3 | 33 | -2194615866876544 |", - "| 20201101 | 4 | | 44 | 1529224133123456 |", - "| 20201102 | 4 | 4 | | |", - "+----------+------+-------+------+--------------------+", + "+----------+------+-------+------+----------------------------+", + "| range | hash | value | name | timestamp |", + "+----------+------+-------+------+----------------------------+", + "| 20201101 | 1 | 1 | 11 | 1000-06-14T08:28:53.123456 |", + "| 20201101 | 2 | 2 | | |", + "| 20201101 | 3 | 3 | 33 | 1900-06-16T08:28:53.123456 |", + "| 20201101 | 4 | | 44 | 2018-06-17T08:28:53.123456 |", + "| 20201102 | 4 | 4 | | |", + "+----------+------+-------+------+----------------------------+", ] ); } From 3fad7b876f736064e586d6d1875f493bc4f869f5 Mon Sep 17 00:00:00 2001 From: zhaishuang <1531939060@qq.com> Date: Tue, 14 Nov 2023 14:53:06 +0800 Subject: [PATCH 3/3] fix the test failure Signed-off-by: zhaishuang <1531939060@qq.com> --- rust/lakesoul-datafusion/src/test/upsert_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/lakesoul-datafusion/src/test/upsert_tests.rs b/rust/lakesoul-datafusion/src/test/upsert_tests.rs index 9e377e29b..20a881e16 100644 --- a/rust/lakesoul-datafusion/src/test/upsert_tests.rs +++ b/rust/lakesoul-datafusion/src/test/upsert_tests.rs @@ -733,7 +733,7 @@ mod upsert_with_io_config_tests { } #[test] - fn test_merge_same_column_with_timestamp_type_i64_time(){ + fn test_merge_same_column_with_timestamp_type_i32_time(){ let dt1=NaiveDate::from_ymd_opt(1000, 6, 14).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); let dt2=NaiveDate::from_ymd_opt(1582, 6, 15).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); let dt3=NaiveDate::from_ymd_opt(1900, 6, 16).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); @@ -750,7 +750,7 @@ mod upsert_with_io_config_tests { table_name, vec!["range".to_string(), "hash".to_string()]); check_upsert_i32_and_timestamp( - create_batch_i64(vec!["range", "hash", "value"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44]]), + create_batch_i32(vec!["range", "hash", "value"], vec![&[20201101, 20201101, 20201101], &[1, 3, 4], &[11, 33, 44]]), table_name, vec!["range", "hash", "value", "timestamp"], None, @@ -770,7 +770,7 @@ mod upsert_with_io_config_tests { } #[test] - fn test_merge_different_columns_with_timestamp_type_i64_time(){ + fn test_merge_different_columns_with_timestamp_type_i32_time(){ let dt1=NaiveDate::from_ymd_opt(1000, 6, 14).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); let _dt2=NaiveDate::from_ymd_opt(1582, 6, 15).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap(); let dt3=NaiveDate::from_ymd_opt(1900, 6, 16).unwrap().and_hms_micro_opt(8, 28, 53, 123456).unwrap();