add integrtion test

tests/integration_tests/run_group.sh

@@ -19,7 +19,7 @@ kafka_only_protocol="canal_json_adapter_compatibility canal_json_basic canal_jso
 kafka_only_v2="kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2"
 
 storage_only="lossy_ddl storage_csv_update"
-storage_only_csv="csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table"
+storage_only_csv="storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table"
 storage_only_canal_json="canal_json_storage_basic canal_json_storage_partition_table"
 
 # Define groups

tests/integration_tests/storage_cleanup/conf/changefeed-default.toml

@@ -0,0 +1,16 @@
+[sink]
+protocol = "csv"
+# Line terminator. Empty value means "\r\n" (CRLF) is line terminators. The default value is empty.
+terminator = "\n"
+# Directory date separator, Optional values are `none`, `year`, `month`, `date`. The default value is none.
+date-separator = 'day'
+
+[sink.csv]
+# Delimiter between fields. Must be ASCII characters. The default value is ','.
+delimiter = ','
+# Quoting character. Empty value means no quoting. The default value is '"'.
+quote = '"'
+# Representation of null values in CSV files, the default value is '\N'
+null = '\N'
+# Include commit-ts in the row data. The default value is false.
+include-commit-ts = true

tests/integration_tests/storage_cleanup/run.sh

@@ -8,7 +8,6 @@ WORK_DIR=$OUT_DIR/$TEST_NAME
 CDC_BINARY=cdc.test
 SINK_TYPE=$1
 
-
 EXIST_FILES=()
 CLEANED_FILES=()
 function generate_single_table_files() {
@@ -23,7 +22,7 @@ function generate_single_table_files() {
 	table_dir=$workdir/$bucket/$schema/$table/$day
 	mkdir -p $table_dir
 	for i in $(seq 1 $file_cnt); do
-	    file=$table_dir/$i.data
+		file=$table_dir/$i.data
 		touch $file
 		if [ "$should_clean" == "true" ]; then
 			CLEANED_FILES+=($file)
@@ -38,8 +37,8 @@ function generate_single_table_files() {
 
 function generate_historic_files() {
 	local target_bucket="storage_test"
-	yesterday=$(date -d "yesterday" +"%Y-%m-%d") # should not be cleaned since file-expiration-days is 1
-	day_before_yesterday=$(date -d "2 days ago" +"%Y-%m-%d") # should be cleaned 
+	yesterday=$(date -d "yesterday" +"%Y-%m-%d")             # should not be cleaned since file-expiration-days is 1
+	day_before_yesterday=$(date -d "2 days ago" +"%Y-%m-%d") # should be cleaned
 
 	# historic files of table in schema.sql
 	generate_single_table_files $WORK_DIR $target_bucket test multi_data_type $yesterday 10 false