From c089eda959456665a69da05b4e4c93fbeaeb0406 Mon Sep 17 00:00:00 2001 From: smartlxh Date: Sat, 23 Nov 2024 22:52:22 +0800 Subject: [PATCH 1/3] support alter table add roll up in shard data Signed-off-by: smartlxh --- .../alter/MaterializedViewHandler.java | 3 -- test/sql/test_lake_rollup/R/test_lake_rollup | 32 +++++++++++++++++++ test/sql/test_lake_rollup/T/test_lake_rollup | 21 ++++++++++++ .../sql/test_lake_sync_mv/R/test_lake_sync_mv | 16 ++++++++-- .../sql/test_lake_sync_mv/T/test_lake_sync_mv | 13 ++------ 5 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 test/sql/test_lake_rollup/R/test_lake_rollup create mode 100644 test/sql/test_lake_rollup/T/test_lake_rollup diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java index 21426feae02c4..5ef8406d1e8f0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java @@ -947,9 +947,6 @@ private void getAlterJobV2Infos(Database db, List> rollupJobInf @Override public ShowResultSet process(List alterClauses, Database db, OlapTable olapTable) throws DdlException, AnalysisException, MetaNotFoundException { - if (olapTable.isCloudNativeTable()) { - throw new DdlException("Does not support add rollup on lake table"); - } if (olapTable.existTempPartitions()) { throw new DdlException("Can not alter table when there are temp partitions in table"); } diff --git a/test/sql/test_lake_rollup/R/test_lake_rollup b/test/sql/test_lake_rollup/R/test_lake_rollup new file mode 100644 index 0000000000000..1c83768a15604 --- /dev/null +++ b/test/sql/test_lake_rollup/R/test_lake_rollup @@ -0,0 +1,32 @@ +-- name: test_lake_rollup @cloud +CREATE TABLE tbl1 ( + k1 date, + k2 int, + v1 int sum +) PARTITION BY RANGE(k1) ( + PARTITION p1 values [('2020-01-01'),('2020-02-01')), + PARTITION p2 values [('2020-02-01'),('2020-03-01'))) + DISTRIBUTED BY HASH(k2) BUCKETS 3 + PROPERTIES('replication_num' = '1'); +-- result: +-- !result + +INSERT INTO tbl1 +VALUES + ("2020-01-12",4,100), + ("2020-01-11",5,100), + ("2020-01-11",4,100); +-- result: +-- !result + +ALTER TABLE tbl1 ADD ROLLUP rollup1 (k1, v1) FROM tbl1; +-- result: +-- !result +function: wait_alter_table_finish("ROLLUP") +-- result: +None +-- !result + +INSERT INTO tbl1 VALUES("2020-01-11",6,100); +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_lake_rollup/T/test_lake_rollup b/test/sql/test_lake_rollup/T/test_lake_rollup new file mode 100644 index 0000000000000..72752d327e988 --- /dev/null +++ b/test/sql/test_lake_rollup/T/test_lake_rollup @@ -0,0 +1,21 @@ +-- name: test_lake_rollup @cloud +CREATE TABLE tbl1 ( + k1 date, + k2 int, + v1 int sum +) PARTITION BY RANGE(k1) ( + PARTITION p1 values [('2020-01-01'),('2020-02-01')), + PARTITION p2 values [('2020-02-01'),('2020-03-01'))) + DISTRIBUTED BY HASH(k2) BUCKETS 3 + PROPERTIES('replication_num' = '1'); + +INSERT INTO tbl1 +VALUES + ("2020-01-12",4,100), + ("2020-01-11",5,100), + ("2020-01-11",4,100); + +ALTER TABLE tbl1 ADD ROLLUP rollup1 (k1, v1) FROM tbl1; +function: wait_alter_table_finish("ROLLUP") + +INSERT INTO tbl1 VALUES("2020-01-11",6,100); \ No newline at end of file diff --git a/test/sql/test_lake_sync_mv/R/test_lake_sync_mv b/test/sql/test_lake_sync_mv/R/test_lake_sync_mv index 23d6f77bdd111..dea9989a7fc66 100644 --- a/test/sql/test_lake_sync_mv/R/test_lake_sync_mv +++ b/test/sql/test_lake_sync_mv/R/test_lake_sync_mv @@ -1,4 +1,4 @@ --- name: test_invalid_json @cloud +-- name: test_lake_sync_mv @cloud CREATE TABLE tbl1 ( k1 date, k2 int, @@ -8,13 +8,25 @@ CREATE TABLE tbl1 ( PARTITION p2 values [('2020-02-01'),('2020-03-01'))) DISTRIBUTED BY HASH(k2) BUCKETS 3 PROPERTIES('replication_num' = '1'); +-- result: +-- !result INSERT INTO tbl1 VALUES ("2020-01-12",4,100), ("2020-01-11",5,100), ("2020-01-11",4,100); +-- result: +-- !result CREATE MATERIALIZED VIEW sync_mv AS SELECT k1, SUM(v1) FROM tbl1 GROUP BY k1; +-- result: +-- !result +function: wait_materialized_view_finish() +-- result: +None +-- !result -INSERT INTO tbl1 VALUES("2020-01-11",6,100); \ No newline at end of file +INSERT INTO tbl1 VALUES("2020-01-11",6,100); +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_lake_sync_mv/T/test_lake_sync_mv b/test/sql/test_lake_sync_mv/T/test_lake_sync_mv index cf78b2aae7654..d78905a01dab5 100644 --- a/test/sql/test_lake_sync_mv/T/test_lake_sync_mv +++ b/test/sql/test_lake_sync_mv/T/test_lake_sync_mv @@ -1,4 +1,4 @@ --- name: test_invalid_json @cloud +-- name: test_lake_sync_mv @cloud CREATE TABLE tbl1 ( k1 date, k2 int, @@ -8,21 +8,14 @@ CREATE TABLE tbl1 ( PARTITION p2 values [('2020-02-01'),('2020-03-01'))) DISTRIBUTED BY HASH(k2) BUCKETS 3 PROPERTIES('replication_num' = '1'); --- result: --- !result INSERT INTO tbl1 VALUES ("2020-01-12",4,100), ("2020-01-11",5,100), ("2020-01-11",4,100); --- result: --- !result CREATE MATERIALIZED VIEW sync_mv AS SELECT k1, SUM(v1) FROM tbl1 GROUP BY k1; --- result: --- !result +function: wait_materialized_view_finish() -INSERT INTO tbl1 VALUES("2020-01-11",6,100); --- result: --- !result \ No newline at end of file +INSERT INTO tbl1 VALUES("2020-01-11",6,100); \ No newline at end of file From 8aaf8d115c3a86a6f262d7e027fb9609e0ace2fd Mon Sep 17 00:00:00 2001 From: smartlxh Date: Mon, 25 Nov 2024 17:59:48 +0800 Subject: [PATCH 2/3] add query of rollup Signed-off-by: smartlxh --- test/sql/test_lake_rollup/R/test_lake_rollup | 6 ++++++ test/sql/test_lake_rollup/T/test_lake_rollup | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/sql/test_lake_rollup/R/test_lake_rollup b/test/sql/test_lake_rollup/R/test_lake_rollup index 1c83768a15604..c2ca3aa44bbe6 100644 --- a/test/sql/test_lake_rollup/R/test_lake_rollup +++ b/test/sql/test_lake_rollup/R/test_lake_rollup @@ -29,4 +29,10 @@ None INSERT INTO tbl1 VALUES("2020-01-11",6,100); -- result: +-- !result + +SELECT v1 FROM rollup1 [_SYNC_MV_] ORDER BY v1; +-- result: +100 +300 -- !result \ No newline at end of file diff --git a/test/sql/test_lake_rollup/T/test_lake_rollup b/test/sql/test_lake_rollup/T/test_lake_rollup index 72752d327e988..705d52d9a0b54 100644 --- a/test/sql/test_lake_rollup/T/test_lake_rollup +++ b/test/sql/test_lake_rollup/T/test_lake_rollup @@ -18,4 +18,6 @@ VALUES ALTER TABLE tbl1 ADD ROLLUP rollup1 (k1, v1) FROM tbl1; function: wait_alter_table_finish("ROLLUP") -INSERT INTO tbl1 VALUES("2020-01-11",6,100); \ No newline at end of file +INSERT INTO tbl1 VALUES("2020-01-11",6,100); + +SELECT v1 FROM rollup1 [_SYNC_MV_] ORDER BY v1; \ No newline at end of file From 4895273ff3c176f385f92b2d6eb635c1b561b0fd Mon Sep 17 00:00:00 2001 From: smartlxh Date: Tue, 26 Nov 2024 11:57:09 +0800 Subject: [PATCH 3/3] add test Signed-off-by: smartlxh --- test/sql/test_lake_sync_mv/R/test_lake_sync_mv | 6 ++++++ test/sql/test_lake_sync_mv/T/test_lake_sync_mv | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/sql/test_lake_sync_mv/R/test_lake_sync_mv b/test/sql/test_lake_sync_mv/R/test_lake_sync_mv index dea9989a7fc66..7350e3bdfb07e 100644 --- a/test/sql/test_lake_sync_mv/R/test_lake_sync_mv +++ b/test/sql/test_lake_sync_mv/R/test_lake_sync_mv @@ -29,4 +29,10 @@ None INSERT INTO tbl1 VALUES("2020-01-11",6,100); -- result: +-- !result + +SELECT mv_sum_v1 FROM sync_mv [_SYNC_MV_] ORDER BY k1; +-- result: +300 +100 -- !result \ No newline at end of file diff --git a/test/sql/test_lake_sync_mv/T/test_lake_sync_mv b/test/sql/test_lake_sync_mv/T/test_lake_sync_mv index d78905a01dab5..8f44212972790 100644 --- a/test/sql/test_lake_sync_mv/T/test_lake_sync_mv +++ b/test/sql/test_lake_sync_mv/T/test_lake_sync_mv @@ -18,4 +18,6 @@ VALUES CREATE MATERIALIZED VIEW sync_mv AS SELECT k1, SUM(v1) FROM tbl1 GROUP BY k1; function: wait_materialized_view_finish() -INSERT INTO tbl1 VALUES("2020-01-11",6,100); \ No newline at end of file +INSERT INTO tbl1 VALUES("2020-01-11",6,100); + +SELECT mv_sum_v1 FROM sync_mv [_SYNC_MV_] ORDER BY k1; \ No newline at end of file