From b88a88f8cde19a78b39d5999726c0dd8e2d2f884 Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 25 Apr 2024 10:11:56 +0800 Subject: [PATCH 001/163] Fix remote scan pool (#33976) --- .../runtime/workload_group/workload_group.cpp | 31 ++++----- be/src/vec/exec/scan/scanner_scheduler.cpp | 24 +++++-- be/src/vec/exec/scan/scanner_scheduler.h | 26 +++++--- .../resource/workloadgroup/WorkloadGroup.java | 11 ++++ .../workload_manager_p0/test_curd_wlg.out | 24 +++++++ .../workload_manager_p0/test_curd_wlg.groovy | 64 +++++++++++++++++++ 6 files changed, 145 insertions(+), 35 deletions(-) diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index 673263f1a17d2e..c82346f040ec82 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -295,7 +295,8 @@ Status WorkloadGroupInfo::parse_topic_info(const TWorkloadGroupInfo& tworkload_g } // 10 max remote scan thread num - workload_group_info->max_remote_scan_thread_num = config::doris_scanner_thread_pool_thread_num; + workload_group_info->max_remote_scan_thread_num = + vectorized::ScannerScheduler::get_remote_scan_thread_num(); if (tworkload_group_info.__isset.max_remote_scan_thread_num && tworkload_group_info.max_remote_scan_thread_num > 0) { workload_group_info->max_remote_scan_thread_num = @@ -303,7 +304,8 @@ Status WorkloadGroupInfo::parse_topic_info(const TWorkloadGroupInfo& tworkload_g } // 11 min remote scan thread num - workload_group_info->min_remote_scan_thread_num = config::doris_scanner_thread_pool_thread_num; + workload_group_info->min_remote_scan_thread_num = + vectorized::ScannerScheduler::get_remote_scan_thread_num(); if (tworkload_group_info.__isset.min_remote_scan_thread_num && tworkload_group_info.min_remote_scan_thread_num > 0) { workload_group_info->min_remote_scan_thread_num = @@ -384,23 +386,18 @@ void WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e } } if (scan_thread_num > 0 && _scan_task_sched) { - _scan_task_sched->reset_thread_num(scan_thread_num); + _scan_task_sched->reset_thread_num(scan_thread_num, scan_thread_num); } if (_remote_scan_task_sched == nullptr) { - int remote_max_thread_num = - config::doris_max_remote_scanner_thread_pool_thread_num != -1 - ? config::doris_max_remote_scanner_thread_pool_thread_num - : std::max(512, CpuInfo::num_cores() * 10); - remote_max_thread_num = - std::max(remote_max_thread_num, config::doris_scanner_thread_pool_thread_num); - + int remote_max_thread_num = vectorized::ScannerScheduler::get_remote_scan_thread_num(); + int remote_scan_thread_queue_size = + vectorized::ScannerScheduler::get_remote_scan_thread_queue_size(); std::unique_ptr remote_scan_scheduler = std::make_unique("RScan_" + tg_name, cg_cpu_ctl_ptr); - Status ret = - remote_scan_scheduler->start(remote_max_thread_num, remote_max_thread_num, - config::doris_remote_scanner_thread_pool_queue_size); + Status ret = remote_scan_scheduler->start(remote_max_thread_num, remote_max_thread_num, + remote_scan_thread_queue_size); if (ret.ok()) { _remote_scan_task_sched = std::move(remote_scan_scheduler); } else { @@ -408,11 +405,9 @@ void WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e << tg_id; } } - if (max_remote_scan_thread_num > 0 && _remote_scan_task_sched) { - _remote_scan_task_sched->reset_max_thread_num(max_remote_scan_thread_num); - } - if (min_remote_scan_thread_num > 0 && _remote_scan_task_sched) { - _remote_scan_task_sched->reset_min_thread_num(min_remote_scan_thread_num); + if (max_remote_scan_thread_num >= min_remote_scan_thread_num && _remote_scan_task_sched) { + _remote_scan_task_sched->reset_thread_num(max_remote_scan_thread_num, + min_remote_scan_thread_num); } if (_non_pipe_thread_pool == nullptr) { diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp index 571df35e55ed85..eba62dcf19a1ea 100644 --- a/be/src/vec/exec/scan/scanner_scheduler.cpp +++ b/be/src/vec/exec/scan/scanner_scheduler.cpp @@ -97,14 +97,10 @@ Status ScannerScheduler::init(ExecEnv* env) { config::doris_scanner_thread_pool_queue_size, "local_scan"); // 2. remote scan thread pool - _remote_thread_pool_max_size = config::doris_max_remote_scanner_thread_pool_thread_num != -1 - ? config::doris_max_remote_scanner_thread_pool_thread_num - : std::max(512, CpuInfo::num_cores() * 10); - _remote_thread_pool_max_size = - std::max(_remote_thread_pool_max_size, config::doris_scanner_thread_pool_thread_num); + _remote_thread_pool_max_size = ScannerScheduler::get_remote_scan_thread_num(); + int remote_scan_pool_queue_size = ScannerScheduler::get_remote_scan_thread_queue_size(); _remote_scan_thread_pool = std::make_unique( - _remote_thread_pool_max_size, config::doris_remote_scanner_thread_pool_queue_size, - "RemoteScanThreadPool"); + _remote_thread_pool_max_size, remote_scan_pool_queue_size, "RemoteScanThreadPool"); // 3. limited scan thread pool static_cast(ThreadPoolBuilder("LimitedScanThreadPool") @@ -329,4 +325,18 @@ void ScannerScheduler::_deregister_metrics() { DEREGISTER_HOOK_METRIC(group_local_scan_thread_pool_queue_size); DEREGISTER_HOOK_METRIC(group_local_scan_thread_pool_thread_num); } + +int ScannerScheduler::get_remote_scan_thread_num() { + int remote_max_thread_num = config::doris_max_remote_scanner_thread_pool_thread_num != -1 + ? config::doris_max_remote_scanner_thread_pool_thread_num + : std::max(512, CpuInfo::num_cores() * 10); + remote_max_thread_num = + std::max(remote_max_thread_num, config::doris_scanner_thread_pool_thread_num); + return remote_max_thread_num; +} + +int ScannerScheduler::get_remote_scan_thread_queue_size() { + return config::doris_remote_scanner_thread_pool_queue_size; +} + } // namespace doris::vectorized diff --git a/be/src/vec/exec/scan/scanner_scheduler.h b/be/src/vec/exec/scan/scanner_scheduler.h index 01e0859643409a..b3d02860f9a3d4 100644 --- a/be/src/vec/exec/scan/scanner_scheduler.h +++ b/be/src/vec/exec/scan/scanner_scheduler.h @@ -65,6 +65,10 @@ class ScannerScheduler { int remote_thread_pool_max_size() const { return _remote_thread_pool_max_size; } + static int get_remote_scan_thread_num(); + + static int get_remote_scan_thread_queue_size(); + private: static void _scanner_scan(std::shared_ptr ctx, std::shared_ptr scan_task); @@ -136,16 +140,18 @@ class SimplifiedScanScheduler { } } - void reset_thread_num(int thread_num) { - int max_thread_num = _scan_thread_pool->max_threads(); - if (max_thread_num != thread_num) { - if (thread_num > max_thread_num) { - static_cast(_scan_thread_pool->set_max_threads(thread_num)); - static_cast(_scan_thread_pool->set_min_threads(thread_num)); - } else { - static_cast(_scan_thread_pool->set_min_threads(thread_num)); - static_cast(_scan_thread_pool->set_max_threads(thread_num)); - } + void reset_thread_num(int new_max_thread_num, int new_min_thread_num) { + int cur_max_thread_num = _scan_thread_pool->max_threads(); + int cur_min_thread_num = _scan_thread_pool->min_threads(); + if (cur_max_thread_num == new_max_thread_num && cur_min_thread_num == new_min_thread_num) { + return; + } + if (new_max_thread_num >= cur_max_thread_num) { + static_cast(_scan_thread_pool->set_max_threads(new_max_thread_num)); + static_cast(_scan_thread_pool->set_min_threads(new_min_thread_num)); + } else { + static_cast(_scan_thread_pool->set_min_threads(new_min_thread_num)); + static_cast(_scan_thread_pool->set_max_threads(new_max_thread_num)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java index 482d2f6f11a301..e5ec2c619b6b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java @@ -272,6 +272,7 @@ private static void checkProperties(Map properties) throws DdlEx } } + int maxRemoteScanNum = -1; if (properties.containsKey(MAX_REMOTE_SCAN_THREAD_NUM)) { String value = properties.get(MAX_REMOTE_SCAN_THREAD_NUM); try { @@ -279,12 +280,14 @@ private static void checkProperties(Map properties) throws DdlEx if (intValue <= 0 && intValue != -1) { throw new NumberFormatException(); } + maxRemoteScanNum = intValue; } catch (NumberFormatException e) { throw new DdlException( MAX_REMOTE_SCAN_THREAD_NUM + " must be a positive integer or -1. but input value is " + value); } } + int minRemoteScanNum = -1; if (properties.containsKey(MIN_REMOTE_SCAN_THREAD_NUM)) { String value = properties.get(MIN_REMOTE_SCAN_THREAD_NUM); try { @@ -292,12 +295,20 @@ private static void checkProperties(Map properties) throws DdlEx if (intValue <= 0 && intValue != -1) { throw new NumberFormatException(); } + minRemoteScanNum = intValue; } catch (NumberFormatException e) { throw new DdlException( MIN_REMOTE_SCAN_THREAD_NUM + " must be a positive integer or -1. but input value is " + value); } } + if ((maxRemoteScanNum == -1 && minRemoteScanNum != -1) || (maxRemoteScanNum != -1 && minRemoteScanNum == -1)) { + throw new DdlException(MAX_REMOTE_SCAN_THREAD_NUM + " and " + MIN_REMOTE_SCAN_THREAD_NUM + + " must be specified simultaneously"); + } else if (maxRemoteScanNum < minRemoteScanNum) { + throw new DdlException(MAX_REMOTE_SCAN_THREAD_NUM + " must bigger or equal " + MIN_REMOTE_SCAN_THREAD_NUM); + } + // check queue property if (properties.containsKey(MAX_CONCURRENCY)) { try { diff --git a/regression-test/data/workload_manager_p0/test_curd_wlg.out b/regression-test/data/workload_manager_p0/test_curd_wlg.out index fca16d077e4095..876be32601a5b1 100644 --- a/regression-test/data/workload_manager_p0/test_curd_wlg.out +++ b/regression-test/data/workload_manager_p0/test_curd_wlg.out @@ -64,3 +64,27 @@ tag1_wg1 0% 10% tag1 tag1_wg2 0% 10% tag1 tag1_wg3 0% 80% tag1 +-- !select_remote_scan_num -- +20 10 + +-- !select_remote_scan_num_2 -- +21 10 + +-- !select_remote_scan_num_3 -- +21 2 + +-- !select_remote_scan_num_4 -- +40 20 + +-- !select_remote_scan_num_5 -- +10 5 + +-- !select_remote_scan_num_6 -- +3 3 + +-- !select_remote_scan_num_7 -- +10 5 + +-- !select_remote_scan_num_8 -- +-1 -1 + diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy index 875eeb668e2a37..05034529726302 100644 --- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy +++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy @@ -528,6 +528,70 @@ suite("test_crud_wlg") { sql "set bypass_workload_group = true;" sql "select count(1) from information_schema.active_queries;" + // test set remote scan pool + sql "drop workload group if exists test_remote_scan_wg;" + test { + sql "create workload group test_remote_scan_wg properties('min_remote_scan_thread_num'='123');" + exception "must be specified simultaneously" + } + + test { + sql "create workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='123');" + exception "must be specified simultaneously" + } + + test { + sql "create workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='10', 'min_remote_scan_thread_num'='123');" + exception "must bigger or equal " + } + + sql "create workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='20', 'min_remote_scan_thread_num'='10');" + qt_select_remote_scan_num "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='21')" + qt_select_remote_scan_num_2 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + test { + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='5')" + exception "must bigger or equal" + } + + sql "alter workload group test_remote_scan_wg properties('min_remote_scan_thread_num'='2')" + qt_select_remote_scan_num_3 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + test { + sql "alter workload group test_remote_scan_wg properties('min_remote_scan_thread_num'='30')" + exception "must bigger or equal" + } + + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='40', 'min_remote_scan_thread_num'='20')" + qt_select_remote_scan_num_4 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='10', 'min_remote_scan_thread_num'='5')" + qt_select_remote_scan_num_5 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='3', 'min_remote_scan_thread_num'='3')" + qt_select_remote_scan_num_6 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + sql "drop workload group test_remote_scan_wg;" + sql "create workload group test_remote_scan_wg properties('cpu_share'='1024');" + test { + sql "alter workload group test_remote_scan_wg properties('min_remote_scan_thread_num'='30')" + exception "must be specified simultaneously" + } + + test { + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='30')" + exception "must be specified simultaneously" + } + + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='10', 'min_remote_scan_thread_num'='5')" + qt_select_remote_scan_num_7 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + + sql "alter workload group test_remote_scan_wg properties('max_remote_scan_thread_num'='-1', 'min_remote_scan_thread_num'='-1')" + qt_select_remote_scan_num_8 "select MAX_REMOTE_SCAN_THREAD_NUM,MIN_REMOTE_SCAN_THREAD_NUM from information_schema.workload_groups where name='test_remote_scan_wg';" + sql "drop workload group test_remote_scan_wg" + sql "drop workload group tag1_wg1;" sql "drop workload group tag1_wg2;" sql "drop workload group if exists tag2_wg1;" From 6b9f1328a7622368fd62412539607085d97f1880 Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 25 Apr 2024 10:13:22 +0800 Subject: [PATCH 002/163] Fix stream load can only use default group (#33875) --- .../java/org/apache/doris/service/FrontendServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index d5f0a5aafd7939..6b60e01d1c02a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -1917,8 +1917,8 @@ public TStreamLoadPutResult streamLoadPut(TStreamLoadPutRequest request) { // mysql load request not carry user info, need fix it later. boolean hasUserName = !StringUtils.isEmpty(request.getUser()); if (Config.enable_workload_group && hasUserName) { - ConnectContext ctx = ConnectContext.get(); - tWorkloadGroupList = Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroup(ctx); + UserIdentity userIdentity = UserIdentity.createAnalyzedUserIdentWithIp(request.getUser(), "%"); + tWorkloadGroupList = Env.getCurrentEnv().getWorkloadGroupMgr().getWorkloadGroupByUser(userIdentity); } if (!Strings.isNullOrEmpty(request.getLoadSql())) { httpStreamPutImpl(request, result); From 3034ac3fe29dab050cbe9572f1db26635a18ec0d Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 10:27:19 +0800 Subject: [PATCH 003/163] [chore](config) Add config to control BufferedReader and S3FileWriter's thread pool's min max nums (#33974) --- be/src/common/config.cpp | 14 ++++++++--- be/src/common/config.h | 14 ++++++++--- .../io/cache/block_file_cache_downloader.cpp | 2 +- be/src/io/fs/s3_file_writer.cpp | 2 +- be/src/runtime/exec_env_init.cpp | 25 ++++++++++++++++--- 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9df1a184f609ef..de1458c240dff3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1025,9 +1025,8 @@ DEFINE_mInt32(tablet_path_check_batch_size, "1000"); DEFINE_mInt64(row_column_page_size, "4096"); // it must be larger than or equal to 5MB DEFINE_mInt64(s3_write_buffer_size, "5242880"); -DEFINE_mInt32(s3_task_check_interval, "60"); -// The timeout config for S3 buffer allocation -DEFINE_mInt32(s3_writer_buffer_allocation_timeout, "300"); +// Log interval when doing s3 upload task +DEFINE_mInt32(s3_file_writer_log_interval_second, "60"); DEFINE_mInt64(file_cache_max_file_reader_cache_size, "1000000"); DEFINE_mInt64(hdfs_write_batch_buffer_size_mb, "4"); // 4MB @@ -1217,6 +1216,15 @@ DEFINE_mBool(enable_injection_point, "false"); DEFINE_mBool(ignore_schema_change_check, "false"); +// The min thread num for BufferedReaderPrefetchThreadPool +DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread, "16"); +// The max thread num for BufferedReaderPrefetchThreadPool +DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread, "64"); +// The min thread num for S3FileUploadThreadPool +DEFINE_Int64(num_s3_file_upload_thread_pool_min_thread, "16"); +// The max thread num for S3FileUploadThreadPool +DEFINE_Int64(num_s3_file_upload_thread_pool_max_thread, "64"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 81910dd2553fad..4139d76b6bcb7a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1070,9 +1070,8 @@ DECLARE_mInt32(tablet_path_check_batch_size); DECLARE_mInt64(row_column_page_size); // it must be larger than or equal to 5MB DECLARE_mInt64(s3_write_buffer_size); -DECLARE_mInt32(s3_task_check_interval); -// The timeout config for S3 buffer allocation -DECLARE_mInt32(s3_writer_buffer_allocation_timeout); +// Log interval when doing s3 upload task +DECLARE_mInt32(s3_file_writer_log_interval_second); // the max number of cached file handle for block segemnt DECLARE_mInt64(file_cache_max_file_reader_cache_size); DECLARE_mInt64(hdfs_write_batch_buffer_size_mb); @@ -1296,6 +1295,15 @@ DECLARE_mBool(enable_injection_point); DECLARE_mBool(ignore_schema_change_check); +// The min thread num for BufferedReaderPrefetchThreadPool +DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread); +// The max thread num for BufferedReaderPrefetchThreadPool +DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread); +// The min thread num for S3FileUploadThreadPool +DECLARE_Int64(num_s3_file_upload_thread_pool_min_thread); +// The max thread num for S3FileUploadThreadPool +DECLARE_Int64(num_s3_file_upload_thread_pool_max_thread); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index 30fb3a8633891c..283605f23bed91 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -184,7 +184,7 @@ struct DownloadTaskExecutor { LOG_WARNING("").error(st); } } - auto timeout_duration = config::s3_task_check_interval; + auto timeout_duration = config::s3_file_writer_log_interval_second; timespec current_time; // We don't need high accuracy here, so we use time(nullptr) // since it's the fastest way to get current time(second) diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 9df1ac847af365..84487f496ac1e9 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -135,7 +135,7 @@ Status S3FileWriter::_create_multi_upload_request() { } void S3FileWriter::_wait_until_finish(std::string_view task_name) { - auto timeout_duration = config::s3_writer_buffer_allocation_timeout; + auto timeout_duration = config::s3_file_writer_log_interval_second; auto msg = fmt::format( "{} multipart upload already takes {} seconds, bucket={}, key={}, upload_id={}", task_name, timeout_duration, _bucket, _path.native(), _upload_id); diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 5cbb5829ee0ac1..5a7e39cf158c41 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -142,6 +142,17 @@ static void init_doris_metrics(const std::vector& store_paths) { DorisMetrics::instance()->initialize(init_system_metrics, disk_devices, network_interfaces); } +// Used to calculate the num of min thread and max thread based on the passed config +static pair get_num_threads(size_t min_num, size_t max_num) { + auto num_cores = doris::CpuInfo::num_cores(); + min_num = (min_num == 0) ? num_cores : min_num; + max_num = (max_num == 0) ? num_cores : max_num; + auto factor = max_num / min_num; + min_num = std::min(num_cores * factor, min_num); + max_num = std::min(min_num * factor, max_num); + return {min_num, max_num}; +} + Status ExecEnv::init(ExecEnv* env, const std::vector& store_paths, const std::vector& spill_store_paths, const std::set& broken_paths) { @@ -184,9 +195,12 @@ Status ExecEnv::_init(const std::vector& store_paths, .set_max_queue_size(config::send_batch_thread_pool_queue_size) .build(&_send_batch_thread_pool)); + auto [buffered_reader_min_threads, buffered_reader_max_threads] = + get_num_threads(config::num_buffered_reader_prefetch_thread_pool_min_thread, + config::num_buffered_reader_prefetch_thread_pool_max_thread); static_cast(ThreadPoolBuilder("BufferedReaderPrefetchThreadPool") - .set_min_threads(16) - .set_max_threads(64) + .set_min_threads(buffered_reader_min_threads) + .set_max_threads(buffered_reader_max_threads) .build(&_buffered_reader_prefetch_thread_pool)); static_cast(ThreadPoolBuilder("SendTableStatsThreadPool") @@ -199,9 +213,12 @@ Status ExecEnv::_init(const std::vector& store_paths, .set_max_threads(16) .build(&_s3_downloader_download_poller_thread_pool)); + auto [s3_file_upload_min_threads, s3_file_upload_max_threads] = + get_num_threads(config::num_s3_file_upload_thread_pool_min_thread, + config::num_s3_file_upload_thread_pool_max_thread); static_cast(ThreadPoolBuilder("S3FileUploadThreadPool") - .set_min_threads(16) - .set_max_threads(64) + .set_min_threads(s3_file_upload_min_threads) + .set_max_threads(s3_file_upload_max_threads) .build(&_s3_file_upload_thread_pool)); // min num equal to fragment pool's min num From 440f60738afba27e4393e74a3461d2bd02843a61 Mon Sep 17 00:00:00 2001 From: Lightman <31928846+Lchangliang@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:33:13 +0800 Subject: [PATCH 004/163] (cloud-merge) Fix create mv failed bacause expr is null (#34010) --- .../src/main/java/org/apache/doris/alter/CloudRollupJobV2.java | 3 ++- .../src/main/java/org/apache/doris/alter/RollupJobV2.java | 2 +- .../java/org/apache/doris/cloud/CacheHotspotManagerUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index 51c4abde769bb4..1474de4d22f858 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -55,7 +55,7 @@ public class CloudRollupJobV2 extends RollupJobV2 { private static final Logger LOG = LogManager.getLogger(CloudRollupJobV2.class); - public static AlterJobV2 buildCloudRollupJobV2(RollupJobV2 job) throws IllegalAccessException { + public static AlterJobV2 buildCloudRollupJobV2(RollupJobV2 job) throws IllegalAccessException, AnalysisException { CloudRollupJobV2 ret = new CloudRollupJobV2(); List allFields = new ArrayList<>(); Class tmpClass = RollupJobV2.class; @@ -70,6 +70,7 @@ public static AlterJobV2 buildCloudRollupJobV2(RollupJobV2 job) throws IllegalAc field.set(ret, field.get(job)); } } + ret.initAnalyzer(); return ret; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index 60c02b6b5252b5..ec0868637e781e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -183,7 +183,7 @@ public void setStorageFormat(TStorageFormat storageFormat) { this.storageFormat = storageFormat; } - private void initAnalyzer() throws AnalysisException { + protected void initAnalyzer() throws AnalysisException { ConnectContext connectContext = new ConnectContext(); Database db; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java index 796b2f99560782..e3fff32035dbcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java @@ -48,7 +48,7 @@ public class CacheHotspotManagerUtils { FeConstants.INTERNAL_DB_NAME, FeConstants.INTERNAL_FILE_CACHE_HOTSPOT_TABLE_NAME); // TODO(yuejing): 如何加字段 private static final String CREATE_CACHE_TABLE = - "create table " + TABLE_NAME + " IF NOT EXISTS (\n" + "create table " + TABLE_NAME + " (\n" + " cluster_id varchar(65530),\n" + " backend_id bigint,\n" + " table_id bigint,\n" From 4e7307d8043a43c731844f17444eabe403e8c09f Mon Sep 17 00:00:00 2001 From: walter Date: Thu, 25 Apr 2024 11:09:12 +0800 Subject: [PATCH 005/163] [fix](fe) Fix SHOW CREATE TABLE with AUTO PARTITION (#34071) AUTO PARTITION grammar has changed since #31585, but the output of SHOW CREATE TABLE was left out to change, so the result is not able to be recognized by the FE parser. --- .../doris/catalog/RangePartitionInfo.java | 10 +---- .../test_create_table_auto_partition.groovy | 45 +++++++++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 regression-test/suites/ddl_p0/test_create_table_auto_partition.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java index 952fa88d259292..9a6c5d353fd78a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java @@ -23,7 +23,6 @@ import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.analysis.RangePartitionDesc; import org.apache.doris.analysis.SinglePartitionDesc; -import org.apache.doris.analysis.SlotRef; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.RangeUtils; @@ -266,14 +265,9 @@ public String toSql(OlapTable table, List partitionId) { if (enableAutomaticPartition()) { sb.append("AUTO PARTITION BY RANGE "); for (Expr e : partitionExprs) { - boolean isSlotRef = (e instanceof SlotRef); - if (isSlotRef) { - sb.append("("); - } + sb.append("("); sb.append(e.toSql()); - if (isSlotRef) { - sb.append(")"); - } + sb.append(")"); } sb.append("\n("); } else { diff --git a/regression-test/suites/ddl_p0/test_create_table_auto_partition.groovy b/regression-test/suites/ddl_p0/test_create_table_auto_partition.groovy new file mode 100644 index 00000000000000..a9ca68a87771ba --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_auto_partition.groovy @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// this suite is for creating table with timestamp datatype in defferent +// case. For example: 'year' and 'Year' datatype should also be valid in definition + + +suite("test_create_table_auto_partition") { + def testTable = "test_create_table_auto_partition_table" + + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE `${testTable}` ( + `TIME_STAMP` datev2 NOT NULL COMMENT 'Date of collection' + ) ENGINE=OLAP + DUPLICATE KEY(`TIME_STAMP`) + AUTO PARTITION BY RANGE (date_trunc(`TIME_STAMP`, 'month')) + ( + ) + DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // The AUTO PARTITION func call must wrapped with (). + def text = sql_return_maparray "show create table ${testTable}" + def createTable = text[0]['Create Table'] + assertTrue(createTable.contains("AUTO PARTITION BY RANGE (date_trunc(`TIME_STAMP`, 'month')")) +} + From 3165b5ea3821b09af0f4dcf2ec4a2e2015bd4dde Mon Sep 17 00:00:00 2001 From: zzzxl <33418555+zzzxl1993@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:10:50 +0800 Subject: [PATCH 006/163] [fix](inverted index) the rowset may be deleted and become nullptr.(#33878) 1. between compaction prepare and execute, a rowset may be deleted by cold down, leading to a nullptr exception 2. intermittent bug that cannot be replicated with a test case --- be/src/olap/compaction.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 55af708c759fe8..061d667df8bb68 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -529,11 +529,9 @@ Status Compaction::do_inverted_index_compaction() { // src index files // format: rowsetId_segmentId std::vector src_index_files(src_segment_num); - std::vector src_rowset_ids; for (const auto& m : src_seg_to_id_map) { std::pair p = m.first; src_index_files[m.second] = p.first.to_string() + "_" + std::to_string(p.second); - src_rowset_ids.push_back(p.first); } // dest index files @@ -671,9 +669,8 @@ Status Compaction::do_inverted_index_compaction() { // if index properties are different, index compaction maybe needs to be skipped. bool is_continue = false; std::optional> first_properties; - for (const auto& rowset_id : src_rowset_ids) { - auto rowset_ptr = _tablet->get_rowset(rowset_id); - const auto* tablet_index = rowset_ptr->tablet_schema()->get_inverted_index(col); + for (const auto& rowset : _input_rowsets) { + const auto* tablet_index = rowset->tablet_schema()->get_inverted_index(col); const auto& properties = tablet_index->properties(); if (!first_properties.has_value()) { first_properties = properties; From 93f9a2c5bcd0cc3aa2803857c422978de1678dc4 Mon Sep 17 00:00:00 2001 From: yujun Date: Thu, 25 Apr 2024 11:13:01 +0800 Subject: [PATCH 007/163] [fix](tablet invert index) fix tablet invert index leaky caused by auto partition (#33973) --- .../doris/datasource/InternalCatalog.java | 24 ++++---- .../doris/alter/AddExistsPartitionTest.java | 56 +++++++++++++++++++ .../doris/utframe/TestWithFeService.java | 3 +- 3 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/alter/AddExistsPartitionTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 10983a955b79c1..dd52fad4f7f68c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1466,8 +1466,10 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa // check partition name if (olapTable.checkPartitionNameExist(partitionName)) { if (singlePartitionDesc.isSetIfNotExists()) { - LOG.info("add partition[{}] which already exists", partitionName); - return; + LOG.info("table[{}] add partition[{}] which already exists", olapTable.getName(), partitionName); + if (!DebugPointUtil.isEnable("InternalCatalog.addPartition.noCheckExists")) { + return; + } } else { ErrorReport.reportDdlException(ErrorCode.ERR_SAME_NAME_PARTITION, partitionName); } @@ -1624,6 +1626,11 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa if (!Strings.isNullOrEmpty(dataProperty.getStoragePolicy())) { storagePolicy = dataProperty.getStoragePolicy(); } + Runnable failedCleanCallback = () -> { + for (Long tabletId : tabletIdSet) { + Env.getCurrentInvertedIndex().deleteTablet(tabletId); + } + }; try { long partitionId = idGeneratorBuffer.getNextId(); List partitionIds = Lists.newArrayList(partitionId); @@ -1646,8 +1653,9 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa olapTable.checkNormalStateForAlter(); // check partition name if (olapTable.checkPartitionNameExist(partitionName)) { + LOG.info("table[{}] add partition[{}] which already exists", olapTable.getName(), partitionName); if (singlePartitionDesc.isSetIfNotExists()) { - LOG.info("add partition[{}] which already exists", partitionName); + failedCleanCallback.run(); return; } else { ErrorReport.reportDdlException(ErrorCode.ERR_SAME_NAME_PARTITION, partitionName); @@ -1696,8 +1704,6 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa } } - - if (metaChanged) { throw new DdlException("Table[" + tableName + "]'s meta has been changed. try again."); } @@ -1741,9 +1747,7 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa olapTable.writeUnlock(); } } catch (DdlException e) { - for (Long tabletId : tabletIdSet) { - Env.getCurrentInvertedIndex().deleteTablet(tabletId); - } + failedCleanCallback.run(); throw e; } } @@ -2844,10 +2848,10 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep Env.getCurrentEnv().getEditLog().logColocateAddTable(info); } LOG.info("successfully create table[{};{}]", tableName, tableId); - // register or remove table from DynamicPartition after table created - DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable, false); Env.getCurrentEnv().getDynamicPartitionScheduler() .executeDynamicPartitionFirstTime(db.getId(), olapTable.getId()); + // register or remove table from DynamicPartition after table created + DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable, false); Env.getCurrentEnv().getDynamicPartitionScheduler() .createOrUpdateRuntimeInfo(tableId, DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AddExistsPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AddExistsPartitionTest.java new file mode 100644 index 00000000000000..0d95ee30cdeb20 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AddExistsPartitionTest.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.DebugPointUtil; +import org.apache.doris.common.util.DebugPointUtil.DebugPoint; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class AddExistsPartitionTest extends TestWithFeService { + + @Override + protected void beforeCreatingConnectContext() throws Exception { + Config.enable_debug_points = true; + } + + @Test + public void testAddExistsPartition() throws Exception { + DebugPointUtil.addDebugPoint("InternalCatalog.addPartition.noCheckExists", new DebugPoint()); + createDatabase("test"); + createTable("CREATE TABLE test.tbl (k INT) DISTRIBUTED BY HASH(k) " + + " BUCKETS 5 PROPERTIES ( \"replication_num\" = \"" + backendNum() + "\" )"); + List backendIds = Env.getCurrentSystemInfo().getAllBackendIds(); + for (long backendId : backendIds) { + Assertions.assertEquals(5, Env.getCurrentInvertedIndex().getTabletIdsByBackendId(backendId).size()); + } + + String addPartitionSql = "ALTER TABLE test.tbl ADD PARTITION IF NOT EXISTS tbl" + + " DISTRIBUTED BY HASH(k) BUCKETS 5"; + Assertions.assertNotNull(getSqlStmtExecutor(addPartitionSql)); + for (long backendId : backendIds) { + Assertions.assertEquals(5, Env.getCurrentInvertedIndex().getTabletIdsByBackendId(backendId).size()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index b590234a3e83e9..063ab21d8bcf62 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -583,7 +583,8 @@ public StmtExecutor getSqlStmtExecutor(String queryStr) throws Exception { connectContext.getState().reset(); StmtExecutor stmtExecutor = new StmtExecutor(connectContext, queryStr); stmtExecutor.execute(); - if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR) { + if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR + && connectContext.getState().getErrorCode() == null) { return stmtExecutor; } else { return null; From 8ffb170462028abd40fe166b9df8687f032bb020 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 11:20:41 +0800 Subject: [PATCH 008/163] [feature](Cloud) Implement gcs accessor for compatibility (#34081) --- cloud/src/recycler/s3_accessor.cpp | 33 ++++++++- cloud/src/recycler/s3_accessor.h | 9 +++ cloud/test/s3_accessor_test.cpp | 103 ++++++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 3 deletions(-) diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 543f84f87fc420..d1ebfe62a1d781 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -30,6 +30,8 @@ #include #include +#include +#include #include #include "common/logging.h" @@ -226,7 +228,21 @@ int S3Accessor::delete_objects(const std::vector& relative_paths) { } int S3Accessor::delete_object(const std::string& relative_path) { - // TODO(cyx) + Aws::S3::Model::DeleteObjectRequest request; + auto key = get_key(relative_path); + request.WithBucket(conf_.bucket).WithKey(key); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(s3_client_->DeleteObject(request), + "s3_client::delete_object", request); + if (!outcome.IsSuccess()) { + LOG_WARNING("failed to delete object") + .tag("endpoint", conf_.endpoint) + .tag("bucket", conf_.bucket) + .tag("key", key) + .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) + .tag("error", outcome.GetError().GetMessage()) + .tag("exception", outcome.GetError().GetExceptionName()); + return -1; + } return 0; } @@ -422,5 +438,20 @@ int S3Accessor::check_bucket_versioning() { return 0; } +int GcsAccessor::delete_objects(const std::vector& relative_paths) { + std::vector delete_rets(relative_paths.size()); + std::transform(std::execution::par, relative_paths.begin(), relative_paths.end(), + delete_rets.begin(), + [this](const std::string& path) { return delete_object(path); }); + int ret = 0; + for (int delete_ret : delete_rets) { + if (delete_ret != 0) { + ret = delete_ret; + break; + } + } + return ret; +} + #undef HELP_MACRO } // namespace doris::cloud diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h index 10291cfd4bad68..1025ceab52e1f7 100644 --- a/cloud/src/recycler/s3_accessor.h +++ b/cloud/src/recycler/s3_accessor.h @@ -90,4 +90,13 @@ class S3Accessor : public ObjStoreAccessor { std::string path_; }; +class GcsAccessor final : public S3Accessor { +public: + explicit GcsAccessor(S3Conf conf) : S3Accessor(std::move(conf)) {} + ~GcsAccessor() override = default; + + // returns 0 for success otherwise error + int delete_objects(const std::vector& relative_paths) override; +}; + } // namespace doris::cloud diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp index bb8b7c27bd989d..972505c3999e09 100644 --- a/cloud/test/s3_accessor_test.cpp +++ b/cloud/test/s3_accessor_test.cpp @@ -58,6 +58,8 @@ class S3ClientInterface { const Aws::S3::Model::ListObjectsV2Request& req) = 0; virtual Aws::S3::Model::DeleteObjectsOutcome DeleteObjects( const Aws::S3::Model::DeleteObjectsRequest& req) = 0; + virtual Aws::S3::Model::DeleteObjectOutcome DeleteObject( + const Aws::S3::Model::DeleteObjectRequest& req) = 0; virtual Aws::S3::Model::PutObjectOutcome PutObject( const Aws::S3::Model::PutObjectRequest& req) = 0; virtual Aws::S3::Model::HeadObjectOutcome HeadObject( @@ -122,6 +124,13 @@ class S3Client : public S3ClientInterface { return Aws::S3::Model::DeleteObjectsOutcome(std::move(result)); } + Aws::S3::Model::DeleteObjectOutcome DeleteObject( + const Aws::S3::Model::DeleteObjectRequest& req) override { + Aws::S3::Model::DeleteObjectResult result; + _mock_fs->delete_object(req.GetKey()); + return Aws::S3::Model::DeleteObjectOutcome(std::move(result)); + } + Aws::S3::Model::PutObjectOutcome PutObject( const Aws::S3::Model::PutObjectRequest& req) override { Aws::S3::Model::PutObjectResult result; @@ -207,6 +216,18 @@ class ErrorS3Client : public S3ClientInterface { return Aws::S3::Model::DeleteObjectsOutcome(std::move(err)); } + Aws::S3::Model::DeleteObjectOutcome DeleteObject( + const Aws::S3::Model::DeleteObjectRequest& req) override { + if (!return_error_for_error_s3_client) { + return _correct_impl->DeleteObject(req); + } + auto err = Aws::Client::AWSError(Aws::S3::S3Errors::RESOURCE_NOT_FOUND, + false); + err.SetResponseCode(Aws::Http::HttpResponseCode::NOT_FOUND); + // return -1 + return Aws::S3::Model::DeleteObjectOutcome(std::move(err)); + } + Aws::S3::Model::PutObjectOutcome PutObject( const Aws::S3::Model::PutObjectRequest& req) override { if (!return_error_for_error_s3_client) { @@ -267,6 +288,10 @@ class MockS3Client { return _impl->DeleteObjects(req); } + auto DeleteObject(const Aws::S3::Model::DeleteObjectRequest& req) { + return _impl->DeleteObject(req); + } + auto PutObject(const Aws::S3::Model::PutObjectRequest& req) { return _impl->PutObject(req); } auto HeadObject(const Aws::S3::Model::HeadObjectRequest& req) { return _impl->HeadObject(req); } @@ -304,6 +329,12 @@ static auto callbacks = std::array { Aws::S3::Model::DeleteObjectsRequest*>*)p; *pair.first = (*_mock_client).DeleteObjects(*pair.second); }}, + MockCallable {"s3_client::delete_object", + [](void* p) { + auto pair = *(std::pair*)p; + *pair.first = (*_mock_client).DeleteObject(*pair.second); + }}, MockCallable {"s3_client::put_object", [](void* p) { auto pair = *(std::pairexist(prefix)); } -// function is not implemented -TEST(S3AccessorTest, DISABLED_delete_object) { +TEST(S3AccessorTest, delete_object) { _mock_fs = std::make_unique(cloud::S3Conf {}); _mock_client = std::make_unique(); auto accessor = std::make_unique(S3Conf {}); @@ -641,6 +671,75 @@ TEST(S3AccessorTest, DISABLED_delete_object) { } } +TEST(S3AccessorTest, gcs_delete_objects) { + _mock_fs = std::make_unique(cloud::S3Conf {}); + _mock_client = std::make_unique(); + auto accessor = std::make_unique(S3Conf {}); + auto sp = SyncPoint::get_instance(); + std::for_each(callbacks.begin(), callbacks.end(), [&](const MockCallable& mock_callback) { + sp->set_call_back(fmt::format("{}::pred", mock_callback.point_name), + [](void* p) { *((bool*)p) = true; }); + sp->set_call_back(mock_callback.point_name, mock_callback.func); + }); + sp->enable_processing(); + std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + sp->disable_processing(); + std::for_each(callbacks.begin(), callbacks.end(), [&](const MockCallable& mock_callback) { + sp->clear_call_back(mock_callback.point_name); + }); + }); + std::string prefix = "test_delete_object"; + std::vector paths; + size_t num = 300; + for (size_t i = 0; i < num; i++) { + auto path = fmt::format("{}{}", prefix, i); + _mock_fs->put_object(path, ""); + paths.emplace_back(std::move(path)); + } + ASSERT_EQ(0, accessor->delete_objects(paths)); + for (size_t i = 0; i < num; i++) { + auto path = fmt::format("{}{}", prefix, i); + ASSERT_EQ(1, accessor->exist(path)); + } +} + +TEST(S3AccessorTest, gcs_delete_objects_error) { + _mock_fs = std::make_unique(cloud::S3Conf {}); + _mock_client = std::make_unique(std::make_unique()); + auto accessor = std::make_unique(S3Conf {}); + auto sp = SyncPoint::get_instance(); + std::for_each(callbacks.begin(), callbacks.end(), [&](const MockCallable& mock_callback) { + sp->set_call_back(fmt::format("{}::pred", mock_callback.point_name), + [](void* p) { *((bool*)p) = true; }); + sp->set_call_back(mock_callback.point_name, mock_callback.func); + }); + sp->enable_processing(); + std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + sp->disable_processing(); + std::for_each(callbacks.begin(), callbacks.end(), [&](const MockCallable& mock_callback) { + sp->clear_call_back(mock_callback.point_name); + }); + return_error_for_error_s3_client = false; + }); + std::string prefix = "test_delete_objects"; + std::vector paths_first_half; + std::vector paths_second_half; + size_t num = 300; + for (size_t i = 0; i < num; i++) { + auto path = fmt::format("{}{}", prefix, i); + _mock_fs->put_object(path, ""); + if (i < 150) { + paths_first_half.emplace_back(std::move(path)); + } else { + paths_second_half.emplace_back(std::move(path)); + } + } + std::vector empty; + ASSERT_EQ(0, accessor->delete_objects(empty)); + return_error_for_error_s3_client = true; + ASSERT_EQ(-1, accessor->delete_objects(paths_first_half)); +} + TEST(S3AccessorTest, delete_objects) { _mock_fs = std::make_unique(cloud::S3Conf {}); _mock_client = std::make_unique(); From df11c8c1e13a69fa20ec4d5d7a55e2edd5b357be Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 11:24:21 +0800 Subject: [PATCH 009/163] [enhance](Cloud) Unify s3 and hdfs vault's check existence logic (#34060) --- .../meta-service/meta_service_resource.cpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 57220e50c5ebf7..1dd04617cc4ac6 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -374,13 +374,6 @@ static int add_hdfs_storage_vault(InstanceInfoPB& instance, Transaction* txn, msg = fmt::format("vault_name={} passed invalid argument", hdfs_param.name()); return -1; } - if (std::find_if(instance.storage_vault_names().begin(), instance.storage_vault_names().end(), - [&hdfs_param](const auto& name) { return name == hdfs_param.name(); }) != - instance.storage_vault_names().end()) { - code = MetaServiceCode::ALREADY_EXISTED; - msg = fmt::format("vault_name={} already created", hdfs_param.name()); - return -1; - } using namespace detail; // Check and normalize hdfs conf @@ -472,13 +465,23 @@ static void create_object_info_with_encrypt(const InstanceInfoPB& instance, Obje static int add_vault_into_instance(InstanceInfoPB& instance, Transaction* txn, StorageVaultPB& vault_param, MetaServiceCode& code, std::string& msg) { + if (std::find_if(instance.storage_vault_names().begin(), instance.storage_vault_names().end(), + [&vault_param](const auto& name) { return name == vault_param.name(); }) != + instance.storage_vault_names().end()) { + code = MetaServiceCode::ALREADY_EXISTED; + msg = fmt::format("vault_name={} already created", vault_param.name()); + return -1; + } + if (vault_param.has_hdfs_info()) { return add_hdfs_storage_vault(instance, txn, vault_param, code, msg); } + create_object_info_with_encrypt(instance, vault_param.mutable_obj_info(), true, code, msg); if (code != MetaServiceCode::OK) { return -1; } + vault_param.mutable_obj_info()->CopyFrom(vault_param.obj_info()); vault_param.set_id(vault_param.obj_info().id()); auto vault_key = storage_vault_key({instance.instance_id(), vault_param.obj_info().id()}); @@ -757,7 +760,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont } } break; case AlterObjStoreInfoRequest::ADD_HDFS_INFO: { - if (auto ret = add_hdfs_storage_vault( + if (auto ret = add_vault_into_instance( instance, txn.get(), const_cast(request->vault()), code, msg); ret != 0) { return; @@ -773,7 +776,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont msg = ss.str(); return; } - if (auto ret = add_hdfs_storage_vault( + if (auto ret = add_vault_into_instance( instance, txn.get(), const_cast(request->vault()), code, msg); ret != 0) { return; From 4073aba5ad53fa35f2a0992316d59b101870b23c Mon Sep 17 00:00:00 2001 From: Pxl Date: Thu, 25 Apr 2024 11:40:05 +0800 Subject: [PATCH 010/163] =?UTF-8?q?[Improvementation](join)=20empty=5Fbloc?= =?UTF-8?q?k=20shall=20be=20set=20true=20when=20build=20blo=E2=80=A6=20(#3?= =?UTF-8?q?3977)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit empty_block shall be set true when build block only one row --- be/src/pipeline/exec/hashjoin_build_sink.cpp | 17 ++-- .../pipeline/exec/hashjoin_probe_operator.cpp | 80 +++++++++---------- .../pipeline/exec/hashjoin_probe_operator.h | 9 ++- be/src/vec/core/column_with_type_and_name.cpp | 12 +-- .../test_half_join_nullable_build_side.out | 6 ++ .../test_half_join_nullable_build_side.groovy | 4 + 6 files changed, 67 insertions(+), 61 deletions(-) diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index a0d111c63a7d98..2b2bdad86f70f8 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -156,21 +156,22 @@ bool HashJoinBuildSinkLocalState::build_unique() const { void HashJoinBuildSinkLocalState::init_short_circuit_for_probe() { auto& p = _parent->cast(); + bool empty_block = + !_shared_state->build_block || + !(_shared_state->build_block->rows() > 1); // build size always mock a row into block _shared_state->short_circuit_for_probe = (_shared_state->_has_null_in_build_side && p._join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN && !p._is_mark_join) || - (!_shared_state->build_block && p._join_op == TJoinOp::INNER_JOIN && - !p._is_mark_join) || - (!_shared_state->build_block && p._join_op == TJoinOp::LEFT_SEMI_JOIN && - !p._is_mark_join) || - (!_shared_state->build_block && p._join_op == TJoinOp::RIGHT_OUTER_JOIN) || - (!_shared_state->build_block && p._join_op == TJoinOp::RIGHT_SEMI_JOIN) || - (!_shared_state->build_block && p._join_op == TJoinOp::RIGHT_ANTI_JOIN); + (empty_block && p._join_op == TJoinOp::INNER_JOIN && !p._is_mark_join) || + (empty_block && p._join_op == TJoinOp::LEFT_SEMI_JOIN && !p._is_mark_join) || + (empty_block && p._join_op == TJoinOp::RIGHT_OUTER_JOIN) || + (empty_block && p._join_op == TJoinOp::RIGHT_SEMI_JOIN) || + (empty_block && p._join_op == TJoinOp::RIGHT_ANTI_JOIN); //when build table rows is 0 and not have other_join_conjunct and not _is_mark_join and join type is one of LEFT_OUTER_JOIN/FULL_OUTER_JOIN/LEFT_ANTI_JOIN //we could get the result is probe table + null-column(if need output) _shared_state->empty_right_table_need_probe_dispose = - (!_shared_state->build_block && !p._have_other_join_conjunct && !p._is_mark_join) && + (empty_block && !p._have_other_join_conjunct && !p._is_mark_join) && (p._join_op == TJoinOp::LEFT_OUTER_JOIN || p._join_op == TJoinOp::FULL_OUTER_JOIN || p._join_op == TJoinOp::LEFT_ANTI_JOIN); } diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp index a58ad62211ceb2..fc6f81f41902a2 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp +++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp @@ -247,7 +247,7 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc } //TODO: this short circuit maybe could refactor, no need to check at here. - if (local_state._shared_state->empty_right_table_need_probe_dispose) { + if (local_state.empty_right_table_shortcut()) { // when build table rows is 0 and not have other_join_conjunct and join type is one of LEFT_OUTER_JOIN/FULL_OUTER_JOIN/LEFT_ANTI_JOIN // we could get the result is probe table + null-column(if need output) // If we use a short-circuit strategy, should return block directly by add additional null data. @@ -257,12 +257,6 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc return Status::OK(); } - vectorized::Block temp_block; - //get probe side output column - for (int i = 0; i < _left_output_slot_flags.size(); ++i) { - temp_block.insert(local_state._probe_block.get_by_position(i)); - } - //create build side null column, if need output for (int i = 0; (_join_op != TJoinOp::LEFT_ANTI_JOIN) && i < _right_output_slot_flags.size(); ++i) { @@ -273,8 +267,8 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc vectorized::ColumnVector::create(block_rows, 1); auto nullable_column = vectorized::ColumnNullable::create(std::move(column), std::move(null_map_column)); - temp_block.insert({std::move(nullable_column), make_nullable(type), - _right_table_column_names[i]}); + local_state._probe_block.insert({std::move(nullable_column), make_nullable(type), + _right_table_column_names[i]}); } if (_is_outer_join) { reinterpret_cast( @@ -290,8 +284,7 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc /// No need to check the block size in `_filter_data_and_build_output` because here dose not /// increase the output rows count(just same as `_probe_block`'s rows count). RETURN_IF_ERROR(local_state.filter_data_and_build_output(state, output_block, eos, - &temp_block, false)); - temp_block.clear(); + &local_state._probe_block, false)); local_state._probe_block.clear_column_data(_child_x->row_desc().num_materialized_slots()); return Status::OK(); } @@ -374,36 +367,52 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc } Status HashJoinProbeLocalState::_extract_join_column(vectorized::Block& block, - vectorized::ColumnUInt8::MutablePtr& null_map, - vectorized::ColumnRawPtrs& raw_ptrs, const std::vector& res_col_ids) { + if (empty_right_table_shortcut()) { + return Status::OK(); + } + + _probe_columns.resize(_probe_expr_ctxs.size()); + + if (!_has_set_need_null_map_for_probe) { + _has_set_need_null_map_for_probe = true; + _need_null_map_for_probe = _need_probe_null_map(block, res_col_ids); + } + if (_need_null_map_for_probe) { + if (_null_map_column == nullptr) { + _null_map_column = vectorized::ColumnUInt8::create(); + } + _null_map_column->get_data().assign(block.rows(), (uint8_t)0); + } + auto& shared_state = *_shared_state; auto& p = _parent->cast(); for (size_t i = 0; i < shared_state.build_exprs_size; ++i) { if (p._should_convert_to_nullable[i]) { _key_columns_holder.emplace_back( vectorized::make_nullable(block.get_by_position(res_col_ids[i]).column)); - raw_ptrs[i] = _key_columns_holder.back().get(); + _probe_columns[i] = _key_columns_holder.back().get(); continue; } if (shared_state.is_null_safe_eq_join[i]) { - raw_ptrs[i] = block.get_by_position(res_col_ids[i]).column.get(); + _probe_columns[i] = block.get_by_position(res_col_ids[i]).column.get(); } else { - auto column = block.get_by_position(res_col_ids[i]).column.get(); - if (auto* nullable = check_and_get_column(*column)) { - auto& col_nested = nullable->get_nested_column(); - auto& col_nullmap = nullable->get_null_map_data(); - - DCHECK(null_map != nullptr); - vectorized::VectorizedUtils::update_null_map(null_map->get_data(), col_nullmap); + const auto* column = block.get_by_position(res_col_ids[i]).column.get(); + if (const auto* nullable = check_and_get_column(*column)) { + const auto& col_nested = nullable->get_nested_column(); + const auto& col_nullmap = nullable->get_null_map_data(); + + DCHECK(_null_map_column != nullptr); + vectorized::VectorizedUtils::update_null_map(_null_map_column->get_data(), + col_nullmap); if (shared_state.store_null_in_hash_table[i]) { - raw_ptrs[i] = nullable; + _probe_columns[i] = nullable; } else { - raw_ptrs[i] = &col_nested; + _probe_columns[i] = &col_nested; } } else { - raw_ptrs[i] = column; + _probe_columns[i] = column; } } } @@ -482,10 +491,7 @@ Status HashJoinProbeOperatorX::push(RuntimeState* state, vectorized::Block* inpu local_state._probe_eos = eos; if (input_block->rows() > 0) { COUNTER_UPDATE(local_state._probe_rows_counter, input_block->rows()); - int probe_expr_ctxs_sz = local_state._probe_expr_ctxs.size(); - local_state._probe_columns.resize(probe_expr_ctxs_sz); - - std::vector res_col_ids(probe_expr_ctxs_sz); + std::vector res_col_ids(local_state._probe_expr_ctxs.size()); RETURN_IF_ERROR(_do_evaluate(*input_block, local_state._probe_expr_ctxs, *local_state._probe_expr_call_timer, res_col_ids)); if (_join_op == TJoinOp::RIGHT_OUTER_JOIN || _join_op == TJoinOp::FULL_OUTER_JOIN) { @@ -493,22 +499,8 @@ Status HashJoinProbeOperatorX::push(RuntimeState* state, vectorized::Block* inpu local_state._convert_block_to_null(*input_block); } - // TODO: Now we are not sure whether a column is nullable only by ExecNode's `row_desc` - // so we have to initialize this flag by the first probe block. - if (!local_state._has_set_need_null_map_for_probe) { - local_state._has_set_need_null_map_for_probe = true; - local_state._need_null_map_for_probe = - local_state._need_probe_null_map(*input_block, res_col_ids); - } - if (local_state._need_null_map_for_probe) { - if (local_state._null_map_column == nullptr) { - local_state._null_map_column = vectorized::ColumnUInt8::create(); - } - local_state._null_map_column->get_data().assign(input_block->rows(), (uint8_t)0); - } + RETURN_IF_ERROR(local_state._extract_join_column(*input_block, res_col_ids)); - RETURN_IF_ERROR(local_state._extract_join_column(*input_block, local_state._null_map_column, - local_state._probe_columns, res_col_ids)); if (&local_state._probe_block != input_block) { input_block->swap(local_state._probe_block); } diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h index b4930307bcc818..1b45a2a258eb07 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.h +++ b/be/src/pipeline/exec/hashjoin_probe_operator.h @@ -94,15 +94,16 @@ class HashJoinProbeLocalState final const std::shared_ptr& build_block() const { return _shared_state->build_block; } + bool empty_right_table_shortcut() const { + // !Base::_projections.empty() means nereids planner + return _shared_state->empty_right_table_need_probe_dispose && !Base::_projections.empty(); + } private: void _prepare_probe_block(); bool _need_probe_null_map(vectorized::Block& block, const std::vector& res_col_ids); std::vector _convert_block_to_null(vectorized::Block& block); - Status _extract_join_column(vectorized::Block& block, - vectorized::ColumnUInt8::MutablePtr& null_map, - vectorized::ColumnRawPtrs& raw_ptrs, - const std::vector& res_col_ids); + Status _extract_join_column(vectorized::Block& block, const std::vector& res_col_ids); friend class HashJoinProbeOperatorX; template friend struct vectorized::ProcessHashTableProbe; diff --git a/be/src/vec/core/column_with_type_and_name.cpp b/be/src/vec/core/column_with_type_and_name.cpp index cd0f7194004073..e93946804ffa60 100644 --- a/be/src/vec/core/column_with_type_and_name.cpp +++ b/be/src/vec/core/column_with_type_and_name.cpp @@ -62,15 +62,17 @@ void ColumnWithTypeAndName::dump_structure(std::ostream& out) const { out << name; } - if (type) + if (type) { out << " " << type->get_name(); - else + } else { out << " nullptr"; + } - if (column) - out << ' ' << column->dump_structure(); - else + if (column) { + out << ' ' << column->dump_structure() << "(use_count=" << column->use_count() << ')'; + } else { out << " nullptr"; + } } String ColumnWithTypeAndName::dump_structure() const { diff --git a/regression-test/data/query_p0/join/test_half_join_nullable_build_side.out b/regression-test/data/query_p0/join/test_half_join_nullable_build_side.out index 8404bee641f5b2..56c5f6e2229ba7 100644 --- a/regression-test/data/query_p0/join/test_half_join_nullable_build_side.out +++ b/regression-test/data/query_p0/join/test_half_join_nullable_build_side.out @@ -134,3 +134,9 @@ 4 \N \N \N \N \N 5 1111 1111 3 1111 1111 +-- !shortcut -- +1 11 11 +2 111 111 +3 1111 1111 +4 111 111 + diff --git a/regression-test/suites/query_p0/join/test_half_join_nullable_build_side.groovy b/regression-test/suites/query_p0/join/test_half_join_nullable_build_side.groovy index 2bb24309960054..230332fdf3bbe2 100644 --- a/regression-test/suites/query_p0/join/test_half_join_nullable_build_side.groovy +++ b/regression-test/suites/query_p0/join/test_half_join_nullable_build_side.groovy @@ -286,4 +286,8 @@ suite("test_half_join_nullable_build_side", "query,p0") { left join test_half_join_nullable_build_side_l r on l.v2 <=> r.v2 order by 1, 2, 3; """ + + qt_shortcut """ + select * from test_half_join_nullable_build_side_l l left anti join test_half_join_nullable_build_side_r r on l.v2 <=> r.v2 and r.k1=5 order by 1, 2, 3; + """ } \ No newline at end of file From 40b1c4c7269b465e39e8c9e8d3916a8ebb8aa444 Mon Sep 17 00:00:00 2001 From: Mryange <59914473+Mryange@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:06:15 +0800 Subject: [PATCH 011/163] [fix](scan) fix ignore expr exec when _non_predicate_columns is empty (#33934) fix ignore expr exec when _non_predicate_columns is empty --- .../olap/rowset/segment_v2/segment_iterator.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index a1b8cf35b353a5..506d7d0b2ad322 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1957,14 +1957,16 @@ void SegmentIterator::_replace_version_col(size_t num_rows) { uint16_t SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size) { SCOPED_RAW_TIMER(&_opts.stats->vec_cond_ns); - if (_is_need_vec_eval) { - _is_need_vec_eval = false; - for (const auto& pred : _pre_eval_block_predicate) { - _is_need_vec_eval |= (!pred->always_true()); + bool all_pred_always_true = true; + for (const auto& pred : _pre_eval_block_predicate) { + if (!pred->always_true()) { + all_pred_always_true = false; + break; } } - if (!_is_need_vec_eval) { - for (uint32_t i = 0; i < selected_size; ++i) { + //If all predicates are always_true, then return directly. + if (all_pred_always_true || !_is_need_vec_eval) { + for (uint16_t i = 0; i < selected_size; ++i) { sel_rowid_idx[i] = i; } return selected_size; @@ -2248,6 +2250,9 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { } if (!_is_need_vec_eval && !_is_need_short_eval && !_is_need_expr_eval) { + if (_non_predicate_columns.empty()) { + return Status::InternalError("_non_predicate_columns is empty"); + } RETURN_IF_ERROR(_convert_to_expected_type(_first_read_column_ids)); RETURN_IF_ERROR(_convert_to_expected_type(_non_predicate_columns)); _output_non_pred_columns(block); From ad4e82bea6d2428827d98eb74946e014511839f2 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:17:42 +0800 Subject: [PATCH 012/163] [chore](version) revert version changes (#34095) --- gensrc/script/gen_build_version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index b99e2edd3a377c..eec7281c449c5b 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -28,10 +28,10 @@ set -eo pipefail build_version_prefix="doris" -build_version_major=4 +build_version_major=0 build_version_minor=0 build_version_patch=0 -build_version_rc_version="preview" +build_version_rc_version="" build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}" From 3aa89110bc6e459b81ead622a9ff794e8073ac2f Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 14:17:48 +0800 Subject: [PATCH 013/163] [enhance](Cloud) Add case to test if vault is forbid for cloud mode without vault (#34006) --- .../doris/analysis/ShowStorageVaultStmt.java | 12 +++++ .../suites/vaults/forbid/forbid.groovy | 50 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 regression-test/suites/vaults/forbid/forbid.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java index e4af07e43b1e37..ddf2529284602b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java @@ -19,9 +19,12 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.StorageVault; +import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -41,6 +44,15 @@ public ShowStorageVaultStmt() { @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { + if (Config.isNotCloudMode()) { + throw new AnalysisException("Storage Vault is only supported for cloud mode"); + } + if (!FeConstants.runningUnitTest) { + // In legacy cloud mode, some s3 back-ended storage does need to use storage vault. + if (!((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) { + throw new AnalysisException("Your cloud instance doesn't support storage vault"); + } + } super.analyze(analyzer); // check auth if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { diff --git a/regression-test/suites/vaults/forbid/forbid.groovy b/regression-test/suites/vaults/forbid/forbid.groovy new file mode 100644 index 00000000000000..15fba18fc6da1f --- /dev/null +++ b/regression-test/suites/vaults/forbid/forbid.groovy @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("forbid_vault") { + if (enableStoragevault()) { + logger.info("skip forbid storage vault case because storage vault enabled") + return + } + + if (!isCloudMode()) { + logger.info("skip forbid storage vault case because not cloud mode") + return + } + + expectExceptionLike({ + sql """ + set not_exist as default storage vault + """ + }, "Your cloud instance doesn't support storage vault") + + expectExceptionLike({ + sql """ + CREATE STORAGE VAULT IF NOT EXISTS hdfs_vault + PROPERTIES ( + "type"="hdfs", + "fs.defaultFS"="hdfs://127.0.0.1:8020" + ); + """ + }, "Your cloud instance doesn't support storage vault") + + expectExceptionLike({ + sql """ + show storage vault + """ + }, "Your cloud instance doesn't support storage vault") +} \ No newline at end of file From b95b37bf0279f2a116c8ec84c6830bcf6423ccf4 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 15:00:36 +0800 Subject: [PATCH 014/163] [enhance](Cloud) Add GCP enum for ObjectStoreInfoPB (#34100) --- .../src/main/java/org/apache/doris/cloud/storage/RemoteBase.java | 1 + gensrc/proto/cloud.proto | 1 + 2 files changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java index 7b12be88d74a24..e146e52f534d94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java @@ -140,6 +140,7 @@ public static RemoteBase newInstance(ObjectInfo obj) throws Exception { switch (obj.provider) { case OSS: return new OssRemote(obj); + case GCP: case S3: return new S3Remote(obj); case COS: diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 0ed9b693988dd7..f132e2eab4d0ee 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -176,6 +176,7 @@ message ObjectStoreInfoPB { COS = 2; OBS = 3; BOS = 4; + GCP = 5; } optional int64 ctime = 1; optional int64 mtime = 2; From 48a306149c11f93363c1197d576444f71d0ef708 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei <53502832+feiniaofeiafei@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:01:55 +0800 Subject: [PATCH 015/163] [Fix](nereids) fix rule SimplifyWindowExpression (#34099) Co-authored-by: feiniaofeiafei --- .../rewrite/SimplifyWindowExpression.java | 12 +++- .../simplify_window_expression.out | 67 +++++++++++-------- .../simplify_window_expression.groovy | 3 + 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java index 872ca789818b5e..c0548a42579da5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java @@ -27,10 +27,12 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -87,11 +89,13 @@ private Plan simplify(MatchingContext> ctx) { if (function instanceof BoundFunction) { BoundFunction boundFunction = (BoundFunction) function; String name = ((BoundFunction) function).getName(); - if ((name.equals(COUNT) && boundFunction.child(0).notNullable()) + if ((name.equals(COUNT) && checkCount((Count) boundFunction)) || REWRRITE_TO_CONST_WINDOW_FUNCTIONS.contains(name)) { projectionsBuilder.add(new Alias(alias.getExprId(), new TinyIntLiteral((byte) 1), alias.getName())); } else if (REWRRITE_TO_SLOT_WINDOW_FUNCTIONS.contains(name)) { - projectionsBuilder.add(new Alias(alias.getExprId(), boundFunction.child(0), alias.getName())); + projectionsBuilder.add(new Alias(alias.getExprId(), + TypeCoercionUtils.castIfNotSameType(boundFunction.child(0), boundFunction.getDataType()), + alias.getName())); } else { remainWindowExpression.add(expr); } @@ -120,4 +124,8 @@ private Plan simplify(MatchingContext> ctx) { window.child(0))); } } + + private boolean checkCount(Count count) { + return count.isCountStar() || count.child(0).notNullable(); + } } diff --git a/regression-test/data/nereids_rules_p0/simplify_window_expression/simplify_window_expression.out b/regression-test/data/nereids_rules_p0/simplify_window_expression/simplify_window_expression.out index 3befc3dcbb2742..e660cd7702c650 100644 --- a/regression-test/data/nereids_rules_p0/simplify_window_expression/simplify_window_expression.out +++ b/regression-test/data/nereids_rules_p0/simplify_window_expression/simplify_window_expression.out @@ -119,28 +119,28 @@ -- !select_avg -- \N \N \N \N \N \N -1 1 1 -1 1 1 -2 2 2 -3 3 3 -3 3 3 -4 4 4 -5 5 5 -5 5 5 -7 7 7 +1 1.0 1.0 +1 1.0 1.0 +2 2.0 2.0 +3 3.0 3.0 +3 3.0 3.0 +4 4.0 4.0 +5 5.0 5.0 +5 5.0 5.0 +7 7.0 7.0 -- !more_than_pk -- \N \N \N \N \N \N -1 1 1 -1 1 1 -2 2 2 -3 3 3 -3 3 3 -4 4 4 -5 5 5 -5 5 5 -7 7 7 +1 1.0 1.0 +1 1.0 1.0 +2 2.0 2.0 +3 3.0 3.0 +3 3.0 3.0 +4 4.0 4.0 +5 5.0 5.0 +5 5.0 5.0 +7 7.0 7.0 -- !select_last_value_shape -- PhysicalResultSink @@ -163,18 +163,31 @@ PhysicalResultSink ------filter((mal_test_simplify_window.__DORIS_DELETE_SIGN__ = 0)) --------PhysicalOlapScan[mal_test_simplify_window] +-- !select_count_star_col1 -- +\N 1 1 +1 1 1 +1 1 1 +2 1 1 +2 1 1 +2 1 1 +3 1 1 +3 1 1 +4 1 1 +6 1 1 +6 1 1 + -- !select_upper_plan_use_all_rewrite -- \N \N \N \N -1 1 -1 1 -2 2 -3 3 -3 3 -4 4 -5 5 -5 5 -7 7 +1 1.0 +1 1.0 +2 2.0 +3 3.0 +3 3.0 +4 4.0 +5 5.0 +5 5.0 +7 7.0 -- !select_upper_plan_use_rewrite_and_not_rewrite -- \N \N \N diff --git a/regression-test/suites/nereids_rules_p0/simplify_window_expression/simplify_window_expression.groovy b/regression-test/suites/nereids_rules_p0/simplify_window_expression/simplify_window_expression.groovy index 11ad672c74ff11..3e247b2a78f577 100644 --- a/regression-test/suites/nereids_rules_p0/simplify_window_expression/simplify_window_expression.groovy +++ b/regression-test/suites/nereids_rules_p0/simplify_window_expression/simplify_window_expression.groovy @@ -78,6 +78,9 @@ suite("simplify_window_expression") { explain shape plan select b, avg(b) over (partition by a,b,c) c1, avg(b) over (partition by a,b,c order by b) c2 from mal_test_simplify_window""" + qt_select_count_star_col1 """ + select a,count() over (partition by a,b) c1, count() over (partition by a,b order by a) c2 + from mal_test_simplify_window order by 1,2,3;""" qt_select_upper_plan_use_all_rewrite """ select b, c1 from (select b,avg(b) over (partition by a,b) c1 From 693a48522187546ec1e1a8519336524c6cb9baa6 Mon Sep 17 00:00:00 2001 From: spaces-x Date: Thu, 25 Apr 2024 15:25:23 +0800 Subject: [PATCH 016/163] [fix](cloud): fix parent directory doesn't exist in HdfsFileWriter (#33985) Co-authored-by: weixiang06 --- be/src/io/fs/hdfs_file_writer.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp index d15745ebf37ad5..c596c0e290fe8c 100644 --- a/be/src/io/fs/hdfs_file_writer.cpp +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -281,6 +281,21 @@ Result HdfsFileWriter::create(Path full_path, std::shared_ptrhdfs_fs, hdfs_dir.c_str()); + if (exists != 0) { + VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir; + int ret = hdfsCreateDirectory(handler->hdfs_fs, hdfs_dir.c_str()); + if (ret != 0) { + std::stringstream ss; + ss << "create dir failed. " + << " fs_name: " << fs_name << " path: " << hdfs_dir << ", err: " << hdfs_error(); + LOG(WARNING) << ss.str(); + return ResultError(Status::InternalError(ss.str())); + } + } +#endif // open file hdfsFile hdfs_file = nullptr; { From 785ffb2aeb1bf02b88a22b8a961f1bede47d2f81 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:27:37 +0800 Subject: [PATCH 017/163] [Fix](nereids) fix cases unstable of hint (#34101) fix cases unstable of hint, remove unused cases and project nodes and use string contains in order to avoid unstable problem. --- .../data/nereids_p0/hint/test_leading.out | 507 ------------------ .../suites/nereids_p0/hint/fix_leading.groovy | 6 +- .../nereids_p0/hint/multi_leading.groovy | 17 +- .../nereids_p0/hint/test_leading.groovy | 110 +++- 4 files changed, 105 insertions(+), 535 deletions(-) diff --git a/regression-test/data/nereids_p0/hint/test_leading.out b/regression-test/data/nereids_p0/hint/test_leading.out index 531950358804d3..31df4aafe732ba 100644 --- a/regression-test/data/nereids_p0/hint/test_leading.out +++ b/regression-test/data/nereids_p0/hint/test_leading.out @@ -2165,25 +2165,6 @@ Used: [broadcast]_2 UnUsed: SyntaxError: --- !select90_2 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t2] - -Hint log: -Used: -UnUsed: [broadcast]_2 -SyntaxError: - -- !select90_3 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -2208,30 +2189,6 @@ Used: [broadcast]_2 [shuffle]_3 UnUsed: SyntaxError: --- !select90_4 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: [shuffle]_3 -UnUsed: [broadcast]_2 -SyntaxError: - -- !select90_5 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -2256,125 +2213,6 @@ Used: [broadcast]_2 [shuffle]_3 UnUsed: SyntaxError: --- !select90_6 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: [shuffle]_2 -UnUsed: [broadcast]_3 -SyntaxError: - --- !select91_1 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: leading(t1 t2 t3 ) -UnUsed: [broadcast]_2 [shuffle]_3 -SyntaxError: - --- !select91_2 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: leading(t1 t2 t3 ) -UnUsed: [broadcast]_2 [shuffle]_3 -SyntaxError: - --- !select91_3 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: leading(t1 t2 t3 ) -UnUsed: [broadcast]_2 [shuffle]_3 -SyntaxError: - --- !select91_4 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: leading(t1 t2 t3 ) -UnUsed: [shuffle]_2 [broadcast]_3 -SyntaxError: - -- !select92_1 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -2806,52 +2644,6 @@ Used: leading(t1 broadcast t2 t3 ) UnUsed: SyntaxError: --- !select95_2 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 leading(t1 broadcast { t2 t3 }) -SyntaxError: - --- !select95_3 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 leading(t1 broadcast { t3 t2 }) -SyntaxError: - -- !select95_4 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -2875,75 +2667,6 @@ Used: leading(t2 broadcast t1 t3 ) UnUsed: SyntaxError: --- !select95_5 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 leading(t2 broadcast { t1 t3 }) -SyntaxError: - --- !select95_6 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 leading(t2 broadcast { t3 t1 }) -SyntaxError: - --- !select95_7 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: leading(t3 broadcast t1 t2) -SyntaxError: - -- !select95_8 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -3013,52 +2736,6 @@ Used: leading(t1 shuffle t2 broadcast t3 ) UnUsed: SyntaxError: --- !select96_2 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [shuffle]_2 [broadcast]_3 leading(t1 shuffle { t2 broadcast t3 }) -SyntaxError: - --- !select96_3 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [shuffle]_2 [broadcast]_3 leading(t1 shuffle { t3 broadcast t2 }) -SyntaxError: - -- !select96_4 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -3082,75 +2759,6 @@ Used: leading(t2 shuffle t1 broadcast t3 ) UnUsed: SyntaxError: --- !select96_5 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [shuffle]_2 leading(t2 shuffle { t1 broadcast t3 }) -SyntaxError: - --- !select96_6 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [shuffle]_2 leading(t2 shuffle { t3 broadcast t1 }) -SyntaxError: - --- !select96_7 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_3 leading(t3 shuffle t1 broadcast t2) -SyntaxError: - -- !select96_8 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -3221,52 +2829,6 @@ Used: leading(t1 broadcast t2 shuffle t3 ) UnUsed: SyntaxError: --- !select97_2 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 [shuffle]_3 leading(t1 broadcast { t2 shuffle t3 }) -SyntaxError: - --- !select97_3 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 [shuffle]_3 leading(t1 broadcast { t3 shuffle t2 }) -SyntaxError: - -- !select97_4 -- PhysicalResultSink --hashAgg[GLOBAL] @@ -3290,75 +2852,6 @@ Used: leading(t2 broadcast t1 shuffle t3 ) UnUsed: SyntaxError: --- !select97_5 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 leading(t2 broadcast { t1 shuffle t3 }) -SyntaxError: - --- !select97_6 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [broadcast]_2 leading(t2 broadcast { t3 shuffle t1 }) -SyntaxError: - --- !select97_7 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: [shuffle]_3 leading(t3 broadcast t1 shuffle t2) -SyntaxError: - -- !select97_8 -- PhysicalResultSink --hashAgg[GLOBAL] diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index 49823e9769830a..4c88fb2ac30f62 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -172,6 +172,8 @@ suite("fix_leading") { qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" // check whether we have all tables - qt_select5_1 """explain shape plan select /*+ leading(t1 t2)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" - + explain { + sql """shape plan select /*+ leading(t1 t2)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" + contains("SyntaxError: leading(t1 t2) Msg:leading should have all tables in query block, missing tables: t3") + } } diff --git a/regression-test/suites/nereids_p0/hint/multi_leading.groovy b/regression-test/suites/nereids_p0/hint/multi_leading.groovy index bd567ecdefa378..f716b5c98d05e0 100644 --- a/regression-test/suites/nereids_p0/hint/multi_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/multi_leading.groovy @@ -113,9 +113,18 @@ suite("multi_leading") { qt_sql4_2 """explain shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" qt_sql4_3 """explain shape plan select count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" qt_sql4_4 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_5 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_6 """explain shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_7 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + explain { + sql """shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + contains("SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause") + } + explain { + sql """shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + contains("SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause") + } + explain { + sql """shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + contains("UnUsed: leading(alias2 t1)") + } qt_sql4_res_0 """select count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" qt_sql4_res_1 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" @@ -129,4 +138,4 @@ suite("multi_leading") { // use cte in scalar query qt_sql5_1 """explain shape plan with cte as (select c11, c1 from t1) SELECT c1 FROM cte group by c1 having sum(cte.c11) > (select 0.05 * avg(t1.c11) from t1 join cte on t1.c1 = cte.c11 )""" qt_sql5_2 """explain shape plan with cte as (select c11, c1 from t1) SELECT c1 FROM cte group by c1 having sum(cte.c11) > (select /*+ leading(cte t1) */ 0.05 * avg(t1.c11) from t1 join cte on t1.c1 = cte.c11 )""" -} \ No newline at end of file +} diff --git a/regression-test/suites/nereids_p0/hint/test_leading.groovy b/regression-test/suites/nereids_p0/hint/test_leading.groovy index a9ad4aa705ccfa..d1e11144b527c7 100644 --- a/regression-test/suites/nereids_p0/hint/test_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/test_leading.groovy @@ -936,19 +936,40 @@ suite("test_leading") { // used qt_select90_1 """explain shape plan select count(*) from t1 join [broadcast] t2 on c1 = c2;""" // unused - qt_select90_2 """explain shape plan select count(*) from t1 right outer join [broadcast] t2 on c1 = c2;""" + explain { + sql """shape plan select count(*) from t1 right outer join [broadcast] t2 on c1 = c2;""" + contains("UnUsed: [broadcast]_2") + } // only distribute hint + multi hints qt_select90_3 """explain shape plan select count(*) from t1 join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" - qt_select90_4 """explain shape plan select count(*) from t1 right outer join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" + explain { + sql """shape plan select count(*) from t1 right outer join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2") + } qt_select90_5 """explain shape plan select count(*) from t1 join [broadcast] t2 on c1 = c2 right outer join[shuffle] t3 on c2 = c3;""" - qt_select90_6 """explain shape plan select count(*) from t1 join [shuffle] t2 on c1 = c2 right outer join[broadcast] t3 on c2 = c3;""" + explain { + sql """shape plan select count(*) from t1 join [shuffle] t2 on c1 = c2 right outer join[broadcast] t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_3") + } // leading + distribute hint outside leading + single hint - qt_select91_1 """explain shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" - qt_select91_2 """explain shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 right outer join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" - qt_select91_3 """explain shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [broadcast] t2 on c1 = c2 right outer join[shuffle] t3 on c2 = c3;""" - qt_select91_4 """explain shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [shuffle] t2 on c1 = c2 right outer join[broadcast] t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 [shuffle]_3") + } + explain { + sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 right outer join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 [shuffle]_3") + } + explain { + sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [broadcast] t2 on c1 = c2 right outer join[shuffle] t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 [shuffle]_3") + } + explain { + sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [shuffle] t2 on c1 = c2 right outer join[broadcast] t3 on c2 = c3;""" + contains("UnUsed: [shuffle]_2 [broadcast]_3") + } // leading + distribute hint inside leading + single hint // inner join @@ -975,32 +996,77 @@ suite("test_leading") { // outer join qt_select95_1 """explain shape plan select /*+ leading(t1 broadcast t2 t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select95_2 """explain shape plan select /*+ leading(t1 broadcast {t2 t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select95_3 """explain shape plan select /*+ leading(t1 broadcast {t3 t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t1 broadcast {t2 t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 leading(t1 broadcast { t2 t3 })") + } + explain { + sql """shape plan select /*+ leading(t1 broadcast {t3 t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 leading(t1 broadcast { t3 t2 })") + } qt_select95_4 """explain shape plan select /*+ leading(t2 broadcast t1 t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select95_5 """explain shape plan select /*+ leading(t2 broadcast {t1 t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select95_6 """explain shape plan select /*+ leading(t2 broadcast {t3 t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select95_7 """explain shape plan select /*+ leading(t3 broadcast t1 t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t2 broadcast {t1 t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 leading(t2 broadcast { t1 t3 })") + } + explain { + sql """shape plan select /*+ leading(t2 broadcast {t3 t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 leading(t2 broadcast { t3 t1 })") + } + explain { + sql """shape plan select /*+ leading(t3 broadcast t1 t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: leading(t3 broadcast t1 t2)") + } qt_select95_8 """explain shape plan select /*+ leading(t3 broadcast {t1 t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" qt_select95_9 """explain shape plan select /*+ leading(t3 broadcast {t2 t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" qt_select96_1 """explain shape plan select /*+ leading(t1 shuffle t2 broadcast t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select96_2 """explain shape plan select /*+ leading(t1 shuffle {t2 broadcast t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select96_3 """explain shape plan select /*+ leading(t1 shuffle {t3 broadcast t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t1 shuffle {t2 broadcast t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [shuffle]_2 [broadcast]_3 leading(t1 shuffle { t2 broadcast t3 })") + } + explain { + sql """shape plan select /*+ leading(t1 shuffle {t3 broadcast t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [shuffle]_2 [broadcast]_3 leading(t1 shuffle { t3 broadcast t2 })") + } qt_select96_4 """explain shape plan select /*+ leading(t2 shuffle t1 broadcast t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select96_5 """explain shape plan select /*+ leading(t2 shuffle {t1 broadcast t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select96_6 """explain shape plan select /*+ leading(t2 shuffle {t3 broadcast t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select96_7 """explain shape plan select /*+ leading(t3 shuffle t1 broadcast t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t2 shuffle {t1 broadcast t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [shuffle]_2 leading(t2 shuffle { t1 broadcast t3 })") + } + explain { + sql """shape plan select /*+ leading(t2 shuffle {t3 broadcast t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [shuffle]_2 leading(t2 shuffle { t3 broadcast t1 })") + } + explain { + sql """shape plan select /*+ leading(t3 shuffle t1 broadcast t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_3 leading(t3 shuffle t1 broadcast t2)") + } qt_select96_8 """explain shape plan select /*+ leading(t3 shuffle {t1 broadcast t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" qt_select96_9 """explain shape plan select /*+ leading(t3 shuffle {t2 broadcast t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" qt_select97_1 """explain shape plan select /*+ leading(t1 broadcast t2 shuffle t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select97_2 """explain shape plan select /*+ leading(t1 broadcast {t2 shuffle t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select97_3 """explain shape plan select /*+ leading(t1 broadcast {t3 shuffle t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t1 broadcast {t2 shuffle t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 [shuffle]_3 leading(t1 broadcast { t2 shuffle t3 })") + } + explain { + sql """shape plan select /*+ leading(t1 broadcast {t3 shuffle t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 [shuffle]_3 leading(t1 broadcast { t3 shuffle t2 })") + } qt_select97_4 """explain shape plan select /*+ leading(t2 broadcast t1 shuffle t3) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select97_5 """explain shape plan select /*+ leading(t2 broadcast {t1 shuffle t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select97_6 """explain shape plan select /*+ leading(t2 broadcast {t3 shuffle t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" - qt_select97_7 """explain shape plan select /*+ leading(t3 broadcast t1 shuffle t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + explain { + sql """shape plan select /*+ leading(t2 broadcast {t1 shuffle t3}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 leading(t2 broadcast { t1 shuffle t3 })") + } + explain { + sql """shape plan select /*+ leading(t2 broadcast {t3 shuffle t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [broadcast]_2 leading(t2 broadcast { t3 shuffle t1 })") + } + explain { + sql """shape plan select /*+ leading(t3 broadcast t1 shuffle t2) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" + contains("UnUsed: [shuffle]_3 leading(t3 broadcast t1 shuffle t2)") + } qt_select97_8 """explain shape plan select /*+ leading(t3 broadcast {t1 shuffle t2}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" qt_select97_9 """explain shape plan select /*+ leading(t3 broadcast {t2 shuffle t1}) */ count(*) from t1 left outer join t2 on c1 = c2 join t3 on c2 = c3;""" From 073fec33b96da14754dbb795275f76e6b97dd2fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Thu, 25 Apr 2024 16:31:38 +0800 Subject: [PATCH 018/163] [feature](Nereids): add equal set in functional dependencies (#33642) --- .../properties/FunctionalDependencies.java | 42 +++- .../trees/plans/BlockFuncDepsPropagation.java | 5 + .../trees/plans/PropagateFuncDeps.java | 5 + .../trees/plans/logical/LogicalAggregate.java | 5 + .../plans/logical/LogicalAssertNumRows.java | 6 + .../plans/logical/LogicalCatalogRelation.java | 5 + .../logical/LogicalDeferMaterializeTopN.java | 5 + .../trees/plans/logical/LogicalExcept.java | 15 ++ .../trees/plans/logical/LogicalFilter.java | 10 + .../trees/plans/logical/LogicalGenerate.java | 6 + .../trees/plans/logical/LogicalHaving.java | 10 + .../trees/plans/logical/LogicalIntersect.java | 9 + .../trees/plans/logical/LogicalJoin.java | 16 ++ .../trees/plans/logical/LogicalLimit.java | 5 + .../plans/logical/LogicalOneRowRelation.java | 16 ++ .../trees/plans/logical/LogicalPlan.java | 3 + .../trees/plans/logical/LogicalProject.java | 21 ++ .../trees/plans/logical/LogicalRepeat.java | 5 + .../trees/plans/logical/LogicalSqlCache.java | 21 +- .../plans/logical/LogicalSubQueryAlias.java | 11 + .../trees/plans/logical/LogicalTopN.java | 5 + .../trees/plans/logical/LogicalUnion.java | 61 +++++ .../trees/plans/logical/LogicalView.java | 6 + .../trees/plans/logical/LogicalWindow.java | 6 + .../doris/nereids/util/ExpressionUtils.java | 8 + .../doris/nereids/util/ImmutableEqualSet.java | 69 +++++- .../nereids/properties/EqualSetTest.java | 230 ++++++++++++++++++ .../nereids_syntax_p0/join_order.groovy | 8 +- 28 files changed, 575 insertions(+), 39 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/properties/EqualSetTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java index 2b0c1f5c9143e3..a516bf9ae1c0e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java @@ -18,11 +18,13 @@ package org.apache.doris.nereids.properties; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.util.ImmutableEqualSet; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -36,15 +38,20 @@ public class FunctionalDependencies { public static final FunctionalDependencies EMPTY_FUNC_DEPS = new FunctionalDependencies(new NestedSet().toImmutable(), - new NestedSet().toImmutable(), new ImmutableSet.Builder().build()); + new NestedSet().toImmutable(), new ImmutableSet.Builder().build(), + ImmutableEqualSet.empty()); private final NestedSet uniqueSet; private final NestedSet uniformSet; private final ImmutableSet fdItems; - private FunctionalDependencies(NestedSet uniqueSet, NestedSet uniformSet, ImmutableSet fdItems) { + private final ImmutableEqualSet equalSet; + + private FunctionalDependencies( + NestedSet uniqueSet, NestedSet uniformSet, ImmutableSet fdItems, ImmutableEqualSet equalSet) { this.uniqueSet = uniqueSet; this.uniformSet = uniformSet; this.fdItems = fdItems; + this.equalSet = equalSet; } public boolean isEmpty() { @@ -87,13 +94,26 @@ public boolean isUniformAndNotNull(ImmutableSet slotSet) { return slotSet.stream().noneMatch(Slot::nullable) && isUniform(slotSet); } + public boolean isNullSafeEqual(Slot l, Slot r) { + return equalSet.isEqual(l, r); + } + + public boolean isEqualAndNotNotNull(Slot l, Slot r) { + return equalSet.isEqual(l, r) && !l.nullable() && !r.nullable(); + } + + public List> calAllEqualSet() { + return equalSet.calEqualSetList(); + } + public ImmutableSet getFdItems() { return fdItems; } @Override public String toString() { - return String.format("FuncDeps[uniform:%s, unique:%s, fdItems:%s]", uniformSet, uniqueSet, fdItems); + return String.format("FuncDeps[uniform:%s, unique:%s, fdItems:%s, equalSet:%s]", + uniformSet, uniqueSet, fdItems, equalSet); } /** @@ -103,17 +123,21 @@ public static class Builder { private final NestedSet uniqueSet; private final NestedSet uniformSet; private ImmutableSet fdItems; + private final ImmutableEqualSet.Builder equalSetBuilder; public Builder() { uniqueSet = new NestedSet(); uniformSet = new NestedSet(); fdItems = new ImmutableSet.Builder().build(); + equalSetBuilder = new ImmutableEqualSet.Builder<>(); } public Builder(FunctionalDependencies other) { this.uniformSet = new NestedSet(other.uniformSet); this.uniqueSet = new NestedSet(other.uniqueSet); this.fdItems = ImmutableSet.copyOf(other.fdItems); + equalSetBuilder = new ImmutableEqualSet.Builder<>(other.equalSet); + } public void addUniformSlot(Slot slot) { @@ -147,11 +171,20 @@ public void addFdItems(ImmutableSet items) { public void addFunctionalDependencies(FunctionalDependencies fd) { uniformSet.add(fd.uniformSet); uniqueSet.add(fd.uniqueSet); + equalSetBuilder.addEqualSet(fd.equalSet); + } + + public void addEqualPair(Slot l, Slot r) { + equalSetBuilder.addEqualPair(l, r); + } + + public void addEqualSet(FunctionalDependencies functionalDependencies) { + equalSetBuilder.addEqualSet(functionalDependencies.equalSet); } public FunctionalDependencies build() { return new FunctionalDependencies(uniqueSet.toImmutable(), uniformSet.toImmutable(), - ImmutableSet.copyOf(fdItems)); + ImmutableSet.copyOf(fdItems), equalSetBuilder.build()); } public void pruneSlots(Set outputSlots) { @@ -162,6 +195,7 @@ public void pruneSlots(Set outputSlots) { public void replace(Map replaceMap) { uniformSet.replace(replaceMap); uniqueSet.replace(replaceMap); + equalSetBuilder.replace(replaceMap); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java index a679ad0f7d26b6..684643643714d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java @@ -46,4 +46,9 @@ default void computeUnique(FunctionalDependencies.Builder fdBuilder) { default void computeUniform(FunctionalDependencies.Builder fdBuilder) { // don't generate } + + @Override + default void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + // don't generate + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java index f37059049ae844..3d3d3cc8271408 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java @@ -66,4 +66,9 @@ default void computeUnique(FunctionalDependencies.Builder fdBuilder) { default void computeUniform(FunctionalDependencies.Builder fdBuilder) { fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } + + @Override + default void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 73ddde6cef5c19..031ea8a46b5e30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -397,4 +397,9 @@ public ImmutableSet computeFdItems() { return builder.build(); } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java index cad8d6e14d6a9f..a3f540d184778c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement; @@ -145,4 +146,9 @@ public void computeUniform(Builder fdBuilder) { getOutput().forEach(fdBuilder::addUniformSlot); } } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 433feb741bac10..277695be0796d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -215,4 +215,9 @@ private ImmutableSet findSlotsByColumn(Set outputSet, Set replaceMap = new HashMap<>(); + List output = getOutput(); + List originalOutputs = regularChildrenOutputs.isEmpty() + ? child(0).getOutput() + : regularChildrenOutputs.get(0); + for (int i = 0; i < output.size(); i++) { + replaceMap.put(originalOutputs.get(i), output.get(i)); + } + fdBuilder.replace(replaceMap); + } + @Override public void computeUniform(Builder fdBuilder) { fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 69d378fd6a5d80..45b96bda4519b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; @@ -166,4 +167,13 @@ public void computeUniform(Builder fdBuilder) { getConjuncts().forEach(e -> fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e))); fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } + + @Override + public void computeEqualSet(Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + for (Expression expression : getConjuncts()) { + Optional> equalSlot = ExpressionUtils.extractEqualSlot(expression); + equalSlot.ifPresent(slotSlotPair -> fdBuilder.addEqualPair(slotSlotPair.first, slotSlotPair.second)); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index a54a7514dbce43..c0195e11fed9db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -169,4 +170,9 @@ public void computeUnique(Builder fdBuilder) { public void computeUniform(Builder fdBuilder) { fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index da526c33af94ef..41ee1b14712a67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; @@ -138,6 +139,15 @@ public ImmutableSet computeFdItems() { return builder.build(); } + @Override + public void computeEqualSet(Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + for (Expression expression : getConjuncts()) { + Optional> equalSlot = ExpressionUtils.extractEqualSlot(expression); + equalSlot.ifPresent(slotSlotPair -> fdBuilder.addEqualPair(slotSlotPair.first, slotSlotPair.second)); + } + } + @Override public String toString() { return Utils.toSqlString("LogicalHaving", "predicates", getPredicate()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index e9e4889a8e553c..3d3ccd9ad8ed14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -139,6 +139,15 @@ public void computeUniform(Builder fdBuilder) { } } + @Override + public void computeEqualSet(Builder fdBuilder) { + for (Plan child : children) { + fdBuilder.addEqualSet( + child.getLogicalProperties().getFunctionalDependencies()); + replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput()); + } + } + @Override public ImmutableSet computeFdItems() { Set output = ImmutableSet.copyOf(getOutput()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index ea92aeca22d3fd..9ad0b5ab23f866 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -696,4 +696,20 @@ public void computeUniform(Builder fdBuilder) { fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies()); } } + + @Override + public void computeEqualSet(Builder fdBuilder) { + if (!joinType.isLeftSemiOrAntiJoin()) { + fdBuilder.addEqualSet(right().getLogicalProperties().getFunctionalDependencies()); + } + if (!joinType.isRightSemiOrAntiJoin()) { + fdBuilder.addEqualSet(left().getLogicalProperties().getFunctionalDependencies()); + } + if (joinType.isInnerJoin()) { + for (Expression expression : getHashJoinConjuncts()) { + Optional> equalSlot = ExpressionUtils.extractEqualSlot(expression); + equalSlot.ifPresent(slotSlotPair -> fdBuilder.addEqualPair(slotSlotPair.first, slotSlotPair.second)); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index 02558fe2ed2188..0fdf3212d88259 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -181,4 +181,9 @@ public ImmutableSet computeFdItems() { } return fdItems; } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 78a1e9a3a5e8b4..b6e5af2c591370 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -37,7 +38,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -157,4 +160,17 @@ public ImmutableSet computeFdItems() { return builder.build(); } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + Map aliasMap = new HashMap<>(); + for (NamedExpression namedExpr : getOutputs()) { + if (namedExpr instanceof Alias) { + if (aliasMap.containsKey(namedExpr.child(0))) { + fdBuilder.addEqualPair(namedExpr.toSlot(), aliasMap.get(namedExpr.child(0)).toSlot()); + } + aliasMap.put(namedExpr.child(0), namedExpr); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java index c0fb6ae7365cba..8c3be3b0fa46c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java @@ -63,6 +63,7 @@ default FunctionalDependencies computeFuncDeps() { FunctionalDependencies.Builder fdBuilder = new FunctionalDependencies.Builder(); computeUniform(fdBuilder); computeUnique(fdBuilder); + computeEqualSet(fdBuilder); ImmutableSet fdItems = computeFdItems(); fdBuilder.addFdItems(fdItems); return fdBuilder.build(); @@ -73,4 +74,6 @@ default FunctionalDependencies computeFuncDeps() { void computeUnique(FunctionalDependencies.Builder fdBuilder); void computeUniform(FunctionalDependencies.Builder fdBuilder); + + void computeEqualSet(FunctionalDependencies.Builder fdBuilder); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index ebecee0d3ae595..de23bc6f5b07c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -41,7 +42,9 @@ import com.google.common.collect.ImmutableSet; import org.json.JSONObject; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -282,4 +285,22 @@ public void computeUniform(FunctionalDependencies.Builder fdBuilder) { } fdBuilder.pruneSlots(getOutputSet()); } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + Map aliasMap = new HashMap<>(); + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + for (NamedExpression expr : getProjects()) { + if (expr instanceof Alias) { + if (aliasMap.containsKey(expr.child(0))) { + fdBuilder.addEqualPair(expr.toSlot(), aliasMap.get(expr.child(0)).toSlot()); + } + aliasMap.put(expr.child(0), expr); + if (expr.child(0).isSlot()) { + fdBuilder.addEqualPair(expr.toSlot(), (Slot) expr.child(0)); + } + } + } + fdBuilder.pruneSlots(getOutputSet()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 95ae92b686674b..9c24fab3352f36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -204,4 +204,9 @@ public ImmutableSet computeFdItems() { return builder.build(); } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child().getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSqlCache.java index 663044d569fc6e..26c3006d5e5c82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSqlCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSqlCache.java @@ -20,11 +20,10 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.FdItem; -import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.TreeStringPlan; @@ -36,14 +35,13 @@ import org.apache.doris.thrift.TUniqueId; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; /** LogicalSqlCache */ -public class LogicalSqlCache extends LogicalLeaf implements SqlCache, TreeStringPlan { +public class LogicalSqlCache extends LogicalLeaf implements SqlCache, TreeStringPlan, BlockFuncDepsPropagation { private final TUniqueId queryId; private final List columnLabels; private final List resultExprs; @@ -137,19 +135,4 @@ public List computeOutput() { public String getChildrenTreeString() { return planBody; } - - @Override - public ImmutableSet computeFdItems() { - return ImmutableSet.of(); - } - - @Override - public void computeUnique(Builder fdBuilder) { - - } - - @Override - public void computeUniform(Builder fdBuilder) { - - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index ea9fd143c9470c..5d23561b1a954c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -191,6 +191,17 @@ public ImmutableSet computeFdItems() { return ImmutableSet.of(); } + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies()); + Map replaceMap = new HashMap<>(); + List outputs = getOutput(); + for (int i = 0; i < outputs.size(); i++) { + replaceMap.put(child(0).getOutput().get(i), outputs.get(i)); + } + fdBuilder.replace(replaceMap); + } + public void setRelationId(RelationId relationId) { this.relationId = relationId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java index 791ded58cceb63..dd1c171ca2c8bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java @@ -187,6 +187,11 @@ public void computeUniform(FunctionalDependencies.Builder fdBuilder) { } } + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies()); + } + @Override public ImmutableSet computeFdItems() { ImmutableSet fdItems = child(0).getLogicalProperties().getFunctionalDependencies().getFdItems(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index 57d944bc2b9c67..8c5bbfcc6a60c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -36,7 +37,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -196,6 +202,61 @@ public void computeUniform(FunctionalDependencies.Builder fdBuilder) { // don't propagate uniform slots } + private List> mapSlotToIndex(Plan plan, List> equalSlotsList) { + Map slotToIndex = new HashMap<>(); + for (int i = 0; i < plan.getOutput().size(); i++) { + slotToIndex.put(plan.getOutput().get(i), i); + } + List> equalSlotIndicesList = new ArrayList<>(); + for (Set equalSlots : equalSlotsList) { + Set equalSlotIndices = new HashSet<>(); + for (Slot slot : equalSlots) { + if (slotToIndex.containsKey(slot)) { + equalSlotIndices.add(slotToIndex.get(slot)); + } + } + if (equalSlotIndices.size() > 1) { + equalSlotIndicesList.add(equalSlotIndices); + } + } + return equalSlotIndicesList; + } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + List> unionEqualSlotIndicesList = new ArrayList<>(); + + for (Plan child : children) { + List> childEqualSlotsList = + child.getLogicalProperties().getFunctionalDependencies().calAllEqualSet(); + List> childEqualSlotsIndicesList = mapSlotToIndex(child, childEqualSlotsList); + if (unionEqualSlotIndicesList.isEmpty()) { + unionEqualSlotIndicesList = childEqualSlotsIndicesList; + } else { + // Only all child of union has the equal pair, we keep the equal pair. + // It means we should calculate the intersection of all child + for (Set childEqualSlotIndices : childEqualSlotsIndicesList) { + for (Set unionEqualSlotIndices : unionEqualSlotIndicesList) { + if (Collections.disjoint(childEqualSlotIndices, unionEqualSlotIndices)) { + unionEqualSlotIndices.retainAll(childEqualSlotIndices); + } + } + } + } + + List ouputList = getOutput(); + for (Set equalSlotIndices : unionEqualSlotIndicesList) { + if (equalSlotIndices.size() <= 1) { + continue; + } + int first = equalSlotIndices.iterator().next(); + for (int idx : equalSlotIndices) { + fdBuilder.addEqualPair(ouputList.get(first), ouputList.get(idx)); + } + } + } + } + @Override public ImmutableSet computeFdItems() { Set output = ImmutableSet.copyOf(getOutput()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java index 200ccc2ffcbb42..371d3cae43fcff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -146,4 +147,9 @@ public void computeUnique(Builder fdBuilder) { public void computeUniform(Builder fdBuilder) { fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies()); } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java index b03a3365402bb0..56a306b60056c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; +import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -324,4 +325,9 @@ public void computeUniform(Builder fdBuilder) { } } } + + @Override + public void computeEqualSet(FunctionalDependencies.Builder fdBuilder) { + fdBuilder.addEqualSet(child(0).getLogicalProperties().getFunctionalDependencies()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 51eef31de2ffe3..a6a4d999a92539 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.MaterializedViewException; import org.apache.doris.common.NereidsException; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; @@ -130,6 +131,13 @@ private static void extract(Class type, Expression expr, C } } + public static Optional> extractEqualSlot(Expression expr) { + if (expr instanceof EqualTo && expr.child(0).isSlot() && expr.child(1).isSlot()) { + return Optional.of(Pair.of((Slot) expr.child(0), (Slot) expr.child(1))); + } + return Optional.empty(); + } + public static Optional optionalAnd(List expressions) { if (expressions.isEmpty()) { return Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEqualSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEqualSet.java index f5f3dd75b51bfc..35bc78ac117bba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEqualSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEqualSet.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; /** @@ -44,34 +45,66 @@ public static ImmutableEqualSet empty() { * Builder for ImmutableEqualSet. */ public static class Builder { - private final Map parent = new LinkedHashMap<>(); - private final Map size = new LinkedHashMap<>(); + private Map parent; + + Builder(Map parent) { + this.parent = parent; + } + + public Builder() { + this(new LinkedHashMap<>()); + } + + public Builder(ImmutableEqualSet equalSet) { + this(new LinkedHashMap<>(equalSet.root)); + } + + /** + * replace all key value according replace map + */ + public void replace(Map replaceMap) { + Map newMap = new LinkedHashMap<>(); + for (Entry entry : parent.entrySet()) { + newMap.put(replaceMap.getOrDefault(entry.getKey(), entry.getKey()), + replaceMap.getOrDefault(entry.getValue(), entry.getValue())); + } + parent = newMap; + } /** * Add a equal pair */ public void addEqualPair(T a, T b) { + if (!parent.containsKey(a)) { + parent.put(a, a); + } + if (!parent.containsKey(b)) { + parent.put(b, b); + } T root1 = findRoot(a); T root2 = findRoot(b); if (root1 != root2) { - parent.put(b, root1); - findRoot(b); + parent.put(root1, root2); } } - private T findRoot(T a) { - parent.putIfAbsent(a, a); // Ensure that the element is added - size.putIfAbsent(a, 1); // Initialize size to 1 + public void addEqualSet(ImmutableEqualSet equalSet) { + this.parent.putAll(equalSet.root); + } - if (!parent.get(a).equals(a)) { - parent.put(a, findRoot(parent.get(a))); // Path compression + private T findRoot(T a) { + if (a.equals(parent.get(a))) { + return parent.get(a); } - return parent.get(a); + return findRoot(parent.get(a)); } public ImmutableEqualSet build() { - parent.keySet().forEach(this::findRoot); - return new ImmutableEqualSet<>(parent); + ImmutableMap.Builder foldMapBuilder = new ImmutableMap.Builder<>(); + for (T k : parent.keySet()) { + foldMapBuilder.put(k, findRoot(k)); + } + return new ImmutableEqualSet<>(foldMapBuilder.build()); } } @@ -103,4 +136,16 @@ public List> calEqualSetList() { public Set getAllItemSet() { return ImmutableSet.copyOf(root.keySet()); } + + public boolean isEqual(T l, T r) { + if (!root.containsKey(l) || !root.containsKey(r)) { + return false; + } + return root.get(l) == root.get(r); + } + + @Override + public String toString() { + return root.toString(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/EqualSetTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/EqualSetTest.java new file mode 100644 index 00000000000000..fec795ccbe2809 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/EqualSetTest.java @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.properties; + +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class EqualSetTest extends TestWithFeService { + Slot slot1 = new SlotReference("1", IntegerType.INSTANCE, false); + Slot slot2 = new SlotReference("2", IntegerType.INSTANCE, false); + Slot slot3 = new SlotReference("1", IntegerType.INSTANCE, false); + Slot slot4 = new SlotReference("1", IntegerType.INSTANCE, false); + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + createTable("create table test.agg (\n" + + "id int not null,\n" + + "id2 int replace not null,\n" + + "name varchar(128) replace not null )\n" + + "AGGREGATE KEY(id)\n" + + "distributed by hash(id) buckets 10\n" + + "properties('replication_num' = '1');"); + createTable("create table test.uni (\n" + + "id int not null,\n" + + "id2 int not null,\n" + + "name varchar(128) not null)\n" + + "UNIQUE KEY(id)\n" + + "distributed by hash(id) buckets 10\n" + + "properties('replication_num' = '1');"); + connectContext.setDatabase("test"); + } + + @Test + void testAgg() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id group by id, id2") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + + @Test + void testTopNLimit() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id limit 1") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id limit 1 order by id") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + + @Test + void testSetOp() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id intersect select id, id2 from agg") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id except select id, id2 from agg") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id union all select id, id2 from agg") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isEmpty()); + plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id union all select id, id2 from agg where id2 = id") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg union all select id, id2 from agg where id2 = id") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isEmpty()); + } + + @Test + void testFilterHaving() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg where id2 = id") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg group by id, id2 having id = id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + + @Test + void testGenerate() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from agg lateral view explode([1,2,3]) tmp1 as e1 where id = id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + + @Test + void testJoin() { + // inner join + Plan plan = PlanChecker.from(connectContext) + .analyze("select uni.id, agg.id from agg join uni " + + "where agg.id = uni.id") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + + // foj + plan = PlanChecker.from(connectContext) + .analyze("select t1.id, t2.id, t3.id from agg as t1 join uni as t2 " + + " on t1.id = t2.id full outer join uni as t3 on t1.id2 = t2.id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies() + .isEqualAndNotNotNull(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(2))); + + // loj + plan = PlanChecker.from(connectContext) + .analyze("select t1.id, t2.id, t3.id from agg as t1 join uni as t2 " + + " on t1.id = t2.id left outer join uni as t3 on t1.id2 = t2.id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isEqualAndNotNotNull(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(2))); + + // roj + plan = PlanChecker.from(connectContext) + .analyze("select t1.id, t2.id, t3.id from agg as t1 join uni as t2 " + + " on t1.id = t2.id right outer join uni as t3 on t1.id2 = t2.id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies() + .isEqualAndNotNotNull(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies() + .isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(2))); + } + + @Test + void testOneRowRelation() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select 1, 1") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + + @Test + void testProject() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id as o1, id as o2, id2 as o4, 1 as c1, 1 as c2 from uni where id = id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(2))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(1), plan.getOutput().get(2))); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(3), plan.getOutput().get(4))); + } + + @Test + void testSubQuery() { + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2 from (select id, id2 from agg where id = id2) t") + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + + @Test + void testWindow() { + // partition by uniform + Plan plan = PlanChecker.from(connectContext) + .analyze("select id, id2, row_number() over(partition by id) from agg where id = id2") + .rewrite() + .getPlan(); + Assertions.assertTrue(plan.getLogicalProperties() + .getFunctionalDependencies().isNullSafeEqual(plan.getOutput().get(0), plan.getOutput().get(1))); + } + +} diff --git a/regression-test/suites/nereids_syntax_p0/join_order.groovy b/regression-test/suites/nereids_syntax_p0/join_order.groovy index 64056e0202bc84..461c2bb448522a 100644 --- a/regression-test/suites/nereids_syntax_p0/join_order.groovy +++ b/regression-test/suites/nereids_syntax_p0/join_order.groovy @@ -43,7 +43,7 @@ suite("join_order") { """ sql """ drop table if exists outerjoin_C_order;""" sql """ - create table outerjoin_C_order ( c int not null ) + create table outerjoin_C_order ( c int not null, c2 int not null ) ENGINE=OLAP DISTRIBUTED BY HASH(c) BUCKETS 1 PROPERTIES ( @@ -77,7 +77,7 @@ suite("join_order") { sql """insert into outerjoin_A_order values( 1,2 );""" sql """insert into outerjoin_B_order values( 1 );""" - sql """insert into outerjoin_C_order values( 1 );""" + sql """insert into outerjoin_C_order values( 1,2 );""" sql """insert into outerjoin_D_order values( 1,2,3 );""" sql """insert into outerjoin_E_order values( 1,2 );""" @@ -96,13 +96,13 @@ suite("join_order") { sql 'set disable_join_reorder=true;' explain { - sql("select * from outerjoin_A_order, outerjoin_B_order, outerjoin_C_order where outerjoin_A_order.a1 = outerjoin_C_order.c and outerjoin_B_order.b = outerjoin_C_order.c;") + sql("select * from outerjoin_A_order, outerjoin_B_order, outerjoin_C_order where outerjoin_A_order.a1 = outerjoin_C_order.c and outerjoin_B_order.b = outerjoin_C_order.c2;") contains "CROSS JOIN" } sql 'set disable_join_reorder=false;' explain { - sql("select * from outerjoin_A_order, outerjoin_B_order, outerjoin_C_order where outerjoin_A_order.a1 = outerjoin_C_order.c and outerjoin_B_order.b = outerjoin_C_order.c;") + sql("select * from outerjoin_A_order, outerjoin_B_order, outerjoin_C_order where outerjoin_A_order.a1 = outerjoin_C_order.c and outerjoin_B_order.b = outerjoin_C_order.c2;") notContains "CROSS JOIN" } From dfe30f542d89f74c008f05d1b7241c8d33c94d6c Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:42:35 +0800 Subject: [PATCH 019/163] [fix](Nereids) support aggregate function only in having statement (#34086) SQL like > SELECT 1 AS c1 FROM t HAVING count(1) > 0 OR c1 IS NOT NULL --- .../rules/analysis/FillUpMissingSlots.java | 71 +++++++++++++++---- .../test_having_with_aggregate_function.out | 4 ++ ...test_having_with_aggregate_function.groovy | 32 +++++++++ 3 files changed, 92 insertions(+), 15 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.out create mode 100644 regression-test/suites/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java index 82de4453c131f1..1cab3614302381 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; @@ -158,20 +159,52 @@ public List buildRules() { // Convert having to filter RuleType.FILL_UP_HAVING_PROJECT.build( logicalHaving(logicalProject()).then(having -> { - LogicalProject project = having.child(); - Set projectOutputSet = project.getOutputSet(); - Set notExistedInProject = having.getExpressions().stream() - .map(Expression::getInputSlots) - .flatMap(Set::stream) - .filter(s -> !projectOutputSet.contains(s)) - .collect(Collectors.toSet()); - if (notExistedInProject.isEmpty()) { - return null; + if (having.getExpressions().stream().anyMatch(e -> e.containsType(AggregateFunction.class))) { + // This is very weird pattern. + // There are some aggregate functions in having, but its child is project. + // There are some slot from project in having too. + // Having should execute after project. + // But aggregate function should execute before project. + // Since no aggregate here, we should add an empty aggregate before project. + // We should push aggregate function into aggregate node first. + // Then put aggregate result slots and original project slots into new project. + // The final plan should be + // Having + // +-- Project + // +-- Aggregate + // Since aggregate node have no group by key. + // So project should not contain any slot from its original child. + // Or otherwise slot cannot find will be thrown. + LogicalProject project = having.child(); + // new an empty agg here + LogicalAggregate agg = new LogicalAggregate<>( + ImmutableList.of(), ImmutableList.of(), project.child()); + // avoid throw exception even if having have slot from its child. + // because we will add a project between having and project. + Resolver resolver = new Resolver(agg, false); + having.getConjuncts().forEach(resolver::resolve); + agg = agg.withAggOutput(resolver.getNewOutputSlots()); + Set newConjuncts = ExpressionUtils.replace( + having.getConjuncts(), resolver.getSubstitution()); + ImmutableList.Builder projects = ImmutableList.builder(); + projects.addAll(project.getOutputs()).addAll(agg.getOutput()); + return new LogicalHaving<>(newConjuncts, new LogicalProject<>(projects.build(), agg)); + } else { + LogicalProject project = having.child(); + Set projectOutputSet = project.getOutputSet(); + Set notExistedInProject = having.getExpressions().stream() + .map(Expression::getInputSlots) + .flatMap(Set::stream) + .filter(s -> !projectOutputSet.contains(s)) + .collect(Collectors.toSet()); + if (notExistedInProject.isEmpty()) { + return null; + } + List projects = ImmutableList.builder() + .addAll(project.getProjects()).addAll(notExistedInProject).build(); + return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), + having.withChildren(new LogicalProject<>(projects, project.child()))); } - List projects = ImmutableList.builder() - .addAll(project.getProjects()).addAll(notExistedInProject).build(); - return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()), - having.withChildren(new LogicalProject<>(projects, project.child()))); }) ) ); @@ -184,13 +217,19 @@ static class Resolver { private final Map substitution = Maps.newHashMap(); private final List newOutputSlots = Lists.newArrayList(); private final Map outputSubstitutionMap; + private final boolean checkSlot; - Resolver(Aggregate aggregate) { + Resolver(Aggregate aggregate, boolean checkSlot) { outputExpressions = aggregate.getOutputExpressions(); groupByExpressions = aggregate.getGroupByExpressions(); outputSubstitutionMap = outputExpressions.stream().filter(Alias.class::isInstance) .collect(Collectors.toMap(NamedExpression::toSlot, alias -> alias.child(0), (k1, k2) -> k1)); + this.checkSlot = checkSlot; + } + + Resolver(Aggregate aggregate) { + this(aggregate, true); } public void resolve(Expression expression) { @@ -218,7 +257,9 @@ public void resolve(Expression expression) { // We couldn't find the equivalent expression in output expressions and group-by expressions, // so we should check whether the expression is valid. if (expression instanceof SlotReference) { - throw new AnalysisException(expression.toSql() + " should be grouped by."); + if (checkSlot) { + throw new AnalysisException(expression.toSql() + " should be grouped by."); + } } else if (expression instanceof AggregateFunction) { if (checkWhetherNestedAggregateFunctionsExist((AggregateFunction) expression)) { throw new AnalysisException("Aggregate functions in having clause can't be nested: " diff --git a/regression-test/data/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.out b/regression-test/data/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.out new file mode 100644 index 00000000000000..2bef87ab2b7a2d --- /dev/null +++ b/regression-test/data/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !having_project_with_having_count_1_and_slot_from_project -- +1 + diff --git a/regression-test/suites/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.groovy b/regression-test/suites/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.groovy new file mode 100644 index 00000000000000..9047a7c85bf456 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/fill_up_missing_slots/test_having_with_aggregate_function.groovy @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_having_project") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql """ + DROP TABLE IF EXISTS t + """ + sql """ + create table t(id smallint) distributed by random properties('replication_num'='1'); + """ + + qt_having_project_with_having_count_1_and_slot_from_project """ + SELECT 1 AS c1 FROM t HAVING count(1) > 0 OR c1 IS NOT NULL + """ +} From c3402ad2cbf0b9f082954418eaa86fd1603f9144 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 25 Apr 2024 16:43:31 +0800 Subject: [PATCH 020/163] [Chore](regression-test) adjust variant tpch/q09_trans.sql batch_size from default to 2048 (#34105) since 50 may cause performance issue introduced by #33853 --- regression-test/suites/variant_p0/tpch/sql/q09_trans.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/variant_p0/tpch/sql/q09_trans.sql b/regression-test/suites/variant_p0/tpch/sql/q09_trans.sql index ddbc82812bc5b2..f501d73dd727aa 100644 --- a/regression-test/suites/variant_p0/tpch/sql/q09_trans.sql +++ b/regression-test/suites/variant_p0/tpch/sql/q09_trans.sql @@ -1,6 +1,6 @@ -- TABLES: part,SUPPLIER,lineitem,partsupp,orders,NATION -- ERROR: not stable -SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false,batch_size=2048) */ NATION, O_YEAR, SUM(AMOUNT) AS SUM_PROFIT From f79e0ca6015b8d5475d967e85ce678cc653178d2 Mon Sep 17 00:00:00 2001 From: Pxl Date: Thu, 25 Apr 2024 17:30:22 +0800 Subject: [PATCH 021/163] [Bug](runtime-filter) fix bloom filter size error on rf merge (#34082) fix bloom filter size error on rf merge W20240424 11:28:56.826277 3494287 ref_count_closure.h:80] RPC meet error status: [INVALID_ARGUMENT]PStatus: (172.21.0.15)[INVALID_ARGUMENT]bloom filter size not the same: already allocated bytes 65536, expected allocated bytes 32768 --- be/src/exprs/bloom_filter_func.h | 2 +- be/src/exprs/runtime_filter.cpp | 88 +++++++++++--------- be/src/pipeline/exec/hashjoin_build_sink.cpp | 4 +- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index 987c2b0c05d9b7..bc56c7b505afa1 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -167,7 +167,7 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase { DCHECK(bloomfilter_func != nullptr); auto* other_func = static_cast(bloomfilter_func); if (_bloom_filter_alloced != other_func->_bloom_filter_alloced) { - return Status::InvalidArgument( + return Status::InternalError( "bloom filter size not the same: already allocated bytes {}, expected " "allocated bytes {}", _bloom_filter_alloced, other_func->_bloom_filter_alloced); diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 174ce8f3fe5104..c77fdcd903ac77 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -246,7 +246,7 @@ Status create_vbin_predicate(const TypeDescriptor& type, TExprOpcode::type opcod fn_name.__set_function_name("ge"); break; default: - return Status::InvalidArgument( + return Status::InternalError( strings::Substitute("Invalid opcode for max_min_runtimefilter: '$0'", opcode)); } fn.__set_name(fn_name); @@ -282,17 +282,16 @@ class RuntimePredicateWrapper { public: RuntimePredicateWrapper(ObjectPool* pool, const RuntimeFilterParams* params) : RuntimePredicateWrapper(pool, params->column_return_type, params->filter_type, - params->filter_id, params->build_bf_exactly) {}; + params->filter_id) {}; // for a 'tmp' runtime predicate wrapper // only could called assign method or as a param for merge RuntimePredicateWrapper(ObjectPool* pool, PrimitiveType column_type, RuntimeFilterType type, - uint32_t filter_id, bool build_bf_exactly = false) + uint32_t filter_id) : _pool(pool), _column_return_type(column_type), _filter_type(type), _context(new RuntimeFilterContext()), - _filter_id(filter_id), - _build_bf_exactly(build_bf_exactly) {} + _filter_id(filter_id) {} // init runtime filter wrapper // alloc memory to init runtime filter function @@ -333,24 +332,26 @@ class RuntimePredicateWrapper { return Status::OK(); } default: - return Status::InvalidArgument("Unknown Filter type"); + return Status::InternalError("Unknown Filter type"); } return Status::OK(); } - Status change_to_bloom_filter(bool need_init_bf = false) { - CHECK(_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) - << "Can not change to bloom filter because of runtime filter type is " - << IRuntimeFilter::to_string(_filter_type); + Status change_to_bloom_filter() { + if (_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) { + return Status::InternalError( + "Can not change to bloom filter because of runtime filter type is {}", + IRuntimeFilter::to_string(_filter_type)); + } BloomFilterFuncBase* bf = _context->bloom_filter_func.get(); - if (need_init_bf) { - // BloomFilter may be not init - RETURN_IF_ERROR(bf->init_with_fixed_length()); + + if (bf != nullptr) { insert_to_bloom_filter(bf); - } else { - DCHECK(_context->hybrid_set == nullptr || _context->hybrid_set->size() == 0) - << "set size: " << (_context->hybrid_set ? _context->hybrid_set->size() : 0); + } else if (_context->hybrid_set != nullptr && _context->hybrid_set->size() != 0) { + return Status::InternalError("change to bloom filter need empty set ", + IRuntimeFilter::to_string(_filter_type)); } + // release in filter _context->hybrid_set.reset(); return Status::OK(); @@ -514,26 +515,27 @@ class RuntimePredicateWrapper { } if (real_filter_type == RuntimeFilterType::IN_FILTER) { - if (other_filter_type == RuntimeFilterType::IN_FILTER) { // in merge in + // when we meet base rf is in-filter, threre only have two case: + // case1: all input-filter's build_bf_exactly is true, inited by synced global size + // case2: all input-filter's build_bf_exactly is false, inited by default size + if (other_filter_type == RuntimeFilterType::IN_FILTER) { _context->hybrid_set->insert(wrapper->_context->hybrid_set.get()); if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) { - VLOG_DEBUG << " change runtime filter to bloom filter(id=" << _filter_id - << ") because: in_num(" << _context->hybrid_set->size() - << ") >= max_in_num(" << _max_in_num << ")"; - RETURN_IF_ERROR(change_to_bloom_filter(true)); + // case2: use default size to init bf + RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length()); + RETURN_IF_ERROR(change_to_bloom_filter()); } } else { - VLOG_DEBUG << " change runtime filter to bloom filter(id=" << _filter_id - << ") because: already exist a bloom filter"; - RETURN_IF_ERROR(change_to_bloom_filter(!_build_bf_exactly)); - RETURN_IF_ERROR(_context->bloom_filter_func->merge( - wrapper->_context->bloom_filter_func.get())); + // case1&case2: use input bf directly and insert hybrid set data into bf + _context->bloom_filter_func = wrapper->_context->bloom_filter_func; + RETURN_IF_ERROR(change_to_bloom_filter()); } } else { - if (other_filter_type == RuntimeFilterType::IN_FILTER) { // bloom filter merge in + if (other_filter_type == RuntimeFilterType::IN_FILTER) { + // case2: insert data to global filter wrapper->insert_to_bloom_filter(_context->bloom_filter_func.get()); - // bloom filter merge bloom filter } else { + // case1&case2: all input bf must has same size RETURN_IF_ERROR(_context->bloom_filter_func->merge( wrapper->_context->bloom_filter_func.get())); } @@ -716,9 +718,8 @@ class RuntimePredicateWrapper { break; } default: { - DCHECK(false) << "unknown type: " << type_to_string(type); - return Status::InvalidArgument("not support assign to in filter, type: " + - type_to_string(type)); + return Status::InternalError("not support assign to in filter, type: " + + type_to_string(type)); } } return Status::OK(); @@ -870,10 +871,9 @@ class RuntimePredicateWrapper { return _context->minmax_func->assign(&min_val, &max_val); } default: - DCHECK(false) << "unknown type"; break; } - return Status::InvalidArgument("not support!"); + return Status::InternalError("not support!"); } HybridSetBase::IteratorBase* get_in_filter_iterator() { return _context->hybrid_set->begin(); } @@ -950,7 +950,6 @@ class RuntimePredicateWrapper { SharedRuntimeFilterContext _context; uint32_t _filter_id; - bool _build_bf_exactly; }; Status IRuntimeFilter::create(RuntimeFilterParamsContext* state, ObjectPool* pool, @@ -1375,11 +1374,11 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue } if (_runtime_filter_type == RuntimeFilterType::BITMAP_FILTER) { if (!build_ctx->root()->type().is_bitmap_type()) { - return Status::InvalidArgument("Unexpected src expr type:{} for bitmap filter.", - build_ctx->root()->type().debug_string()); + return Status::InternalError("Unexpected src expr type:{} for bitmap filter.", + build_ctx->root()->type().debug_string()); } if (!desc->__isset.bitmap_target_expr) { - return Status::InvalidArgument("Unknown bitmap filter target expr."); + return Status::InternalError("Unknown bitmap filter target expr."); } vectorized::VExprContextSPtr bitmap_target_ctx; RETURN_IF_ERROR( @@ -1455,7 +1454,7 @@ Status IRuntimeFilter::create_wrapper(const UpdateRuntimeFilterParamsV2* param, return (*wrapper)->assign(¶m->request->minmax_filter(), param->request->contain_null()); } default: - return Status::InvalidArgument("unknown filter type"); + return Status::InternalError("unknown filter type"); } } @@ -1504,7 +1503,7 @@ Status IRuntimeFilter::_create_wrapper(const T* param, ObjectPool* pool, return (*wrapper)->assign(¶m->request->minmax_filter(), param->request->contain_null()); } default: - return Status::InvalidArgument("unknown filter type"); + return Status::InternalError("unknown filter type"); } } @@ -1523,7 +1522,14 @@ void IRuntimeFilter::update_runtime_filter_type_to_profile() { } Status IRuntimeFilter::merge_from(const RuntimePredicateWrapper* wrapper) { - return _wrapper->merge(wrapper); + auto status = _wrapper->merge(wrapper); + if (!status) { + LOG(WARNING) << "runtime filter merge failed: " << _name + << " ,need_local_merge: " << _need_local_merge + << " ,is_broadcast: " << _is_broadcast_join; + DCHECK(false); // rpc response is often ignored, so let it crash directly here + } + return status; } template @@ -1558,7 +1564,7 @@ Status IRuntimeFilter::serialize_impl(T* request, void** data, int* len) { auto minmax_filter = request->mutable_minmax_filter(); to_protobuf(minmax_filter); } else { - return Status::InvalidArgument("not implemented !"); + return Status::InternalError("not implemented !"); } return Status::OK(); } diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index 2b2bdad86f70f8..da3614e4479fce 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -137,7 +137,9 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu uint64_t hash_table_size = block ? block->rows() : 0; { SCOPED_TIMER(_runtime_filter_init_timer); - RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size)); + if (_should_build_hash_table) { + RETURN_IF_ERROR(_runtime_filter_slots->init_filters(state, hash_table_size)); + } RETURN_IF_ERROR(_runtime_filter_slots->ignore_filters(state)); } if (_should_build_hash_table && hash_table_size > 1) { From 2efe79d9e56d8a230e865fd44fb2b627746f7116 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Thu, 25 Apr 2024 19:45:09 +0800 Subject: [PATCH 022/163] [test](streamload) add load empty file regression test (#34110) --- .../load_p0/stream_load/test_empty_file.csv | 0 .../test_stream_load_empty_file.out | 3 + .../test_stream_load_empty_file.groovy | 73 +++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 regression-test/data/load_p0/stream_load/test_empty_file.csv create mode 100644 regression-test/data/load_p0/stream_load/test_stream_load_empty_file.out create mode 100644 regression-test/suites/load_p0/stream_load/test_stream_load_empty_file.groovy diff --git a/regression-test/data/load_p0/stream_load/test_empty_file.csv b/regression-test/data/load_p0/stream_load/test_empty_file.csv new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/regression-test/data/load_p0/stream_load/test_stream_load_empty_file.out b/regression-test/data/load_p0/stream_load/test_stream_load_empty_file.out new file mode 100644 index 00000000000000..9c9c4c6c8a2be1 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_stream_load_empty_file.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- + diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_empty_file.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_empty_file.groovy new file mode 100644 index 00000000000000..9265280ecae7a0 --- /dev/null +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_empty_file.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_stream_load_empty_file", "p0") { + def tableName = "test_stream_load_empty_file" + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` bigint(20) NULL, + `k2` bigint(20) NULL, + `v1` tinyint(4) SUM NULL, + `v2` tinyint(4) REPLACE NULL, + `v3` tinyint(4) REPLACE_IF_NOT_NULL NULL, + `v4` smallint(6) REPLACE_IF_NOT_NULL NULL, + `v5` int(11) REPLACE_IF_NOT_NULL NULL, + `v6` bigint(20) REPLACE_IF_NOT_NULL NULL, + `v7` largeint(40) REPLACE_IF_NOT_NULL NULL, + `v8` datetime REPLACE_IF_NOT_NULL NULL, + `v9` date REPLACE_IF_NOT_NULL NULL, + `v10` char(10) REPLACE_IF_NOT_NULL NULL, + `v11` varchar(6) REPLACE_IF_NOT_NULL NULL, + `v12` decimal(27, 9) REPLACE_IF_NOT_NULL NULL + ) ENGINE=OLAP + AGGREGATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + PARTITION BY RANGE(`k1`) + (PARTITION partition_a VALUES [("-9223372036854775808"), ("100000")), + PARTITION partition_b VALUES [("100000"), ("1000000000")), + PARTITION partition_c VALUES [("1000000000"), ("10000000000")), + PARTITION partition_d VALUES [("10000000000"), (MAXVALUE))) + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + // test strict_mode success + streamLoad { + table "${tableName}" + + file 'test_empty_file.csv' + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(0, json.NumberTotalRows) + } + time 10000 // limit inflight 10s + } + + sql "sync" + qt_sql "select * from ${tableName}" + } finally { + sql """ DROP TABLE IF EXISTS ${tableName} """ + } +} \ No newline at end of file From 77e53ca970db3b26c22cc20d5e4e62a650cc9147 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com> Date: Thu, 25 Apr 2024 19:59:49 +0800 Subject: [PATCH 023/163] [fix](cloud) exclude some cases from cloud_p0 (#33725) --- bin/start_be.sh | 10 ++-- .../system/test_query_sys_rowsets.out | 27 ++++++++++ .../query_p0/system/test_query_sys_tables.out | 28 ----------- .../pipeline/cloud_p0/conf/be_custom.conf | 1 + .../conf/regression-conf-custom.groovy | 6 ++- .../test_map_load_and_compaction.groovy | 2 +- .../test_stream_load_properties.groovy | 6 ++- .../select_tablets/select_with_tablets.groovy | 3 +- .../system/test_query_sys_rowsets.groovy | 49 +++++++++++++++++++ .../system/test_query_sys_tables.groovy | 28 ----------- 10 files changed, 94 insertions(+), 66 deletions(-) create mode 100644 regression-test/data/query_p0/system/test_query_sys_rowsets.out create mode 100644 regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy diff --git a/bin/start_be.sh b/bin/start_be.sh index 518eecc9740c63..743c2f98a97ec6 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -187,10 +187,6 @@ export ODBCSYSINI="${DORIS_HOME}/conf" # support utf8 for oracle database export NLS_LANG='AMERICAN_AMERICA.AL32UTF8' -# filter known leak -export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/lsan_suppr.conf" -export ASAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf" - while read -r line; do envline="$(echo "${line}" | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | @@ -251,9 +247,13 @@ fi export AWS_MAX_ATTEMPTS=2 +# filter known leak +export LSAN_OPTIONS=suppressions=${DORIS_HOME}/conf/lsan_suppr.conf +export ASAN_OPTIONS=suppressions=${DORIS_HOME}/conf/asan_suppr.conf + ## set asan and ubsan env to generate core file ## detect_container_overflow=0, https://github.com/google/sanitizers/issues/193 -export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0:check_malloc_usable_size=0 +export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0:check_malloc_usable_size=0:${ASAN_OPTIONS} export UBSAN_OPTIONS=print_stacktrace=1 ## set TCMALLOC_HEAP_LIMIT_MB to limit memory used by tcmalloc diff --git a/regression-test/data/query_p0/system/test_query_sys_rowsets.out b/regression-test/data/query_p0/system/test_query_sys_rowsets.out new file mode 100644 index 00000000000000..958d57d68e44ba --- /dev/null +++ b/regression-test/data/query_p0/system/test_query_sys_rowsets.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_rowsets -- +BACKEND_ID BIGINT Yes false \N +ROWSET_ID VARCHAR(64) Yes false \N +TABLET_ID BIGINT Yes false \N +ROWSET_NUM_ROWS BIGINT Yes false \N +TXN_ID BIGINT Yes false \N +NUM_SEGMENTS BIGINT Yes false \N +START_VERSION BIGINT Yes false \N +END_VERSION BIGINT Yes false \N +INDEX_DISK_SIZE BIGINT Yes false \N +DATA_DISK_SIZE BIGINT Yes false \N +CREATION_TIME BIGINT Yes false \N +NEWEST_WRITE_TIMESTAMP BIGINT Yes false \N + +-- !rowsets1 -- +0 1 + +-- !rowsets2 -- +0 1 +2 2 + +-- !rowsets3 -- +0 1 +2 2 +3 3 +4 4 \ No newline at end of file diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 91f54556ae8a35..74574ae818ba62 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -154,40 +154,12 @@ TABLESPACE_NAME VARCHAR(268) Yes false \N -- !select_partitions -- --- !desc_rowsets -- -BACKEND_ID BIGINT Yes false \N -ROWSET_ID VARCHAR(64) Yes false \N -TABLET_ID BIGINT Yes false \N -ROWSET_NUM_ROWS BIGINT Yes false \N -TXN_ID BIGINT Yes false \N -NUM_SEGMENTS BIGINT Yes false \N -START_VERSION BIGINT Yes false \N -END_VERSION BIGINT Yes false \N -INDEX_DISK_SIZE BIGINT Yes false \N -DATA_DISK_SIZE BIGINT Yes false \N -CREATION_TIME BIGINT Yes false \N -NEWEST_WRITE_TIMESTAMP BIGINT Yes false \N - --- !rowsets1 -- -0 1 - --- !rowsets2 -- -0 1 -2 2 - --- !rowsets3 -- -0 1 -2 2 -3 3 -4 4 - -- !schemata -- internal test_query_sys_db_1 \N internal test_query_sys_db_2 \N internal test_query_sys_db_3 \N -- !tables -- -internal test_query_rowset BASE TABLE 0 \N \N internal test_query_sys_tb_1 BASE TABLE 0 \N \N internal test_query_sys_tb_2 BASE TABLE 0 \N \N internal test_query_sys_tb_3 BASE TABLE 0 \N \N diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf index 1f4104304fcaf8..9f85d1c98fac37 100644 --- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf @@ -29,6 +29,7 @@ meta_service_use_load_balancer = false enable_file_cache = true file_cache_path = [{"path":"/data/doris_cloud/file_cache","total_size":104857600,"query_limit":104857600}] tmp_file_dirs = [{"path":"/data/doris_cloud/tmp","max_cache_bytes":104857600,"max_upload_bytes":104857600}] +thrift_rpc_timeout_ms = 360000 # For debug sys_log_verbose_modules=vrow_distribution,tablet_info sys_log_verbose_level=7 \ No newline at end of file diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy index d7e02406196881..0810ac8a403ff3 100644 --- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy +++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy @@ -39,9 +39,11 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th "test_set_partition_version," + "test_show_transaction," + // not supported yet "test_spark_load," + - "test_array_index1," + - "test_array_index2," + "test_index_lowercase_fault_injection," + + "test_partial_update_2pc_schema_change," + // mow 2pc + "test_query_sys_rowsets," + // rowsets sys table + "test_unique_table_debug_data," + // disable auto compaction + "test_insert," + // txn insert "zzz_the_end_sentinel_do_not_touch" // keep this line as the last line excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line diff --git a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy index 4a80004169b46f..703f5dde345587 100644 --- a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy +++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy @@ -77,7 +77,7 @@ suite("test_map_load_and_compaction", "p0") { for (String rowset in (List) compactStatusJson.rowsets) { rowsetsCount += Integer.parseInt(rowset.split(" ")[1]) } - assertTrue(assertRowSetNum==rowsetsCount) + assertEquals(assertRowSetNum, rowsetsCount) } diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy index 83dc83efdfeaf5..15ef2fbb5c68b9 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_properties.groovy @@ -741,7 +741,11 @@ suite("test_stream_load_properties", "p0") { // Commit the same txnId again to trigger operate_txn_2pc() failure body = do_streamload_2pc.call(txnId, "commit", tableName1) - assertEquals("analysis_error", parseJson(body).status.toLowerCase()) + if (isCloudMode()) { + assertEquals("success", parseJson(body).status.toLowerCase()) + } else { + assertEquals("analysis_error", parseJson(body).status.toLowerCase()) + } assertTrue(parseJson(body).msg.toLowerCase().contains("is already visible")) i++ diff --git a/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy b/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy index e3de6448678771..ffaac34200e479 100644 --- a/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy +++ b/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy @@ -46,7 +46,8 @@ suite("select_with_tablets") { def res = sql_return_maparray """ show tablets from ${table_name1} where version = 2 """ res = deduplicate_tablets(res) - assertTrue(res.size() == 1) + log.info("res: " + res.toString()) + assertEquals(res.size(), 1) assertEquals("2", res[0].Version) order_qt_select2 """ SELECT * FROM ${table_name1} TABLET(${res[0].TabletId}) """ diff --git a/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy b/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy new file mode 100644 index 00000000000000..1f159e3832979e --- /dev/null +++ b/regression-test/suites/query_p0/system/test_query_sys_rowsets.groovy @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_rowsets", "query,p0") { + def dbName1 = "test_query_sys_rowsets" + + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + + // test rowsets + qt_desc_rowsets """ desc information_schema.rowsets """ + def rowsets_table_name = """ test_query_sys_rowsets.test_query_rowset """ + sql """ drop table if exists ${rowsets_table_name} """ + + sql """ + create table ${rowsets_table_name}( + a int , + b boolean , + c string ) + DISTRIBUTED BY HASH(`a`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_single_replica_compaction"="true" + ); + """ + + List> rowsets_table_name_tablets = sql """ show tablets from ${rowsets_table_name} """ + order_qt_rowsets1 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ + sql """ insert into ${rowsets_table_name} values (1,0,"abc"); """ + order_qt_rowsets2 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ + sql """ insert into ${rowsets_table_name} values (2,1,"hello world"); """ + sql """ insert into ${rowsets_table_name} values (3,0,"dssadasdsafafdf"); """ + order_qt_rowsets3 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ + +} \ No newline at end of file diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index 7d943894168410..b8f14da041b634 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -139,34 +139,6 @@ suite("test_query_sys_tables", "query,p0") { qt_desc_partitions """ desc `information_schema`.`partitions` """ order_qt_select_partitions """ select * from `information_schema`.`partitions`; """ - - // test rowsets - qt_desc_rowsets """ desc information_schema.rowsets """ - def rowsets_table_name = """ test_query_sys_db_1.test_query_rowset """ - sql """ drop table if exists ${rowsets_table_name} """ - - sql """ - create table ${rowsets_table_name}( - a int , - b boolean , - c string ) - DISTRIBUTED BY HASH(`a`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1", - "disable_auto_compaction" = "true", - "enable_single_replica_compaction"="true" - ); - """ - - List> rowsets_table_name_tablets = sql """ show tablets from ${rowsets_table_name} """ - order_qt_rowsets1 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ - sql """ insert into ${rowsets_table_name} values (1,0,"abc"); """ - order_qt_rowsets2 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ - sql """ insert into ${rowsets_table_name} values (2,1,"hello world"); """ - sql """ insert into ${rowsets_table_name} values (3,0,"dssadasdsafafdf"); """ - order_qt_rowsets3 """ select START_VERSION,END_VERSION from information_schema.rowsets where TABLET_ID=${rowsets_table_name_tablets[0][0]} order by START_VERSION,END_VERSION; """ - - // test schemata // create test dbs sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") From 044f4f9aa8fffeeb35d790478784077366f21af6 Mon Sep 17 00:00:00 2001 From: huanghg1994 <519500479@qq.com> Date: Thu, 25 Apr 2024 20:05:03 +0800 Subject: [PATCH 024/163] [fix](resource)fix check available fail when s3 aws_token is set and reset as, sk faild on be. (#34057) --- be/src/agent/task_worker_pool.cpp | 2 ++ be/src/io/fs/s3_file_system.cpp | 12 ++++++++---- .../org/apache/doris/catalog/S3Resource.java | 16 +++++++++++++--- .../property/constants/S3Properties.java | 1 + gensrc/thrift/AgentService.thrift | 1 + .../cold_heat_separation/policy/alter.groovy | 11 +++++++++++ 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 10590a7faeae16..3b8d63f7a95dbe 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1365,6 +1365,7 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr .region = param.s3_storage_param.region, .ak = param.s3_storage_param.ak, .sk = param.s3_storage_param.sk, + .token = param.s3_storage_param.token, .max_connections = param.s3_storage_param.max_conn, .request_timeout_ms = param.s3_storage_param.request_timeout_ms, .connect_timeout_ms = param.s3_storage_param.conn_timeout_ms, @@ -1384,6 +1385,7 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr S3ClientConf conf { .ak = param.s3_storage_param.ak, .sk = param.s3_storage_param.sk, + .token = param.s3_storage_param.token, }; st = client->reset(conf); fs = std::move(existed_fs); diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 3d9f25686a88f8..dea3279350900d 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -115,16 +115,21 @@ Status S3ClientHolder::init() { } Status S3ClientHolder::reset(const S3ClientConf& conf) { + S3ClientConf reset_conf; { std::shared_lock lock(_mtx); - if (conf.ak == _conf.ak && conf.sk == _conf.sk) { + if (conf.ak == _conf.ak && conf.sk == _conf.sk && conf.token == _conf.token) { return Status::OK(); // Same conf } + reset_conf = _conf; + reset_conf.ak = conf.ak; + reset_conf.sk = conf.sk; + reset_conf.token = conf.token; // Should check endpoint here? } - auto client = S3ClientFactory::instance().create(conf); + auto client = S3ClientFactory::instance().create(reset_conf); if (!client) { return Status::InternalError("failed to init s3 client with conf {}", conf.to_string()); } @@ -134,8 +139,7 @@ Status S3ClientHolder::reset(const S3ClientConf& conf) { { std::lock_guard lock(_mtx); _client = std::move(client); - _conf.ak = conf.ak; - _conf.sk = conf.sk; + _conf = std::move(reset_conf); } return Status::OK(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 5bcb5123c64f52..a26038970477a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -105,7 +105,8 @@ protected void setProperties(Map properties) throws DdlException properties.putIfAbsent(S3Properties.REGION, region); String ak = properties.get(S3Properties.ACCESS_KEY); String sk = properties.get(S3Properties.SECRET_KEY); - CloudCredentialWithEndpoint credential = new CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk); + String token = properties.get(S3Properties.SESSION_TOKEN); + CloudCredentialWithEndpoint credential = new CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk, token); if (needCheck) { String bucketName = properties.get(S3Properties.BUCKET); @@ -123,6 +124,7 @@ private static void pingS3(CloudCredentialWithEndpoint credential, String bucket Map propertiesPing = new HashMap<>(); propertiesPing.put(S3Properties.Env.ACCESS_KEY, credential.getAccessKey()); propertiesPing.put(S3Properties.Env.SECRET_KEY, credential.getSecretKey()); + propertiesPing.put(S3Properties.Env.TOKEN, credential.getSessionToken()); propertiesPing.put(S3Properties.Env.ENDPOINT, credential.getEndpoint()); propertiesPing.put(S3Properties.Env.REGION, credential.getRegion()); propertiesPing.put(PropertyConverter.USE_PATH_STYLE, @@ -188,6 +190,10 @@ public void modifyProperties(Map properties) throws DdlException writeLock(); for (Map.Entry kv : properties.entrySet()) { replaceIfEffectiveValue(this.properties, kv.getKey(), kv.getValue()); + if (kv.getKey().equals(S3Properties.Env.TOKEN) + || kv.getKey().equals(S3Properties.SESSION_TOKEN)) { + this.properties.put(kv.getKey(), kv.getValue()); + } } ++version; writeUnlock(); @@ -197,11 +203,13 @@ public void modifyProperties(Map properties) throws DdlException private CloudCredentialWithEndpoint getS3PingCredentials(Map properties) { String ak = properties.getOrDefault(S3Properties.ACCESS_KEY, this.properties.get(S3Properties.ACCESS_KEY)); String sk = properties.getOrDefault(S3Properties.SECRET_KEY, this.properties.get(S3Properties.SECRET_KEY)); + String token = properties.getOrDefault(S3Properties.SESSION_TOKEN, + this.properties.get(S3Properties.SESSION_TOKEN)); String endpoint = properties.getOrDefault(S3Properties.ENDPOINT, this.properties.get(S3Properties.ENDPOINT)); String pingEndpoint = "http://" + endpoint; String region = S3Properties.getRegionOfEndpoint(pingEndpoint); properties.putIfAbsent(S3Properties.REGION, region); - return new CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk); + return new CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk, token); } private boolean isNeedCheck(Map newProperties) { @@ -231,7 +239,9 @@ protected void getProcNodeData(BaseProcResult result) { // it's dangerous to show password in show odbc resource, // so we use empty string to replace the real password if (entry.getKey().equals(S3Properties.Env.SECRET_KEY) - || entry.getKey().equals(S3Properties.SECRET_KEY)) { + || entry.getKey().equals(S3Properties.SECRET_KEY) + || entry.getKey().equals(S3Properties.Env.TOKEN) + || entry.getKey().equals(S3Properties.SESSION_TOKEN)) { result.addRow(Lists.newArrayList(name, lowerCaseType, entry.getKey(), "******")); } else { result.addRow(Lists.newArrayList(name, lowerCaseType, entry.getKey(), entry.getValue())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 947174f86ef600..26bd37b0cac0e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -265,6 +265,7 @@ public static TS3StorageParam getS3TStorageParam(Map properties) s3Info.setRegion(properties.get(S3Properties.REGION)); s3Info.setAk(properties.get(S3Properties.ACCESS_KEY)); s3Info.setSk(properties.get(S3Properties.SECRET_KEY)); + s3Info.setToken(properties.get(S3Properties.SESSION_TOKEN)); s3Info.setRootPath(properties.get(S3Properties.ROOT_PATH)); s3Info.setBucket(properties.get(S3Properties.BUCKET)); diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index df478a75a1ba70..4e381b168052ad 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -73,6 +73,7 @@ struct TS3StorageParam { 8: optional string root_path 9: optional string bucket 10: optional bool use_path_style = false + 11: optional string token } struct TStoragePolicy { diff --git a/regression-test/suites/cold_heat_separation/policy/alter.groovy b/regression-test/suites/cold_heat_separation/policy/alter.groovy index 672f79427630b0..cfd8e0da6fcfd2 100644 --- a/regression-test/suites/cold_heat_separation/policy/alter.groovy +++ b/regression-test/suites/cold_heat_separation/policy/alter.groovy @@ -39,6 +39,7 @@ suite("alter_policy") { "AWS_ROOT_PATH" = "path/to/rootaaaa", "AWS_ACCESS_KEY" = "bbba", "AWS_SECRET_KEY" = "aaaa", + "AWS_TOKEN" = "session_token", "AWS_MAX_CONNECTIONS" = "50", "AWS_REQUEST_TIMEOUT_MS" = "3000", "AWS_CONNECTION_TIMEOUT_MS" = "1000", @@ -70,6 +71,10 @@ suite("alter_policy") { ALTER RESOURCE "${resource_name}" PROPERTIES("AWS_REQUEST_TIMEOUT_MS" = "7777"); """ + def alter_result_succ_8 = try_sql """ + ALTER RESOURCE "${resource_name}" PROPERTIES("AWS_TOKEN" = "new_session_token"); + """ + // errCode = 2, detailMessage = current not support modify property : AWS_REGION def alter_result_fail_1 = try_sql """ ALTER RESOURCE "${resource_name}" PROPERTIES("AWS_REGION" = "8888"); @@ -112,6 +117,7 @@ suite("alter_policy") { // [has_resouce_policy_alter, s3, AWS_REQUEST_TIMEOUT_MS, 7777], // [has_resouce_policy_alter, s3, AWS_ROOT_PATH, path/to/rootaaaa], // [has_resouce_policy_alter, s3, AWS_SECRET_KEY, ******], + // [has_resouce_policy_alter, s3, AWS_TOKEN, ******], // [has_resouce_policy_alter, s3, id, {id}], // [has_resouce_policy_alter, s3, type, s3] // [has_resouce_policy_alter, s3, version, {version}]] @@ -133,6 +139,8 @@ suite("alter_policy") { assertEquals(show_alter_result[8][3], "10101010") // AWS_SECRET_KEY assertEquals(show_alter_result[9][3], "******") + // AWS_SECRET_KEY + assertEquals(show_alter_result[10][3], "******") } def check_alter_resource_result_with_policy = { resource_name -> @@ -151,6 +159,7 @@ suite("alter_policy") { // [has_resouce_policy_alter, s3, AWS_REQUEST_TIMEOUT_MS, 7777], // [has_resouce_policy_alter, s3, AWS_ROOT_PATH, path/to/rootaaaa], // [has_resouce_policy_alter, s3, AWS_SECRET_KEY, ******], + // [has_resouce_policy_alter, s3, AWS_TOKEN, ******], // [has_resouce_policy_alter, s3, id, {id}], // [has_resouce_policy_alter, s3, type, s3] // [has_resouce_policy_alter, s3, version, {version}]] @@ -172,6 +181,8 @@ suite("alter_policy") { assertEquals(show_alter_result[8][3], "path/to/rootaaaa") // AWS_SECRET_KEY assertEquals(show_alter_result[9][3], "******") + // AWS_SECRET_KEY + assertEquals(show_alter_result[10][3], "******") } From a6bc92edc5870481d9a7091f422054589d394150 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Thu, 25 Apr 2024 20:15:04 +0800 Subject: [PATCH 025/163] [RegressionTest](test) Add fuzzy test config for column string overflow and p2 case (#34091) Add fuzzy test config for column string overflow and p2 case --- be/src/common/config.cpp | 4 + be/src/common/config.h | 3 + be/src/vec/columns/column_string.cpp | 3 +- .../load_from_big_lateral_view.out | 721 ++++++++++++++++++ .../string_over_flow/string_over_flow.groovy | 24 + 5 files changed, 753 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out create mode 100644 regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index de1458c240dff3..11a8b97842df9d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1216,6 +1216,8 @@ DEFINE_mBool(enable_injection_point, "false"); DEFINE_mBool(ignore_schema_change_check, "false"); +DEFINE_mInt64(string_overflow_size, "4294967295"); // std::numic_limits::max() + // The min thread num for BufferedReaderPrefetchThreadPool DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread, "16"); // The max thread num for BufferedReaderPrefetchThreadPool @@ -1663,6 +1665,8 @@ Status set_fuzzy_configs() { ((distribution(*generator) % 2) == 0) ? "true" : "false"; fuzzy_field_and_value["enable_shrink_memory"] = ((distribution(*generator) % 2) == 0) ? "true" : "false"; + fuzzy_field_and_value["string_overflow_size"] = + ((distribution(*generator) % 2) == 0) ? "10" : "4294967295"; fmt::memory_buffer buf; for (auto& it : fuzzy_field_and_value) { diff --git a/be/src/common/config.h b/be/src/common/config.h index 4139d76b6bcb7a..fe7009c7691fe9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1295,6 +1295,9 @@ DECLARE_mBool(enable_injection_point); DECLARE_mBool(ignore_schema_change_check); +/** Only use in fuzzy test **/ +DECLARE_mInt64(string_overflow_size); + // The min thread num for BufferedReaderPrefetchThreadPool DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_min_thread); // The max thread num for BufferedReaderPrefetchThreadPool diff --git a/be/src/vec/columns/column_string.cpp b/be/src/vec/columns/column_string.cpp index 843b2fa72a7eb4..514980cc8bab5c 100644 --- a/be/src/vec/columns/column_string.cpp +++ b/be/src/vec/columns/column_string.cpp @@ -608,8 +608,7 @@ ColumnPtr ColumnStr::index(const IColumn& indexes, size_t limit) const { template ColumnPtr ColumnStr::convert_column_if_overflow() { - // TODO: Try to fuzzy the overflow size to test more case in CI - if (std::is_same_v && chars.size() > std::numeric_limits::max()) { + if (std::is_same_v && chars.size() > config::string_overflow_size) { auto new_col = ColumnStr::create(); const auto length = offsets.size(); diff --git a/regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out b/regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out new file mode 100644 index 00000000000000..8426d697d00e96 --- /dev/null +++ b/regression-test/data/query_p2/string_over_flow/load_from_big_lateral_view.out @@ -0,0 +1,721 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- + above the unusual accounts. slyl + according to the blith + accounts haggle blithely above the s + across the fu + along the slyly regula + alongside of + alongside of the carefully exp + among the ironic, final requests ma + asymptotes + beans. de + blithely + blithely bold forges above t + blithely regular deposits cajole fluff + blithely. quickly final deposits wa + bold accounts. fluffil + bold foxes sleep finally + boost blithely after the inst + cajole quickly regular, bold packa + cajole regular packages. special, ir + carefully even ideas cajole + carefully even packages are carefully pend + carefully regular ac + carefully regular instructions. regula + deposits run blithely furiously special re + deposits sleep furiously across the pendin + deposits. e + deposits. furiously close t + deposits. ironic, regular foxes throu + dolphins are furiously slyly regular depo + even deposi + even deposits. carefully ir + even foxes. final pinto beans sleep + even instructions according to the furio + even packages co + excuses. bl + express foxes + express, regular packages hag + final dep + final pinto + furiously blithely unusual requests. + furiously bold dolphins; slyly expr + furiously bold platel + furiously quickly bold frets. bra + furiously regular accounts cajole carefu + furiously regular pinto be + furiously stealthy deposits cajole around + haggle carefully. reg + ideas use blithely a + instructions sleep + ironic accounts. special theod + ironic packages. + ironic requests nag accord + ironic, even accounts. accounts use b + of the careful + on the furiously permanent dol + orbits haggle + packages a + packages according to the final, even p + packages sleep blithely fluffily final + packages. busy, + packages. car + packages. careful + packages. carefull + pending foxes cajole slyly carefull + pending ideas. quickly quiet requests i + pending, pending platelets nag e + pinto beans above the furiously + pinto beans are slyly + poach unusual foxes. fo + quickly even accounts. furiously i + quickly pending p + quickly regular ideas: express, final requ + quietly carefull + regular deposits cajole blithel + regular deposits unwind slyly al + regular packag + regular, even instructions lose s + requests cajole qu + requests. blithely ironic instructions sl + requests. quickly regula + silent orbits across the + sleep blithely at the slow packages! bli + slyly above th + slyly across the + slyly at + slyly even ideas. + slyly ironic accoun + slyly pending + slyly special theodolit + special excuses. carefully expres + special packages after the d + special, silent deposi + the ideas. + the regular, ironic excuses. furious + the unusual accounts hagg + theodolites. quickly final deposits + unusual instructions doze bol + use about the carefully pending deposits. +, express ideas. carefully regular wa +, express packages accord +, final platelets. quick theodolites us +, ironic accounts +, ironic instructions. iro +- carefully daring foxes wake furiously. +. blithely final decoy +. carefully regular packages nod +. fluffily final platelets play ca +. furiously bold theodolites +. furiously expre +. furiously ironic dependenc +. ironic i +above the quickly ironic dependenc +accounts ar +accounts boost sl +accounts haggle foxes. un +accounts. ironic accounts haggle aft +accounts? +ackages whitho +ackages. carefully special instructio +ackages. silent theodolites detect alo +across the furiousl +across the s +after the quickly +ag slyly pac +ages affix according to the express accoun +ages boost. even accounts haggle carefull +ages haggle slyly among the ironic, r +ages haggle. pending deposits kind +ages. slyly ironic +aggle acro +aggle carefu +aggle quickly requests. fluffily final +ainst the furiously +aintain idly ironic, express +ajole quickly. ironic accounts cajo +ake furiously. furiously bold deposits nag +ake slyly about the +ake. carefully bold deposits boost. i +ake. silent dependencies x-ray finally. pac +al accounts. slyly final foxes aff +al dolphins. so +al foxes caj +al foxes. slyly final theodolit +al ideas should have to integrate +al ideas sleep acc +al ideas. busy requests wake car +al packages use fluffily alon +al, express requests. blithely exp +alms sleep furiou +along the quickly final theodolite +althily regular requests. f +ans wake after the evenly p +ans. regular, special ideas ha +apades are alongside of the saut +ar pinto beans about the bold, unusua +ar pinto beans. carefully unusual pinto bea +ar theodolites along the final +are carefully deposits! furiously bold pac +arefully dogged, regular packages. +arefully final dependenc +arefully. care +arthogs lose at +arthogs lose quickly +as. slyly silent dolphins sublat +at the thin, ironic packages +beans use carefully abo +beans. even, ironic dep +blithely e +blithely ironi +blithely. furiou +bold foxes cajole blithely after the flu +bold, silent accounts detect carefully un +boost against the slyly regular pl +boost slyly +c packages. ironic accounts believe +c packages. slyly express shea +carefully f +carefully ir +carefully pending requ +carefully quiet packages wake blithely alon +carefully speci +ccounts along the ironic re +ccounts wake i +ccounts wake quickly. furiousl +ccounts. slyly iron +ch carefully excuses. carefully e +cies according to the fluffily ruthless +ckages affix. ca +ckages snooze never after the carefull +ckly above the quickly +ckly bold asymptotes cajole blit +ckly even, busy pinto beans. +ckly express pinto b +ckly ironic dolphins. +ckly pending foxes. deposits ha +counts according +counts across the special, unusual p +counts. blithe +cross the pending +d ideas. requests +d theodolites cajo +de of the br +deas was beside +deas. furiously bold packages alo +dencies. bold theodolites haggle. blit +dependencies among the packa +dependencies. p +deposits along the furiously special ac +deposits nag. carefully ironic depos +ding deposits above the c +doggedly ironic ideas +doubt among the ironic, regular +doubt ironic multipliers. quick +e above the blithely +e accounts. regular, even theodolites do en +e blithely bold foxes. accounts serve furi +e busily pending requests. fu +e carefull +e carefully regular foxes sleep furio +e carefully. bl +e fluffily bold de +e fluffily final account +e fluffily unusual hockey players. sl +e furiously final foxes. fur +e ironically even requests. sl +e pinto beans. quickly unusual +e quickly even requests sleep furiousl +e the blithely unusual packag +e the carefully final d +e. deposits alon +e. fluffy depths x-ray around the furious +e. furiously +ecial dependenc +ecial deposits. blithely even requests +ecial foxes affix furiously agains +ecial theodolites. carefully bol +eep blithely. ruthless pac +eep silently de +efully against +efully express requests. boldly silent theo +efully. quickly express deposits use. iro +efully: slyly e +egular accounts. unusual, special reques +egular deposits nag about t +egular ideas n +egular ideas. blit +egular packages integrate blithel +egular pearls integrate carefully. sl +egular pla +egularly regu +ely after the fluffily bold +ely silent foxes +en account +en deposits cajole slyly fin +en escapades +en excuses beside the furiously final acco +encies affix slyly: unusual deposits +encies snooz +endencies. furiously express depend +ending dependencie +ending deposits. fluffi +ending packages wake around the +eodolites. bold, brave patterns sleep. qui +ep ironic dependencies. regula +ep slyly. slyly brave deposits +ependencies. +eposits sleep blithely final id +eposits wake furiously unusual th +eposits. blithely unusual idea +equests. fina +equests. final +ernes use at +erve daringly quick packages. +es detect. deposits wa +es sleep. slyly even packages a +es use final, final dependencies. pl +ess packages. ironic, ironic +ests boost according +ests haggle. carefully pending +ests. blithely even deposit +ests. regular, final theodolites sleep thi +evenly special deposits. ir +express re +f the quickly pend +ffily bold +ffily bold +ffily bold accounts. final, regular +ffily busy packages haggle +fily final packages boost furiously sil +fily regular deposits. sl +final deposits wake furiously pen +final, final deposits. quickly regular pin +final, ironic accounts. regular +foxes wake quickly plat +foxes. fluffily ironic theodolites affi +fter the accounts. fluffily ironic packag +ful deposits. blithely +fully even i +fully final theodolites +furiously b +furiously final accounts. request +furiously quiet pinto beans boos +furiously. +g pinto beans. +g requests wake car +g the final, pending pinto be +g the ironic, regular +g to the regular, r +gage fluffily ru +gage. slyly pending platelets in pl +gainst the special request +ges are sometimes special dolphins. +ges boost along the fu +ges sleep. final deposits sleep +ges-- carefully ironic accounts +ges. furious +ges. regular theodolites +ges; regular theodolites cajo +gifts x-ray carefully +gle slyly final deposits. daringl +gle. packages haggle furiously a +gular pinto beans. finally fin +gularly expre +haggle furiously alo +haggle-- even foxes affix. de +have to caj +he fluffily silent accounts. careful +he furiously bold de +he ironic plat +he ironic, express realms. furiously ironic +he quickly bold ideas might +he slyly special +he slyly special asympt +hely pending reques +hely regular theodolites. bold p +hins nag furiously. regular theodolites +hins. ideas dazzle a +hlessly even accou +hogs. furiously regular accounts again +ial foxes. furiously final instructions abo +ic asymptotes cajole even packages. +ic theodolites. +icingly even accounts use +ick ideas. foxes along the special +ickly ironic deposits +ickly regula +ideas. furiously regular r +idly across the final, special +ies. special e +ilent accounts. even dependencies haggl +ily express pac +ily ironic +iments wake. slyly pending requests hinde +inal accounts. express, r +ince the furiously final a +ins sleep fluffily about t +inst the carefully bold instructions +instructions. slow +into beans alo +into beans cajole blithely about the +iously ironic deposits sleep blithely bl +ironic instruct +ites must have to inte +ites nag slyly alongside of th +ites. furio +ites. slyly special ideas wake furiously ag +ithely about the fluffily even requests. +ithely according to t +ithely among the slyly regular excus +ithely idle foxes nod alongside of the +ithely ironic accounts solve +ithely ironic requests! furiously regula +ithely pending f +ithely. sly +its doze furi +ix carefully silently final +jole quickly fluf +kages sleep. blithely regul +kages wake above the asymptotes. carefull +kages. blith +kages. blithely +ke carefully carefu +key players. sil +l accounts-- furi +l deposits sleep blithely about the qui +l packages are carefully above the +l packages. blithely permanent ideas +l requests. bl +l tithes. packages play quickly according +l, thin foxes boost slyly platelets +lar ideas nag alongside of the slyly ironi +lar pinto beans dazz +lar requests. blithely expr +le packages wake slyly among the +le quickly slyly regu +le slyly af +le. packages boost slyly. final, ex +lent foxes cajole fur +lent packages. silent, ironic asym +lently regular deposits. carefully unusua +lently spe +less warthogs wake fluff +lites against the carefully +lites use furiously alongside of the +lites. blithely silent +lites. carefully ironic courts alongsid +lithely bold packages sleep fluffily. f +lithely furiously +lithely slyly bold pinto beans. e +lly furious request +lly regular excuses. qui +lly unusual deposits impress. doggedly bo +long the c +long the carefully fin +longside of the quic +ls. busily regular reques +lve furiously am +ly about the ir +ly above the blithely final foxes. iro +ly according to the fluffily final packages +ly among the slyly even deposits. even ex +ly bold foxes. ironic, expres +ly even deposits are fluffily after t +ly even deposits. a +ly even packages. ironic packag +ly furiously +ly idly bold requests. evenl +ly ironic warthogs are slyly regular, +ly pending requests bo +ly quickly ev +ly regular instructions integrate carefu +ly regular requests. thinly bold deposi +ly regular theodolites wake th +ly silent deposits are bra +ly silent requests haggle f +ly special war +ly stealth +ly. carefully speci +ly. fluffily +lyly after the furiously regular instr +lyly ironic instructions run unusual pack +lyly slyly ironic pinto beans. foxes +mptotes. furiously final +must boost. qu +n requests. final, final packages haggle +n, final deposits wake slyly blithe +nal ideas integrate slyly furiously final +nal sauternes cajole. ironic pac +nal, express courts c +nal, regular deposits p +nal, unusual theodolites +ncies hang aroun +ndencies sleep fluffily +nding pearls. furiously even ep +nding request +nding requests wake above the carefully b +ng deposits d +ng ideas above the final instructi +ng the carefully sil +ng the quickly reg +ng the regular fo +ng the unusual, special inst +ng, even courts nod furiously. careful +ngside of the fu +ngside of the slyly express reque +nic depths. even deposits about the +nic foxes. ideas are. sly +nic frays hag +ns haggle against +ns. carefully pending requests us +ns. furiously even deposits integrat +nst the furiously r +nstructions slee +ntly ironic instructions u +nto beans. f +nts are. furiously enticing packages +nts detect quickly against the blit +nts sleep slyly ironic accounts. ironic pac +nts use bold, pending +nusual sheaves wake blithely am +o the slyly final packages. furiously fi +odolites. deposits haggle car +old ideas. blithely final depo +olites. ca +olites. daringly express req +olphins. blithe +ong the regular packages x-ray fluffi +ongside of the regular theodoli +onic foxes. quickly iron +onic packages cajole +ons. fluffily final foxes above the +oost busily ironic, ir +orges. unusual packages cajole furiou +osits. ideas hang after +otes wake enticing dep +ound the ideas haggle even, fluffy ideas. +ounts are carefull +ounts cajole closely regular reque +ounts must have to +ounts. ideas ar +ounts. slyly ruthless dugouts wake +ously bold courts. regular +ously ironic accounts. even dependencies +ously regular requests caj +ously special instructions among the +out the blithely +out the carefully ironic pa +ove the requests sleep above +p alongside of the ironic, unusual in +p slyly. theodolites alongside +p. furiously pending in +packages cajol +packages wake sometim +packages. packages integrate carefull +packages. slyly regular foxes boost. slyly +packages? slyly regular pin +pecial pinto beans ha +pecial pinto beans. regular packages +permanent courts detec +pinto beans haggle carefully bold req +pon the pending, ev +press, special +pths sleep along th +ptotes: th +quests are furiously express theodo +quests nag sly +quests. fluffi +quickly express ideas. u +r accounts. blithely regular deposits in +r instructions slee +r packages cajole sometimes +r pinto beans are fluffily a +r the carefully +r the regular, express requests are fur +rate. pending packages against the +ray. slyly regular accounts sleep slowly +rding to the depths. ironic foxes in +re slyly carefu +refully even deposits. accounts use +regular asymptotes are quickly regular pint +regular dolphins wake +regular frets serve furiously. +regular ideas. slyly express pack +regular packages. blithely final foxes wake +requests sleep +ress, express instructi +riously bold deposits sleep +riously express ideas. Tiresias at t +riously slyly ironic pinto be +riously special packages n +ronic packages sleep. blithely unus +round the exp +s cajole furiously express request +s cajole slyly even req +s except the furiously regular requ +s hinder fu +s integrate. slyly ironic foxes wake quick +s outside the permanent, even instructio +s sleep bravely +s sleep carefully. slyly special +s sleep. idle multipliers w +s the express, silent reque +s wake evenl +s wake furiously regular asymp +s wake quickly. slyly pending p +s. blithely final re +s. final ideas boost slyl +s. quickly regular accounts acros +s. slyly special accounts sleep blithely +s. special packages a +s; ironic, silent pain +s? final, ironic accounts cajole quickly +se quickly? +se theodolites integrate +side of the expr +side of the quickly exp +silent pack +sits cajole blithely after the regularly i +sits sleep furiously fin +sits. furiously even packages snooze across +sleep slyly iron +sly after the even accounts. blithe +sly final packages along the bold packag +sly theodolites haggle carefully quickly +slyly ironi +slyly slyly regular theodo +solve-- slyly silent instructio +ss the regular accounts. daringly fin +ss, bold courts boos +st the carefully regular ideas- +st the ironic dep +sts. ironi +sual pinto beans boost blithely re +sublate according to the unusual, f +symptotes breach carefully according to th +symptotes wake carefully pending i +t deposits boost blithely s +t the blithely unusual deposits. bu +t the quickly pen +tainments. +telets across the final sheaves h +ter the package +tes are silent +tes kindle fluffily blit +tes. final, express deposits against the +the carefully regular ideas cajo +the furiou +the furiously even acc +the furiously f +the ideas. caref +the instructions. carefully ironic pa +the quickly bold packages. carefu +the regular instructions. quickly even pack +the slyly +the slyly final requests: blithe packa +the warhorses nag enticingly at th +thely carefully silent dep +thely never even accounts. express, fi +thely regular deposits detect furiously. +times. furiously idle instr +tions affix. blithe +tions. blithely special foxes above the un +tructions about the quickly final +ts above the quickly unus +ts are quickl +ts haggle above the final as +ts nag. blithely +ts poach. carefully darin +ts use around +ts use furiousl +ts wake. blithely sil +ts. carefully regular th +ual packages haggl +ual, ironic ac +uests boost along the final, regular acc +uests need to wake fu +uests snooze blith +ugouts are blithely regu +uickly according +uickly alongside of the care +ular accounts nag blithely iron +ular accounts sublate careful +ular accounts! +ular excuses. fluffily s +ular packages. final p +ular pinto bean +ully bold packages. s +ully thin wa +und the final foxes. enticing p +und the packages. express exc +unts affix sl +unts nag blithely b +unts promise. i +unts. furiousl +unusual asymptotes cajole around the fu +unusual excuses. ironic, even account +unusual, regular foxes sno +uring the carefully eve +uriously at the pending, final frets. acco +uriously even pi +uriously even requests. slyly fi +uriously final accounts. furiously pending +uriously pending +uriously regular +uriously regular foxes. ev +use. quietl +usily final +usly regular +usly. regular p +usual accounts cajole a +usy packages. even, sly ideas sleep slyl +ve dolphins sleep permanentl +ve the careful +ve to nag furiously. fluffily iro +ven packages. even requests across the +ven, final ins +ven, special foxes. pending inst +wake acros +wake carefully +wake furio +wake furiously about the regular request +would detect ironic, even pinto be +x-ray slyly fluffily bus +xcuses thrash according to th +xpress accounts. regular, regular pint +xpress depos +xpress deposits. requests shall ha +xpress escapades. fl +y according to the express +y according to the fluf +y according to the regular, fluff +y against the quickly ironic pin +y bold pains? final pinto beans +y final accounts. carefully even theo +y final asymptotes. blithely fluffy req +y final foxe +y final foxes. fluffily fina +y final requests wake busily e +y final requests wake. blithely +y final sentiments. reg +y ironic braids during the +y ironic grouches. blithely even +y ironic instructions +y ironic packages boost permanentl +y pending instructions hagg +y regular requests nag furio +y! even accou +y. close accounts according to the final +y. regular pinto b +y. slyly fin +yly express instructions cajole +yly final instructions across th +yly final requests cajole furiou +yly ironic theodolites at the quickly iron +yly ruthless accounts wake carefull +ymptotes above the pend +ymptotes dazzle; bold, regular re +ys carefully regular instructi + diff --git a/regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy b/regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy new file mode 100644 index 00000000000000..71f95493138d9b --- /dev/null +++ b/regression-test/suites/query_p2/string_over_flow/string_over_flow.groovy @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("string_over_flow") { + def dbName = "regression_test_tpch_sf100_p2" + sql "USE ${dbName}" + sql "set parallel_pipeline_task_num = 4;" + qt_sql """select b.l_comment from nation a join[broadcast] lineitem b on a.n_nationkey = b.l_partkey order by 1;""" +} + From 85f37257c2238ebc4e69ada3acc7ac089f847dfb Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 20:39:25 +0800 Subject: [PATCH 026/163] [feature](Cache) Limit cache usage of TTL (#34084) --- be/src/common/config.cpp | 2 + be/src/common/config.h | 2 + be/src/io/cache/block_file_cache.cpp | 13 +++- be/src/io/cache/block_file_cache.h | 1 + be/src/io/cache/file_block.cpp | 5 ++ be/src/io/cache/file_block.h | 2 + be/test/io/cache/block_file_cache_test.cpp | 79 ++++++++++++++++++---- 7 files changed, 89 insertions(+), 15 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 11a8b97842df9d..e1c8114800a521 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1226,6 +1226,8 @@ DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread, "64"); DEFINE_Int64(num_s3_file_upload_thread_pool_min_thread, "16"); // The max thread num for S3FileUploadThreadPool DEFINE_Int64(num_s3_file_upload_thread_pool_max_thread, "64"); +// The max ratio for ttl cache's size +DEFINE_mInt64(max_ttl_cache_ratio, "90"); // clang-format off #ifdef BE_TEST diff --git a/be/src/common/config.h b/be/src/common/config.h index fe7009c7691fe9..ce244da36d53b9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1306,6 +1306,8 @@ DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread); DECLARE_Int64(num_s3_file_upload_thread_pool_min_thread); // The max thread num for S3FileUploadThreadPool DECLARE_Int64(num_s3_file_upload_thread_pool_max_thread); +// The max ratio for ttl cache's size +DECLARE_mInt64(max_ttl_cache_ratio); #ifdef BE_TEST // test s3 diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index b41cb9f6a5fed9..26ca8e47596f75 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -623,7 +623,9 @@ BlockFileCache::FileBlockCell* BlockFileCache::add_cell(const UInt128Wrapper& ha auto& offsets = _files[hash]; DCHECK((context.expiration_time == 0 && context.cache_type != FileCacheType::TTL) || - (context.cache_type == FileCacheType::TTL && context.expiration_time != 0)); + (context.cache_type == FileCacheType::TTL && context.expiration_time != 0)) + << fmt::format("expiration time {}, cache type {}", context.expiration_time, + context.cache_type); FileCacheKey key; key.hash = hash; @@ -639,6 +641,7 @@ BlockFileCache::FileBlockCell* BlockFileCache::add_cell(const UInt128Wrapper& ha _key_to_time[hash] = context.expiration_time; _time_to_key.insert(std::make_pair(context.expiration_time, hash)); } + _cur_ttl_size += cell.size(); } auto [it, _] = offsets.insert(std::make_pair(offset, std::move(cell))); _cur_cache_size += size; @@ -695,6 +698,10 @@ const BlockFileCache::LRUQueue& BlockFileCache::get_queue(FileCacheType type) co bool BlockFileCache::try_reserve_for_ttl(size_t size, std::lock_guard& cache_lock) { size_t removed_size = 0; size_t cur_cache_size = _cur_cache_size; + auto limit = config::max_ttl_cache_ratio * _capacity; + if ((_cur_ttl_size + size) * 100 > limit) { + return false; + } auto is_overflow = [&] { return _disk_resource_limit_mode ? removed_size < size : cur_cache_size + size - removed_size > _capacity; @@ -1129,6 +1136,9 @@ void BlockFileCache::remove(FileBlockSPtr file_block, T& cache_lock, U& block_lo } } _cur_cache_size -= file_block->range().size(); + if (FileCacheType::TTL == type) { + _cur_ttl_size -= file_block->range().size(); + } auto& offsets = _files[hash]; offsets.erase(file_block->offset()); if (offsets.empty()) { @@ -1544,6 +1554,7 @@ std::string BlockFileCache::clear_file_cache_directly() { int64_t disposible_queue_size = _disposable_queue.get_elements_num(cache_lock); _files.clear(); _cur_cache_size = 0; + _cur_ttl_size = 0; _time_to_key.clear(); _key_to_time.clear(); _index_queue.clear(cache_lock); diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 282148aa566dd3..f086c2c680ee16 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -394,6 +394,7 @@ class BlockFileCache { CachedFiles _files; QueryFileCacheContextMap _query_map; size_t _cur_cache_size = 0; + size_t _cur_ttl_size = 0; std::multimap _time_to_key; std::unordered_map _key_to_time; // The three queues are level queue. diff --git a/be/src/io/cache/file_block.cpp b/be/src/io/cache/file_block.cpp index 2efc26fb1a6a12..5985aa95f7abdc 100644 --- a/be/src/io/cache/file_block.cpp +++ b/be/src/io/cache/file_block.cpp @@ -30,6 +30,11 @@ namespace doris { namespace io { +std::ostream& operator<<(std::ostream& os, const FileBlock::State& value) { + os << FileBlock::state_to_string(value); + return os; +} + FileBlock::FileBlock(const FileCacheKey& key, size_t size, BlockFileCache* mgr, State download_state) : _block_range(key.offset, key.offset + size - 1), diff --git a/be/src/io/cache/file_block.h b/be/src/io/cache/file_block.h index dd4ef3757074ee..2587cd8607fc5a 100644 --- a/be/src/io/cache/file_block.h +++ b/be/src/io/cache/file_block.h @@ -154,6 +154,8 @@ class FileBlock { size_t _downloaded_size {0}; }; +extern std::ostream& operator<<(std::ostream& os, const FileBlock::State& value); + using FileBlockSPtr = std::shared_ptr; using FileBlocks = std::list; diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 6b1139d1b41c02..64778b396a29b2 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -679,6 +679,59 @@ TEST_F(BlockFileCacheTest, resize) { } } +TEST_F(BlockFileCacheTest, max_ttl_size) { + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } + fs::create_directories(cache_base_path); + TUniqueId query_id; + query_id.hi = 1; + query_id.lo = 1; + io::FileCacheSettings settings; + settings.query_queue_size = 100000000; + settings.query_queue_elements = 100000; + settings.capacity = 100000000; + settings.max_file_block_size = 100000; + settings.max_query_cache_size = 30; + io::CacheContext context; + context.cache_type = io::FileCacheType::TTL; + context.query_id = query_id; + int64_t cur_time = UnixSeconds(); + context.expiration_time = cur_time + 120; + auto key1 = io::BlockFileCache::hash("key5"); + io::BlockFileCache cache(cache_base_path, settings); + ASSERT_TRUE(cache.initialize()); + int i = 0; + for (; i < 100; i++) { + if (cache.get_lazy_open_success()) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(cache.get_lazy_open_success()); + int64_t offset = 0; + for (; offset < 100000000; offset += 100000) { + auto holder = cache.get_or_set(key1, offset, 100000, context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + if (offset < 90000000) { + assert_range(1, blocks[0], io::FileBlock::Range(offset, offset + 99999), + io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + download(blocks[0]); + assert_range(1, blocks[0], io::FileBlock::Range(offset, offset + 99999), + io::FileBlock::State::DOWNLOADED); + } else { + assert_range(1, blocks[0], io::FileBlock::Range(offset, offset + 99999), + io::FileBlock::State::SKIP_CACHE); + } + blocks.clear(); + } + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } +} + TEST_F(BlockFileCacheTest, query_limit_heap_use_after_free) { if (fs::exists(cache_base_path)) { fs::remove_all(cache_base_path); @@ -1773,10 +1826,10 @@ TEST_F(BlockFileCacheTest, ttl_reverse) { query_id.hi = 1; query_id.lo = 1; io::FileCacheSettings settings; - settings.query_queue_size = 30; + settings.query_queue_size = 36; settings.query_queue_elements = 5; - settings.capacity = 30; - settings.max_file_block_size = 5; + settings.capacity = 36; + settings.max_file_block_size = 7; settings.max_query_cache_size = 30; io::CacheContext context; context.cache_type = io::FileCacheType::TTL; @@ -1792,25 +1845,23 @@ TEST_F(BlockFileCacheTest, ttl_reverse) { }; std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - { - auto holder = cache.get_or_set(key2, 0, 30, context); /// Add range [0, 29] + ASSERT_TRUE(cache.get_lazy_open_success()); + for (size_t offset = 0; offset < 30; offset += 6) { + auto holder = cache.get_or_set(key2, offset, 6, context); auto blocks = fromHolder(holder); - for (auto& block : blocks) { - ASSERT_TRUE(block->get_or_set_downloader() == io::FileBlock::get_caller_id()); - download(block); - } - EXPECT_EQ(blocks.size(), 6); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + download(blocks[0]); } { - auto holder = cache.get_or_set(key2, 50, 5, context); /// Add range [50, 54] + auto holder = cache.get_or_set(key2, 50, 7, context); /// Add range [50, 57] auto blocks = fromHolder(holder); - assert_range(1, blocks[0], io::FileBlock::Range(50, 54), io::FileBlock::State::SKIP_CACHE); + assert_range(1, blocks[0], io::FileBlock::Range(50, 56), io::FileBlock::State::SKIP_CACHE); } { context.cache_type = io::FileCacheType::NORMAL; - auto holder = cache.get_or_set(key2, 50, 5, context); /// Add range [50, 54] + auto holder = cache.get_or_set(key2, 50, 7, context); /// Add range [50, 57] auto blocks = fromHolder(holder); - assert_range(1, blocks[0], io::FileBlock::Range(50, 54), io::FileBlock::State::SKIP_CACHE); + assert_range(1, blocks[0], io::FileBlock::Range(50, 56), io::FileBlock::State::SKIP_CACHE); } if (fs::exists(cache_base_path)) { From 0644f45df3b5a63a35c67d56dfeb92a6f3fc80d0 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Thu, 25 Apr 2024 20:41:45 +0800 Subject: [PATCH 027/163] [feature](Cloud) Load index data into index cache when writing data (#34046) --- be/src/io/fs/broker_file_writer.h | 3 +- be/src/io/fs/file_writer.h | 3 ++ be/src/io/fs/hdfs_file_writer.cpp | 3 -- be/src/io/fs/hdfs_file_writer.h | 5 ++- be/src/io/fs/local_file_writer.h | 4 ++- be/src/io/fs/s3_file_bufferpool.cpp | 23 +------------- be/src/io/fs/s3_file_bufferpool.h | 23 ++------------ be/src/io/fs/s3_file_writer.cpp | 31 +++++++------------ be/src/io/fs/s3_file_writer.h | 12 +++---- be/src/io/fs/stream_sink_file_writer.h | 3 ++ .../olap/rowset/segment_v2/segment_writer.cpp | 15 +++++++++ be/test/olap/tablet_cooldown_test.cpp | 2 ++ 12 files changed, 53 insertions(+), 74 deletions(-) diff --git a/be/src/io/fs/broker_file_writer.h b/be/src/io/fs/broker_file_writer.h index 58bba9febd3db2..d6fce52a05c662 100644 --- a/be/src/io/fs/broker_file_writer.h +++ b/be/src/io/fs/broker_file_writer.h @@ -33,7 +33,7 @@ namespace doris { class ExecEnv; namespace io { - +struct FileCacheAllocatorBuilder; class BrokerFileWriter final : public FileWriter { public: // Create and open file writer @@ -50,6 +50,7 @@ class BrokerFileWriter final : public FileWriter { const Path& path() const override { return _path; } size_t bytes_appended() const override { return _cur_offset; } bool closed() const override { return _closed; } + FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } private: Status _write(const uint8_t* buf, size_t buf_len, size_t* written_bytes); diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h index 4feab99c09ff24..5d22dca60551c8 100644 --- a/be/src/io/fs/file_writer.h +++ b/be/src/io/fs/file_writer.h @@ -28,6 +28,7 @@ namespace doris::io { class FileSystem; +struct FileCacheAllocatorBuilder; // Only affects remote file writers struct FileWriterOptions { @@ -62,6 +63,8 @@ class FileWriter { virtual size_t bytes_appended() const = 0; virtual bool closed() const = 0; + + virtual FileCacheAllocatorBuilder* cache_builder() const = 0; }; } // namespace doris::io diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp index c596c0e290fe8c..9ea66ca4da13eb 100644 --- a/be/src/io/fs/hdfs_file_writer.cpp +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -159,9 +159,6 @@ void HdfsFileWriter::_write_into_local_file_cache() { size_t block_size = block->range().size(); size_t append_size = std::min(data_remain_size, block_size); if (block->state() == FileBlock::State::EMPTY) { - if (_index_offset != 0 && block->range().right >= _index_offset) { - static_cast(block->change_cache_type_self(FileCacheType::INDEX)); - } block->get_or_set_downloader(); if (block->is_downloader()) { Slice s(_batch_buffer.data() + pos, append_size); diff --git a/be/src/io/fs/hdfs_file_writer.h b/be/src/io/fs/hdfs_file_writer.h index 2ce865ecfedb20..234835e083f4c5 100644 --- a/be/src/io/fs/hdfs_file_writer.h +++ b/be/src/io/fs/hdfs_file_writer.h @@ -51,6 +51,10 @@ class HdfsFileWriter final : public FileWriter { size_t bytes_appended() const override { return _bytes_appended; } bool closed() const override { return _closed; } + FileCacheAllocatorBuilder* cache_builder() const override { + return _cache_builder == nullptr ? nullptr : _cache_builder.get(); + } + private: // Flush buffered data into HDFS client and write local file cache if enabled // **Notice**: this would clear the underlying buffer @@ -83,7 +87,6 @@ class HdfsFileWriter final : public FileWriter { std::string _batch_buffer; }; BatchBuffer _batch_buffer; - size_t _index_offset = 0; }; } // namespace io diff --git a/be/src/io/fs/local_file_writer.h b/be/src/io/fs/local_file_writer.h index 4cd6712b04e1ef..81ebb0ebd1fcb7 100644 --- a/be/src/io/fs/local_file_writer.h +++ b/be/src/io/fs/local_file_writer.h @@ -25,7 +25,7 @@ #include "util/slice.h" namespace doris::io { - +struct FileCacheAllocatorBuilder; class LocalFileWriter final : public FileWriter { public: LocalFileWriter(Path path, int fd, bool sync_data = true); @@ -38,6 +38,8 @@ class LocalFileWriter final : public FileWriter { size_t bytes_appended() const override; bool closed() const override { return _closed; } + FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } + private: void _abort(); Status _close(bool sync); diff --git a/be/src/io/fs/s3_file_bufferpool.cpp b/be/src/io/fs/s3_file_bufferpool.cpp index 9df60dc1fd3042..82493fa9514f10 100644 --- a/be/src/io/fs/s3_file_bufferpool.cpp +++ b/be/src/io/fs/s3_file_bufferpool.cpp @@ -90,24 +90,6 @@ FileBuffer::~FileBuffer() { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->s3_file_buffer_tracker()); _inner_data.reset(); } -/** - * 0. check if file cache holder allocated - * 1. update the cache's type to index cache - */ -void UploadFileBuffer::set_index_offset(size_t offset) { - _index_offset = offset; - if (_holder) { - bool change_to_index_cache = false; - for (auto iter = _holder->file_blocks.begin(); iter != _holder->file_blocks.end(); ++iter) { - if (iter == _cur_file_block) { - change_to_index_cache = true; - } - if (change_to_index_cache) { - static_cast((*iter)->change_cache_type_self(FileCacheType::INDEX)); - } - } - } -} /** * 0. when there is memory preserved, directly write data to buf @@ -222,9 +204,6 @@ void UploadFileBuffer::upload_to_local_file_cache(bool is_cancelled) { size_t block_size = block->range().size(); size_t append_size = std::min(data_remain_size, block_size); if (block->state() == FileBlock::State::EMPTY) { - if (_index_offset != 0 && block->range().right >= _index_offset) { - static_cast(block->change_cache_type_self(FileCacheType::INDEX)); - } block->get_or_set_downloader(); // Another thread may have started downloading due to a query // Just skip putting to cache from UploadFileBuffer @@ -279,7 +258,7 @@ Status FileBufferBuilder::build(std::shared_ptr* buf) { if (_type == BufferType::UPLOAD) { RETURN_IF_CATCH_EXCEPTION(*buf = std::make_shared( std::move(_upload_cb), std::move(state), _offset, - std::move(_alloc_holder_cb), _index_offset)); + std::move(_alloc_holder_cb))); return Status::OK(); } if (_type == BufferType::DOWNLOAD) { diff --git a/be/src/io/fs/s3_file_bufferpool.h b/be/src/io/fs/s3_file_bufferpool.h index 189f7696967759..15d0976df6b880 100644 --- a/be/src/io/fs/s3_file_bufferpool.h +++ b/be/src/io/fs/s3_file_bufferpool.h @@ -151,18 +151,10 @@ struct DownloadFileBuffer final : public FileBuffer { struct UploadFileBuffer final : public FileBuffer { UploadFileBuffer(std::function upload_cb, OperationState state, - size_t offset, std::function alloc_holder, - size_t index_offset) + size_t offset, std::function alloc_holder) : FileBuffer(BufferType::UPLOAD, alloc_holder, offset, state), - _upload_to_remote(std::move(upload_cb)), - _index_offset(index_offset) {} + _upload_to_remote(std::move(upload_cb)) {} ~UploadFileBuffer() override = default; - /** - * set the index offset - * - * @param offset the index offset - */ - void set_index_offset(size_t offset); Status append_data(const Slice& s) override; /** * read the content from local file cache @@ -206,7 +198,6 @@ struct UploadFileBuffer final : public FileBuffer { FileBlocksHolderPtr _holder; decltype(_holder->file_blocks.begin()) _cur_file_block; size_t _append_offset {0}; - size_t _index_offset {0}; uint32_t _crc_value = 0; }; @@ -272,15 +263,6 @@ struct FileBufferBuilder { return *this; } /** - * set the index offset of the file buffer - * - * @param cb - */ - FileBufferBuilder& set_index_offset(size_t index_offset) { - _index_offset = index_offset; - return *this; - } - /** * set the callback which write the content into local file cache * * @param cb @@ -309,7 +291,6 @@ struct FileBufferBuilder { std::function _download; std::function _write_to_use_buffer; size_t _offset; - size_t _index_offset; }; } // namespace io } // namespace doris diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 84487f496ac1e9..69202bd22fe98f 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -87,16 +87,16 @@ S3FileWriter::S3FileWriter(std::shared_ptr client, std::strin : _path(fmt::format("s3://{}/{}", bucket, key)), _bucket(std::move(bucket)), _key(std::move(key)), - _client(std::move(client)), - _expiration_time(opts ? opts->file_cache_expiration : 0), - _is_cold_data(opts ? opts->is_cold_data : true), - _write_file_cache(opts ? opts->write_file_cache : false) { + _client(std::move(client)) { s3_file_writer_total << 1; s3_file_being_written << 1; Aws::Http::SetCompliantRfc3986Encoding(true); - if (config::enable_file_cache && _write_file_cache) { - _cache_hash = BlockFileCache::hash(_path.filename().native()); - _cache = FileCacheFactory::instance()->get_by_path(_cache_hash); + if (config::enable_file_cache && opts != nullptr && opts->write_file_cache) { + _cache_builder = std::make_unique(FileCacheAllocatorBuilder { + opts ? opts->is_cold_data : false, opts ? opts->file_cache_expiration : 0, + BlockFileCache::hash(_path.filename().native()), + FileCacheFactory::instance()->get_by_path( + BlockFileCache::hash(_path.filename().native()))}); } } @@ -264,7 +264,6 @@ Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) { _upload_one_part(part_num, buf); }) .set_file_offset(_bytes_appended) - .set_index_offset(_index_offset) .set_sync_after_complete_task([this, part_num = _cur_part_num](Status s) { bool ret = false; if (!s.ok()) [[unlikely]] { @@ -282,22 +281,16 @@ Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) { return ret; }) .set_is_cancelled([this]() { return _failed.load(); }); - if (_write_file_cache) { + if (_cache_builder != nullptr) { // We would load the data into file cache asynchronously which indicates // that this instance of S3FileWriter might have been destructed when we // try to do writing into file cache, so we make the lambda capture the variable // we need by value to extend their lifetime builder.set_allocate_file_blocks_holder( - [cache = _cache, k = _cache_hash, offset = _bytes_appended, - t = _expiration_time, cold = _is_cold_data]() -> FileBlocksHolderPtr { - CacheContext ctx; - ctx.cache_type = - t == 0 ? FileCacheType::NORMAL : FileCacheType::TTL; - ctx.expiration_time = t; - ctx.is_cold_data = cold; - auto holder = cache->get_or_set(k, offset, - config::s3_write_buffer_size, ctx); - return std::make_unique(std::move(holder)); + [builder = *_cache_builder, + offset = _bytes_appended]() -> FileBlocksHolderPtr { + return builder.allocate_cache_holder(offset, + config::s3_write_buffer_size); }); } RETURN_IF_ERROR(builder.build(&_pending_buf)); diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index a2c2ec0422a4f1..8f27b202369444 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -58,6 +58,10 @@ class S3FileWriter final : public FileWriter { size_t bytes_appended() const override { return _bytes_appended; } bool closed() const override { return _closed; } + FileCacheAllocatorBuilder* cache_builder() const override { + return _cache_builder == nullptr ? nullptr : _cache_builder.get(); + } + private: Status _abort(); [[nodiscard]] std::string _dump_completed_part() const; @@ -73,15 +77,12 @@ class S3FileWriter final : public FileWriter { std::shared_ptr _client; std::string _upload_id; - size_t _index_offset {0}; // Current Part Num for CompletedPart int _cur_part_num = 1; std::mutex _completed_lock; std::vector> _completed_parts; - UInt128Wrapper _cache_hash; - BlockFileCache* _cache; // **Attention** call add_count() before submitting buf to async thread pool bthread::CountdownEvent _countdown_event {0}; @@ -92,9 +93,8 @@ class S3FileWriter final : public FileWriter { size_t _bytes_appended = 0; std::shared_ptr _pending_buf; - uint64_t _expiration_time; - bool _is_cold_data; - bool _write_file_cache; + std::unique_ptr + _cache_builder; // nullptr if disable write file cache }; } // namespace io diff --git a/be/src/io/fs/stream_sink_file_writer.h b/be/src/io/fs/stream_sink_file_writer.h index 2bd91075ad11a0..4a0eb955c26171 100644 --- a/be/src/io/fs/stream_sink_file_writer.h +++ b/be/src/io/fs/stream_sink_file_writer.h @@ -33,6 +33,7 @@ struct RowsetId; struct SegmentStatistics; namespace io { +struct FileCacheAllocatorBuilder; class StreamSinkFileWriter final : public FileWriter { public: StreamSinkFileWriter(std::vector> streams) @@ -57,6 +58,8 @@ class StreamSinkFileWriter final : public FileWriter { return dummy; } + FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } + private: std::vector> _streams; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index f6be1917e57840..7a83496b7fb481 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -34,6 +34,8 @@ #include "common/status.h" #include "gutil/port.h" #include "inverted_index_fs_directory.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" @@ -1113,6 +1115,8 @@ Status SegmentWriter::finalize(uint64_t* segment_file_size, uint64_t* index_size } // write data RETURN_IF_ERROR(finalize_columns_data()); + // Get the index start before finalize_footer since this function would write new data. + uint64_t index_start = _file_writer->bytes_appended(); // write index RETURN_IF_ERROR(finalize_columns_index(index_size)); // write footer @@ -1122,6 +1126,17 @@ Status SegmentWriter::finalize(uint64_t* segment_file_size, uint64_t* index_size LOG(INFO) << "segment flush consumes a lot time_ns " << timer.elapsed_time() << ", segmemt_size " << *segment_file_size; } + // When the cache type is not ttl(expiration time == 0), the data should be split into normal cache queue + // and index cache queue + if (auto* cache_builder = _file_writer->cache_builder(); cache_builder != nullptr && + cache_builder->_expiration_time == 0 && + config::is_cloud_mode()) { + auto size = *index_size + *segment_file_size; + auto holder = cache_builder->allocate_cache_holder(index_start, size); + for (auto& segment : holder->file_blocks) { + static_cast(segment->change_cache_type_self(io::FileCacheType::INDEX)); + } + } return Status::OK(); } diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 45186246006317..49de182610459d 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -113,6 +113,8 @@ class FileWriterMock final : public io::FileWriter { const Path& path() const override { return _local_file_writer->path(); } + io::FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } + private: std::unique_ptr _local_file_writer; }; From 6736f02f279a32d5776f3b04b9516cd5b58e76d9 Mon Sep 17 00:00:00 2001 From: zhengyu Date: Thu, 25 Apr 2024 21:14:06 +0800 Subject: [PATCH 028/163] [enhancement](cloud) add bvar to monitor s3 throughput&QPS (#34087) Signed-off-by: freemandealer --- be/src/io/fs/hdfs_file_reader.cpp | 13 +++++++++++++ be/src/io/fs/s3_file_reader.cpp | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index 358663b65d0dad..a26448c90e2ce3 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -24,6 +24,8 @@ #include #include +#include "bvar/latency_recorder.h" +#include "bvar/reducer.h" #include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "common/sync_point.h" @@ -33,6 +35,13 @@ #include "util/doris_metrics.h" namespace doris::io { + +bvar::Adder hdfs_bytes_read_total("hdfs_file_reader", "bytes_read"); +bvar::LatencyRecorder hdfs_bytes_per_read("hdfs_file_reader", "bytes_per_read"); // also QPS +bvar::PerSecond> hdfs_read_througthput("hdfs_file_reader", + "hdfs_read_throughput", + &hdfs_bytes_read_total); + namespace { Result get_file(const hdfsFS& fs, const Path& file, int64_t mtime, @@ -148,6 +157,8 @@ Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_r has_read += loop_read; } *bytes_read = has_read; + hdfs_bytes_read_total << *bytes_read; + hdfs_bytes_per_read << *bytes_read; return Status::OK(); } @@ -206,6 +217,8 @@ Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_r has_read += loop_read; } *bytes_read = has_read; + hdfs_bytes_read_total << *bytes_read; + hdfs_bytes_per_read << *bytes_read; return Status::OK(); } #endif diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 2bd40fbbf43861..68acbf47eb18a1 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,9 @@ bvar::Adder s3_file_reader_read_counter("s3_file_reader", "read_at"); bvar::Adder s3_file_reader_total("s3_file_reader", "total_num"); bvar::Adder s3_bytes_read_total("s3_file_reader", "bytes_read"); bvar::Adder s3_file_being_read("s3_file_reader", "file_being_read"); +bvar::LatencyRecorder s3_bytes_per_read("s3_file_reader", "bytes_per_read"); // also QPS +bvar::PerSecond> s3_read_througthput("s3_file_reader", "s3_read_throughput", + &s3_bytes_read_total); Result S3FileReader::create(std::shared_ptr client, std::string bucket, std::string key, @@ -125,6 +129,7 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea _path.native(), *bytes_read, bytes_req); } s3_bytes_read_total << *bytes_read; + s3_bytes_per_read << *bytes_read; s3_file_reader_read_counter << 1; DorisMetrics::instance()->s3_bytes_read_total->increment(*bytes_read); return Status::OK(); From eed69d18d8395fd8f27bbb9278c0009bb0c2ea79 Mon Sep 17 00:00:00 2001 From: walter Date: Thu, 25 Apr 2024 21:15:35 +0800 Subject: [PATCH 029/163] [feature](merge-cloud) Add txn kv stats API (#32823) --- cloud/src/meta-service/mem_txn_kv.cpp | 24 +++++++++++- cloud/src/meta-service/mem_txn_kv.h | 16 ++++++++ cloud/src/meta-service/meta_service_txn.cpp | 30 ++------------- cloud/src/meta-service/txn_kv.cpp | 25 +++++++++++- cloud/src/meta-service/txn_kv.h | 42 ++++++++++++++++++++- 5 files changed, 107 insertions(+), 30 deletions(-) diff --git a/cloud/src/meta-service/mem_txn_kv.cpp b/cloud/src/meta-service/mem_txn_kv.cpp index e10f5dd3d3ed13..9f817fa4398b3e 100644 --- a/cloud/src/meta-service/mem_txn_kv.cpp +++ b/cloud/src/meta-service/mem_txn_kv.cpp @@ -38,7 +38,7 @@ int MemTxnKv::init() { } TxnErrorCode MemTxnKv::create_txn(std::unique_ptr* txn) { - auto t = new memkv::Transaction(this->shared_from_this()); + auto* t = new memkv::Transaction(this->shared_from_this()); txn->reset(t); return TxnErrorCode::TXN_OK; } @@ -241,6 +241,9 @@ void Transaction::put(std::string_view key, std::string_view val) { std::string v(val.data(), val.size()); writes_.insert_or_assign(k, v); op_list_.emplace_back(ModifyOpType::PUT, k, v); + ++num_put_keys_; + put_bytes_ += key.size() + val.size(); + approximate_bytes_ += key.size() + val.size(); } TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snapshot) { @@ -345,6 +348,10 @@ void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_vi std::string v(val.data(), val.size()); unreadable_keys_.insert(k); op_list_.emplace_back(ModifyOpType::ATOMIC_SET_VER_KEY, k, v); + + ++num_put_keys_; + put_bytes_ += key_prefix.size() + val.size(); + approximate_bytes_ += key_prefix.size() + val.size(); } void Transaction::atomic_set_ver_value(std::string_view key, std::string_view value) { @@ -353,6 +360,10 @@ void Transaction::atomic_set_ver_value(std::string_view key, std::string_view va std::string v(value.data(), value.size()); unreadable_keys_.insert(k); op_list_.emplace_back(ModifyOpType::ATOMIC_SET_VER_VAL, k, v); + + ++num_put_keys_; + put_bytes_ += key.size() + value.size(); + approximate_bytes_ += key.size() + value.size(); } void Transaction::atomic_add(std::string_view key, int64_t to_add) { @@ -361,6 +372,10 @@ void Transaction::atomic_add(std::string_view key, int64_t to_add) { memcpy(v.data(), &to_add, sizeof(to_add)); std::lock_guard l(lock_); op_list_.emplace_back(ModifyOpType::ATOMIC_ADD, std::move(k), std::move(v)); + + ++num_put_keys_; + put_bytes_ += key.size() + 8; + approximate_bytes_ += key.size() + 8; } void Transaction::remove(std::string_view key) { @@ -371,6 +386,10 @@ void Transaction::remove(std::string_view key) { end_key.push_back(0x0); remove_ranges_.emplace_back(k, end_key); op_list_.emplace_back(ModifyOpType::REMOVE, k, ""); + + ++num_del_keys_; + delete_bytes_ += key.size(); + approximate_bytes_ += key.size(); } void Transaction::remove(std::string_view begin, std::string_view end) { @@ -387,6 +406,9 @@ void Transaction::remove(std::string_view begin, std::string_view end) { remove_ranges_.emplace_back(begin_k, end_k); op_list_.emplace_back(ModifyOpType::REMOVE_RANGE, begin_k, end_k); } + ++num_del_keys_; + delete_bytes_ += begin.size() + end.size(); + approximate_bytes_ += begin.size() + end.size(); } TxnErrorCode Transaction::commit() { diff --git a/cloud/src/meta-service/mem_txn_kv.h b/cloud/src/meta-service/mem_txn_kv.h index e385c414bc5e69..359a8dcafc84a1 100644 --- a/cloud/src/meta-service/mem_txn_kv.h +++ b/cloud/src/meta-service/mem_txn_kv.h @@ -182,6 +182,16 @@ class Transaction : public cloud::Transaction { const std::vector& keys, const BatchGetOptions& opts = BatchGetOptions()) override; + size_t approximate_bytes() const override { return approximate_bytes_; } + + size_t num_del_keys() const override { return num_del_keys_; } + + size_t num_put_keys() const override { return num_put_keys_; } + + size_t delete_bytes() const override { return delete_bytes_; } + + size_t put_bytes() const override { return put_bytes_; } + private: TxnErrorCode inner_get(const std::string& key, std::string* val, bool snapshot); @@ -201,6 +211,12 @@ class Transaction : public cloud::Transaction { int64_t committed_version_ = -1; int64_t read_version_ = -1; + + size_t approximate_bytes_ {0}; + size_t num_del_keys_ {0}; + size_t num_put_keys_ {0}; + size_t delete_bytes_ {0}; + size_t put_bytes_ {0}; }; class RangeGetIterator : public cloud::RangeGetIterator { diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 0afdc31d10a095..b80f9f8619c9d8 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -789,10 +789,6 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, return; } - int64_t put_size = 0; - int64_t del_size = 0; - int num_put_keys = 0, num_del_keys = 0; - // Get txn info with db_id and txn_id std::string info_val; // Will be reused when saving updated txn const std::string info_key = txn_info_key({instance_id, db_id, txn_id}); @@ -1039,17 +1035,14 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, lock_values.clear(); // Save rowset meta - num_put_keys += rowsets.size(); for (auto& i : rowsets) { size_t rowset_size = i.first.size() + i.second.size(); txn->put(i.first, i.second); - put_size += rowset_size; LOG(INFO) << "xxx put rowset_key=" << hex(i.first) << " txn_id=" << txn_id << " rowset_size=" << rowset_size; } // Save versions - num_put_keys += new_versions.size(); for (auto& i : new_versions) { std::string ver_val; VersionPB version_pb; @@ -1062,7 +1055,6 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, } txn->put(i.first, ver_val); - put_size += i.first.size() + ver_val.size(); LOG(INFO) << "xxx put partition_version_key=" << hex(i.first) << " version:" << i.second << " txn_id=" << txn_id; @@ -1090,11 +1082,9 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, } // Save table versions - num_put_keys += table_id_tablet_ids.size(); for (auto& i : table_id_tablet_ids) { std::string ver_key = table_version_key({instance_id, db_id, i.first}); txn->atomic_add(ver_key, 1); - put_size += ver_key.size(); LOG(INFO) << "xxx atomic add table_version_key=" << hex(ver_key) << " txn_id=" << txn_id; } @@ -1126,8 +1116,6 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, return; } txn->put(info_key, info_val); - put_size += info_key.size() + info_val.size(); - ++num_put_keys; LOG(INFO) << "xxx put info_key=" << hex(info_key) << " txn_id=" << txn_id; // Update stats of affected tablet @@ -1145,14 +1133,10 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, auto& num_segs_key = kv_pool.emplace_back(); stats_tablet_num_segs_key(info, &num_segs_key); txn->atomic_add(num_segs_key, stats.num_segs); - put_size += data_size_key.size() + num_rows_key.size() + num_segs_key.size() + 24; - num_put_keys += 3; } auto& num_rowsets_key = kv_pool.emplace_back(); stats_tablet_num_rowsets_key(info, &num_rowsets_key); txn->atomic_add(num_rowsets_key, stats.num_rowsets); - put_size += num_rowsets_key.size() + 8; - ++num_put_keys; }; } else { update_tablet_stats = [&](const StatsTabletKeyInfo& info, const TabletStats& stats) { @@ -1179,8 +1163,6 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs); stats_pb.SerializeToString(&val); txn->put(key, val); - put_size += key.size() + val.size(); - ++num_put_keys; }; } for (auto& [tablet_id, stats] : tablet_stats) { @@ -1192,18 +1174,14 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; } // Remove tmp rowset meta - num_del_keys += tmp_rowsets_meta.size(); for (auto& [k, _] : tmp_rowsets_meta) { txn->remove(k); - del_size += k.size(); LOG(INFO) << "xxx remove tmp_rowset_key=" << hex(k) << " txn_id=" << txn_id; } const std::string running_key = txn_running_key({instance_id, db_id, txn_id}); LOG(INFO) << "xxx remove running_key=" << hex(running_key) << " txn_id=" << txn_id; txn->remove(running_key); - del_size += running_key.size(); - ++num_del_keys; std::string recycle_val; std::string recycle_key = recycle_txn_key({instance_id, db_id, txn_id}); @@ -1218,8 +1196,6 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, return; } txn->put(recycle_key, recycle_val); - put_size += recycle_key.size() + recycle_val.size(); - ++num_put_keys; if (txn_info.load_job_source_type() == LoadJobSourceTypePB::LOAD_JOB_SRC_TYPE_ROUTINE_LOAD_TASK) { @@ -1227,9 +1203,9 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, } LOG(INFO) << "xxx commit_txn put recycle_key key=" << hex(recycle_key) << " txn_id=" << txn_id; - LOG(INFO) << "commit_txn put_size=" << put_size << " del_size=" << del_size - << " num_put_keys=" << num_put_keys << " num_del_keys=" << num_del_keys - << " txn_id=" << txn_id; + LOG(INFO) << "commit_txn put_size=" << txn->put_bytes() << " del_size=" << txn->delete_bytes() + << " num_put_keys=" << txn->num_put_keys() << " num_del_keys=" << txn->num_del_keys() + << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id; // Finally we are done... err = txn->commit(); diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-service/txn_kv.cpp index 5afe0edb40965f..013d21f84fcd58 100644 --- a/cloud/src/meta-service/txn_kv.cpp +++ b/cloud/src/meta-service/txn_kv.cpp @@ -254,6 +254,10 @@ void Transaction::put(std::string_view key, std::string_view val) { StopWatch sw; fdb_transaction_set(txn_, (uint8_t*)key.data(), key.size(), (uint8_t*)val.data(), val.size()); g_bvar_txn_kv_put << sw.elapsed_us(); + + ++num_put_keys_; + put_bytes_ += key.size() + val.size(); + approximate_bytes_ = key.size() * 3 + val.size(); // See fdbclient/ReadYourWrites.actor.cpp } // return 0 for success otherwise error @@ -290,6 +294,7 @@ static TxnErrorCode await_future(FDBFuture* fut) { TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snapshot) { StopWatch sw; + approximate_bytes_ += key.size() * 2; // See fdbclient/ReadYourWrites.actor.cpp for details auto* fut = fdb_transaction_get(txn_, (uint8_t*)key.data(), key.size(), snapshot); auto release_fut = [fut, &sw](int*) { @@ -328,6 +333,7 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, std::unique_ptr* iter, bool snapshot, int limit) { StopWatch sw; + approximate_bytes_ += begin.size() + end.size(); std::unique_ptr> defer( (int*)0x01, [&sw](int*) { g_bvar_txn_kv_range_get << sw.elapsed_us(); }); @@ -368,6 +374,9 @@ void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_vi FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY); g_bvar_txn_kv_atomic_set_ver_key << sw.elapsed_us(); + ++num_put_keys_; + put_bytes_ += key_prefix.size() + val.size(); + approximate_bytes_ += key_prefix.size() * 3 + val.size(); } void Transaction::atomic_set_ver_value(std::string_view key, std::string_view value) { @@ -384,6 +393,9 @@ void Transaction::atomic_set_ver_value(std::string_view key, std::string_view va FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); g_bvar_txn_kv_atomic_set_ver_value << sw.elapsed_us(); + ++num_put_keys_; + put_bytes_ += key.size() + value.size(); + approximate_bytes_ += key.size() * 3 + value.size(); } void Transaction::atomic_add(std::string_view key, int64_t to_add) { @@ -394,12 +406,18 @@ void Transaction::atomic_add(std::string_view key, int64_t to_add) { sizeof(to_add), FDBMutationType::FDB_MUTATION_TYPE_ADD); g_bvar_txn_kv_atomic_add << sw.elapsed_us(); + ++num_put_keys_; + put_bytes_ += key.size() + 8; + approximate_bytes_ += key.size() * 3 + 8; } void Transaction::remove(std::string_view key) { StopWatch sw; fdb_transaction_clear(txn_, (uint8_t*)key.data(), key.size()); g_bvar_txn_kv_remove << sw.elapsed_us(); + ++num_del_keys_; + delete_bytes_ += key.size(); + approximate_bytes_ += key.size() * 4; // See fdbclient/ReadYourWrites.actor.cpp for details. } void Transaction::remove(std::string_view begin, std::string_view end) { @@ -407,6 +425,10 @@ void Transaction::remove(std::string_view begin, std::string_view end) { fdb_transaction_clear_range(txn_, (uint8_t*)begin.data(), begin.size(), (uint8_t*)end.data(), end.size()); g_bvar_txn_kv_range_remove << sw.elapsed_us(); + num_del_keys_ += 2; + delete_bytes_ += begin.size() + end.size(); + approximate_bytes_ += + (begin.size() + end.size()) * 2; // See fdbclient/ReadYourWrites.actor.cpp for details. } TxnErrorCode Transaction::commit() { @@ -435,7 +457,7 @@ TxnErrorCode Transaction::commit() { TxnErrorCode Transaction::get_read_version(int64_t* version) { StopWatch sw; - auto fut = fdb_transaction_get_read_version(txn_); + auto* fut = fdb_transaction_get_read_version(txn_); std::unique_ptr> defer((int*)0x01, [fut, &sw](...) { fdb_future_destroy(fut); g_bvar_txn_kv_get_read_version << sw.elapsed_us(); @@ -512,6 +534,7 @@ TxnErrorCode Transaction::batch_get(std::vector>* res const auto& k = keys[j]; futures.emplace_back( fdb_transaction_get(txn_, (uint8_t*)k.data(), k.size(), opts.snapshot)); + approximate_bytes_ += k.size() * 2; } size_t num_futures = futures.size(); diff --git a/cloud/src/meta-service/txn_kv.h b/cloud/src/meta-service/txn_kv.h index 12ee38823f943a..0fc6d160bf94fc 100644 --- a/cloud/src/meta-service/txn_kv.h +++ b/cloud/src/meta-service/txn_kv.h @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -171,6 +170,31 @@ class Transaction { virtual TxnErrorCode batch_get(std::vector>* res, const std::vector& keys, const BatchGetOptions& opts = BatchGetOptions()) = 0; + + /** + * @brief return the approximate bytes consumed by the underlying transaction buffer. + **/ + virtual size_t approximate_bytes() const = 0; + + /** + * @brief return the num delete keys submitted to this txn. + **/ + virtual size_t num_del_keys() const = 0; + + /** + * @brief return the num put keys submitted to this txn. + **/ + virtual size_t num_put_keys() const = 0; + + /** + * @brief return the bytes of the delete keys consumed. + **/ + virtual size_t delete_bytes() const = 0; + + /** + * @brief return the bytes of the put key and values consumed. + **/ + virtual size_t put_bytes() const = 0; }; class RangeGetIterator { @@ -460,11 +484,27 @@ class Transaction : public cloud::Transaction { const std::vector& keys, const BatchGetOptions& opts = BatchGetOptions()) override; + size_t approximate_bytes() const override { return approximate_bytes_; } + + size_t num_del_keys() const override { return num_del_keys_; } + + size_t num_put_keys() const override { return num_put_keys_; } + + size_t delete_bytes() const override { return delete_bytes_; } + + size_t put_bytes() const override { return put_bytes_; } + private: std::shared_ptr db_ {nullptr}; bool commited_ = false; bool aborted_ = false; FDBTransaction* txn_ = nullptr; + + size_t num_del_keys_ {0}; + size_t num_put_keys_ {0}; + size_t delete_bytes_ {0}; + size_t put_bytes_ {0}; + size_t approximate_bytes_ {0}; }; } // namespace fdb From 152c586b13de34e8f842780e5239e17de25e74ec Mon Sep 17 00:00:00 2001 From: zxealous Date: Thu, 25 Apr 2024 21:32:16 +0800 Subject: [PATCH 030/163] [improve](disk) Not add disk path to broken list if check status is not IO_ERROR (#34111) --- be/src/olap/data_dir.cpp | 2 +- be/src/service/doris_main.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index d1b5185b7894ce..da7302ef605445 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -231,7 +231,7 @@ void DataDir::health_check() { // check disk if (_is_used) { Status res = _read_and_write_test_file(); - if (!res) { + if (!res && res.is()) { LOG(WARNING) << "store read/write test file occur IO Error. path=" << _path << ", err: " << res; _engine.add_broken_path(_path); diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 731e09c6be9fc2..98712a8168be5a 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -439,6 +439,7 @@ int main(int argc, char** argv) { it = paths.erase(it); } else { LOG(ERROR) << "read write test file failed, path=" << it->path; + // if only one disk and the disk is full, also need exit because rocksdb will open failed exit(-1); } } else { From 98e9937c9e829228ff1ca6581cdb7b2d260d7260 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Thu, 25 Apr 2024 22:11:14 +0800 Subject: [PATCH 031/163] [improvement](mtmv) Optimize the performance of nested materialized view rewriting (#34127) Optimize the performance of nested materialized view rewriting gracefully, future performance optimzie base on this. --- .../org/apache/doris/nereids/memo/Memo.java | 17 +++- .../doris/nereids/memo/StructInfoMap.java | 86 ++++++++++--------- .../mv/AbstractMaterializedViewRule.java | 1 + .../exploration/mv/MaterializedViewUtils.java | 13 ++- .../doris/nereids/memo/StructInfoMapTest.java | 20 ++--- 5 files changed, 77 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index db12c02e79a185..8793cb5be51f75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.memo; +import org.apache.doris.catalog.MTMV; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.cost.Cost; @@ -33,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -55,6 +57,7 @@ import java.util.Optional; import java.util.PriorityQueue; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -69,6 +72,8 @@ public class Memo { EventChannel.getDefaultChannel().addConsumers(new LogConsumer(GroupMergeEvent.class, EventChannel.LOG))); private static long stateId = 0; private final ConnectContext connectContext; + private final Set needRefreshTableIdSet = new HashSet<>(); + private final AtomicLong refreshVersion = new AtomicLong(1); private final IdGenerator groupIdGenerator = GroupId.createGenerator(); private final Map groups = Maps.newLinkedHashMap(); // we could not use Set, because Set does not have get method. @@ -118,6 +123,10 @@ public int getGroupExpressionsSize() { return groupExpressions.size(); } + public long getRefreshVersion() { + return refreshVersion.get(); + } + private Plan skipProject(Plan plan, Group targetGroup) { // Some top project can't be eliminated if (plan instanceof LogicalProject && ((LogicalProject) plan).canEliminate()) { @@ -406,14 +415,15 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable plan.getLogicalProperties(), targetGroup.getLogicalProperties()); throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties"); } + // TODO Support sync materialized view in the future + if (plan instanceof CatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV) { + refreshVersion.incrementAndGet(); + } Optional groupExpr = plan.getGroupExpression(); if (groupExpr.isPresent()) { Preconditions.checkState(groupExpressions.containsKey(groupExpr.get())); return CopyInResult.of(false, groupExpr.get()); } - if (targetGroup != null) { - targetGroup.getstructInfoMap().setRefreshed(false); - } List childrenGroups = Lists.newArrayList(); for (int i = 0; i < plan.children().size(); i++) { // skip useless project. @@ -562,7 +572,6 @@ public void mergeGroup(Group source, Group destination, HashMap pla if (source == root) { root = destination; } - destination.getstructInfoMap().setRefreshed(false); groups.remove(source.getGroupId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index 4119c6f2f89967..efa2bef1792417 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -42,31 +42,34 @@ public class StructInfoMap { private final Map>> groupExpressionMap = new HashMap<>(); private final Map infoMap = new HashMap<>(); - private boolean refreshed; + private long refreshVersion = 0; /** * get struct info according to table map * - * @param mvTableMap the original table map + * @param tableMap the original table map * @param foldTableMap the fold table map * @param group the group that the mv matched * @return struct info or null if not found */ - public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet foldTableMap, Group group, Plan originPlan) { - if (!infoMap.containsKey(mvTableMap)) { - if ((groupExpressionMap.containsKey(foldTableMap) || groupExpressionMap.isEmpty()) - && !groupExpressionMap.containsKey(mvTableMap)) { - refresh(group); - } - if (groupExpressionMap.containsKey(mvTableMap)) { - Pair> groupExpressionBitSetPair = getGroupExpressionWithChildren( - mvTableMap); - StructInfo structInfo = constructStructInfo(groupExpressionBitSetPair.first, - groupExpressionBitSetPair.second, mvTableMap, originPlan); - infoMap.put(mvTableMap, structInfo); - } + public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, BitSet foldTableMap, + Group group, Plan originPlan) { + StructInfo structInfo = infoMap.get(tableMap); + if (structInfo != null) { + return structInfo; + } + if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) { + refresh(group, memo.getRefreshVersion(), foldTableMap); + group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion()); } - return infoMap.get(mvTableMap); + if (groupExpressionMap.containsKey(tableMap)) { + Pair> groupExpressionBitSetPair = getGroupExpressionWithChildren( + tableMap); + structInfo = constructStructInfo(groupExpressionBitSetPair.first, + groupExpressionBitSetPair.second, tableMap, originPlan); + infoMap.put(tableMap, structInfo); + } + return structInfo; } public Set getTableMaps() { @@ -81,12 +84,12 @@ public Pair> getGroupExpressionWithChildren(BitSet return groupExpressionMap.get(tableMap); } - public boolean isRefreshed() { - return refreshed; + public long getRefreshVersion() { + return refreshVersion; } - public void setRefreshed(boolean refreshed) { - this.refreshed = refreshed; + public void setRefreshVersion(long refreshVersion) { + this.refreshVersion = refreshVersion; } private StructInfo constructStructInfo(GroupExpression groupExpression, List children, @@ -114,27 +117,24 @@ private Plan constructPlan(GroupExpression groupExpression, List childre * * @param group the root group * - * @return whether groupExpressionMap is updated */ - public boolean refresh(Group group) { - Set refreshedGroup = new HashSet<>(); - int originSize = groupExpressionMap.size(); + public void refresh(Group group, long refreshVersion, BitSet targetBitSet) { + Set refreshedGroup = new HashSet<>(); for (GroupExpression groupExpression : group.getLogicalExpressions()) { - List> childrenTableMap = new ArrayList<>(); - boolean needRefresh = groupExpressionMap.isEmpty(); + List> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { BitSet leaf = constructLeaf(groupExpression); - groupExpressionMap.put(leaf, Pair.of(groupExpression, new ArrayList<>())); + groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); continue; } - for (Group child : groupExpression.children()) { - if (!refreshedGroup.contains(child) && !child.getstructInfoMap().isRefreshed()) { - StructInfoMap childStructInfoMap = child.getstructInfoMap(); - needRefresh |= childStructInfoMap.refresh(child); - childStructInfoMap.setRefreshed(true); + StructInfoMap childStructInfoMap = child.getstructInfoMap(); + if (!refreshedGroup.contains(child.getGroupId().asInt()) + && refreshVersion != childStructInfoMap.getRefreshVersion()) { + childStructInfoMap.refresh(child, refreshVersion, targetBitSet); + childStructInfoMap.setRefreshVersion(refreshVersion); } - refreshedGroup.add(child); + refreshedGroup.add(child.getGroupId().asInt()); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); } // if one same groupExpression have refreshed, continue @@ -150,15 +150,14 @@ public boolean refresh(Group group) { } // if cumulative child table map is different from current // or current group expression map is empty, should update the groupExpressionMap currently - Collection>> bitSetWithChildren = cartesianProduct(childrenTableMap); - if (needRefresh) { - for (Pair> bitSetWithChild : bitSetWithChildren) { - groupExpressionMap.putIfAbsent(bitSetWithChild.first, - Pair.of(groupExpression, bitSetWithChild.second)); - } + Collection>> bitSetWithChildren = cartesianProduct(childrenTableMap, + new BitSet()); + for (Pair> bitSetWithChild : bitSetWithChildren) { + groupExpressionMap.putIfAbsent(bitSetWithChild.first, + Pair.of(groupExpression, bitSetWithChild.second)); } + } - return originSize != groupExpressionMap.size(); } private BitSet constructLeaf(GroupExpression groupExpression) { @@ -172,7 +171,8 @@ private BitSet constructLeaf(GroupExpression groupExpression) { return tableMap; } - private Collection>> cartesianProduct(List> childrenTableMap) { + private Collection>> cartesianProduct(List> childrenTableMap, + BitSet targetBitSet) { Set> cartesianLists = Sets.cartesianProduct(childrenTableMap); List>> resultPairSet = new LinkedList<>(); for (List bitSetList : cartesianLists) { @@ -180,6 +180,10 @@ private Collection>> cartesianProduct(List for (BitSet b : bitSetList) { bitSet.or(b); } + // filter the useless bitset which targetBitSet not contains, avoid exponential expansion + if (!targetBitSet.isEmpty() && !StructInfo.containsAll(targetBitSet, bitSet)) { + continue; + } resultPairSet.add(Pair.of(bitSet, bitSetList)); } return resultPairSet; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 6bf47f00c359c0..3405942c3a8817 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -142,6 +142,7 @@ public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { protected List getValidQueryStructInfos(Plan queryPlan, CascadesContext cascadesContext, BitSet materializedViewTableSet) { List validStructInfos = new ArrayList<>(); + // For every materialized view we should trigger refreshing struct info map List uncheckedStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext, materializedViewTableSet); uncheckedStructInfos.forEach(queryStructInfo -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 46d0adde06e978..5f7dc419eafd22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -148,16 +148,23 @@ public static List extractStructInfo(Plan plan, CascadesContext casc if (plan.getGroupExpression().isPresent()) { Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); - structInfoMap.refresh(ownerGroup); + if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion() + || structInfoMap.getTableMaps().isEmpty()) { + structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion(), + materializedViewTableSet); + structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); + } Set queryTableSets = structInfoMap.getTableMaps(); ImmutableList.Builder structInfosBuilder = ImmutableList.builder(); if (!queryTableSets.isEmpty()) { for (BitSet queryTableSet : queryTableSets) { + // TODO As only support MatchMode.COMPLETE, so only get equaled query table struct info if (!materializedViewTableSet.isEmpty() - && !StructInfo.containsAll(materializedViewTableSet, queryTableSet)) { + && !materializedViewTableSet.equals(queryTableSet)) { continue; } - StructInfo structInfo = structInfoMap.getStructInfo(queryTableSet, queryTableSet, ownerGroup, plan); + StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext.getMemo(), + queryTableSet, queryTableSet, ownerGroup, plan); if (structInfo != null) { structInfosBuilder.add(structInfo); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 13bdf35252edd2..9192f86cf3bb41 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -50,7 +50,7 @@ void testTableMap() throws Exception { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root); + root.getstructInfoMap().refresh(root, 1, new BitSet()); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -76,7 +76,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root); + root.getstructInfoMap().refresh(root, 1, new BitSet()); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -97,10 +97,8 @@ void testLazyRefresh() throws Exception { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - boolean refreshed = root.getstructInfoMap().refresh(root); - Assertions.assertTrue(refreshed); - refreshed = root.getstructInfoMap().refresh(root); - Assertions.assertFalse(refreshed); + root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1, new BitSet()); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -126,10 +124,8 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - refreshed = root.getstructInfoMap().refresh(root); - Assertions.assertTrue(refreshed); - refreshed = root.getstructInfoMap().refresh(root); - Assertions.assertFalse(refreshed); + root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1, new BitSet()); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -166,13 +162,13 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .rewrite() .optimize(); Group root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root); + root.getstructInfoMap().refresh(root, 1, new BitSet()); StructInfoMap structInfoMap = root.getstructInfoMap(); Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); BitSet mvMap = structInfoMap.getTableMaps().stream() .filter(b -> b.cardinality() == 2) .collect(Collectors.toList()).get(0); - StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap, root, null); + StructInfo structInfo = structInfoMap.getStructInfo(c1.getMemo(), mvMap, mvMap, root, null); System.out.println(structInfo.getOriginalPlan().treeString()); BitSet bitSet = new BitSet(); structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId())); From 5d5448fd25cf0cb711db6118cb4870e17072601f Mon Sep 17 00:00:00 2001 From: Mryange <59914473+Mryange@users.noreply.github.com> Date: Fri, 26 Apr 2024 01:10:07 +0800 Subject: [PATCH 032/163] [profile](scan) add projection time in scaner #34120 --- be/src/vec/exec/scan/scanner_context.cpp | 7 +++++++ be/src/vec/exec/scan/vscanner.cpp | 1 + be/src/vec/exec/scan/vscanner.h | 2 ++ 3 files changed, 10 insertions(+) diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp index 8d4a9b3a16436e..81e4dacba57c31 100644 --- a/be/src/vec/exec/scan/scanner_context.cpp +++ b/be/src/vec/exec/scan/scanner_context.cpp @@ -405,9 +405,11 @@ void ScannerContext::stop_scanners(RuntimeState* state) { std::stringstream scanner_statistics; std::stringstream scanner_rows_read; std::stringstream scanner_wait_worker_time; + std::stringstream scanner_projection; scanner_statistics << "["; scanner_rows_read << "["; scanner_wait_worker_time << "["; + scanner_projection << "["; // Scanners can in 3 state // state 1: in scanner context, not scheduled // state 2: in scanner worker pool's queue, scheduled but not running @@ -421,6 +423,9 @@ void ScannerContext::stop_scanners(RuntimeState* state) { scanner_statistics << PrettyPrinter::print(scanner->_scanner->get_time_cost_ns(), TUnit::TIME_NS) << ", "; + scanner_projection << PrettyPrinter::print(scanner->_scanner->projection_time(), + TUnit::TIME_NS) + << ", "; scanner_rows_read << PrettyPrinter::print(scanner->_scanner->get_rows_read(), TUnit::UNIT) << ", "; @@ -434,9 +439,11 @@ void ScannerContext::stop_scanners(RuntimeState* state) { scanner_statistics << "]"; scanner_rows_read << "]"; scanner_wait_worker_time << "]"; + scanner_projection << "]"; _scanner_profile->add_info_string("PerScannerRunningTime", scanner_statistics.str()); _scanner_profile->add_info_string("PerScannerRowsRead", scanner_rows_read.str()); _scanner_profile->add_info_string("PerScannerWaitTime", scanner_wait_worker_time.str()); + _scanner_profile->add_info_string("PerScannerProjectionTime", scanner_projection.str()); } _blocks_queue_added_cv.notify_one(); diff --git a/be/src/vec/exec/scan/vscanner.cpp b/be/src/vec/exec/scan/vscanner.cpp index f3835e6c889000..79fa4019687b42 100644 --- a/be/src/vec/exec/scan/vscanner.cpp +++ b/be/src/vec/exec/scan/vscanner.cpp @@ -187,6 +187,7 @@ Status VScanner::_filter_output_block(Block* block) { Status VScanner::_do_projections(vectorized::Block* origin_block, vectorized::Block* output_block) { SCOPED_RAW_TIMER(&_per_scanner_timer); + SCOPED_RAW_TIMER(&_projection_timer); const size_t rows = origin_block->rows(); if (rows == 0) { diff --git a/be/src/vec/exec/scan/vscanner.h b/be/src/vec/exec/scan/vscanner.h index ba953192507c8e..6e83c059706a45 100644 --- a/be/src/vec/exec/scan/vscanner.h +++ b/be/src/vec/exec/scan/vscanner.h @@ -109,6 +109,7 @@ class VScanner { int64_t get_time_cost_ns() const { return _per_scanner_timer; } + int64_t projection_time() const { return _projection_timer; } int64_t get_rows_read() const { return _num_rows_read; } bool is_init() const { return _is_init; } @@ -237,6 +238,7 @@ class VScanner { ScannerCounter _counter; int64_t _per_scanner_timer = 0; + int64_t _projection_timer = 0; bool _should_stop = false; }; From 09525e7e905db5984a11568a7ca1a63e6bdee94a Mon Sep 17 00:00:00 2001 From: StarryVerse <113903752+StarryVerse@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:19:39 +0800 Subject: [PATCH 033/163] Update doris_main.cpp (#34128) * Update doris_main.cpp Log(FATAL) introduces a core dump, which is confusing for users. We should print error msg and exit without a core dump. * Update doris_main.cpp --- be/src/service/doris_main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 98712a8168be5a..ca7fca1ce62de3 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -169,8 +169,9 @@ auto instruction_fail_to_string(InstructionFail fail) { case InstructionFail::ARM_NEON: ret("ARM_NEON"); } - LOG(FATAL) << "__builtin_unreachable"; - __builtin_unreachable(); + + LOG(ERROR) << "Unrecognized instruction fail value." << std::endl; + exit(-1); } sigjmp_buf jmpbuf; From a12b410273ec1bc2a2ed965dcccae000eb5fcb92 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:20:18 +0800 Subject: [PATCH 034/163] [fix](Nereids) check after rewrite cannot handle agg in other opeator (#34114) this is a stupid mistake. we import a same name class from another package --- .../apache/doris/nereids/rules/analysis/CheckAfterRewrite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 40a59697514209..3e079cd19af9d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.analysis; -import org.apache.doris.catalog.AggregateFunction; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; @@ -33,6 +32,7 @@ import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.window.WindowFunction; From 132d974fa21039d43e616355a13eff442cc187b6 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 26 Apr 2024 07:21:00 +0800 Subject: [PATCH 035/163] Revert "[fix](memory) Fix Jemalloc hook failed to start BE with JDK 17 #33946" (#34107) This reverts commit 5fee7157ccfc8ed88bc3405eb404a2916b5aac3e. --- be/CMakeLists.txt | 5 ----- be/src/runtime/CMakeLists.txt | 2 +- build.sh | 6 ------ cloud/CMakeLists.txt | 3 --- cloud/src/common/CMakeLists.txt | 2 +- regression-test/pipeline/performance/compile.sh | 1 - 6 files changed, 2 insertions(+), 17 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index fd1f2c1db4a4cd..28b3fee115aab0 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -72,7 +72,6 @@ option(USE_LIBCPP "Use libc++" OFF) option(USE_MEM_TRACKER, "Use memory tracker" ON) option(USE_UNWIND "Use libunwind" ON) option(USE_JEMALLOC "Use jemalloc" ON) -option(USE_JEMALLOC_HOOK "Use jemalloc hook" ON) if (OS_MACOSX) set(GLIBC_COMPATIBILITY OFF) set(USE_LIBCPP ON) @@ -88,7 +87,6 @@ message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}") message(STATUS "USE_LIBCPP is ${USE_LIBCPP}") message(STATUS "USE_MEM_TRACKER is ${USE_MEM_TRACKER}") message(STATUS "USE_JEMALLOC is ${USE_JEMALLOC}") -message(STATUS "USE_JEMALLOC_HOOK is ${USE_JEMALLOC_HOOK}") message(STATUS "USE_UNWIND is ${USE_UNWIND}") message(STATUS "ENABLE_PCH is ${ENABLE_PCH}") @@ -348,9 +346,6 @@ endif() if (USE_JEMALLOC) add_definitions(-DUSE_JEMALLOC) endif() -if (USE_JEMALLOC_HOOK) - add_definitions(-DUSE_JEMALLOC_HOOK) -endif() # Compile with libunwind if (USE_UNWIND) diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 3bfec93edfc083..a0b3b799a764cb 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -25,7 +25,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime") file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc) -if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER OR NOT USE_JEMALLOC_HOOK) +if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER) list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp) endif() diff --git a/build.sh b/build.sh index f813999f8ba8bc..dbe6fb38b67df1 100755 --- a/build.sh +++ b/build.sh @@ -356,9 +356,6 @@ fi if [[ -z "${USE_JEMALLOC}" ]]; then USE_JEMALLOC='ON' fi -if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then - USE_JEMALLOC_HOOK='OFF' -fi if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then USE_BTHREAD_SCANNER='OFF' fi @@ -464,7 +461,6 @@ echo "Get params: STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO} USE_MEM_TRACKER -- ${USE_MEM_TRACKER} USE_JEMALLOC -- ${USE_JEMALLOC} - USE_JEMALLOC_HOOK -- ${USE_JEMALLOC_HOOK} USE_BTHREAD_SCANNER -- ${USE_BTHREAD_SCANNER} ENABLE_STACKTRACE -- ${ENABLE_STACKTRACE} ENABLE_INJECTION_POINT -- ${ENABLE_INJECTION_POINT} @@ -567,7 +563,6 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_PCH="${ENABLE_PCH}" \ -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ - -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \ -DENABLE_STACKTRACE="${ENABLE_STACKTRACE}" \ -DUSE_AVX2="${USE_AVX2}" \ -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ @@ -611,7 +606,6 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \ -DUSE_DWARF="${USE_DWARF}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ - -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \ "${DORIS_HOME}/cloud/" diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index 9b9929ae1d5e0c..35164af9f9c546 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -188,9 +188,6 @@ endif () if (USE_JEMALLOC) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC") endif() -if (USE_JEMALLOC_HOOK) - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC_HOOK") -endif() if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new") diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt index b18947b04a1e5b..a2abfb075bf934 100644 --- a/cloud/src/common/CMakeLists.txt +++ b/cloud/src/common/CMakeLists.txt @@ -16,7 +16,7 @@ set(COMMON_FILES network_util.cpp ) -if (USE_JEMALLOC AND USE_JEMALLOC_HOOK) +if (USE_JEMALLOC) set(COMMON_FILES ${COMMON_FILES} jemalloc_hook.cpp ) diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh index 2a30fec0a7b494..cfeae6af5f2d36 100644 --- a/regression-test/pipeline/performance/compile.sh +++ b/regression-test/pipeline/performance/compile.sh @@ -135,7 +135,6 @@ sudo docker run -i --rm \ && export CCACHE_REMOTE_STORAGE=file:///root/ccache \ && export EXTRA_CXX_FLAGS=-O3 \ && export USE_JEMALLOC='ON' \ - && export USE_JEMALLOC_HOOK='OFF' \ && export ENABLE_PCH=OFF ${jdk17_str}\ && export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \ && bash build.sh --fe --be --clean 2>&1 | tee build.log" From 0038423c026cbe057237a25d5654fedad5cd3fd3 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:26:36 +0800 Subject: [PATCH 036/163] [fix](nereids)prevent null pointer access if translate expression fails (#33990) --- .../nereids/rules/expression/rules/FoldConstantRuleOnBE.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 3c2f0d546dd04a..e0e19bd19e2b06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -215,6 +215,11 @@ private static void collectConst(Expression expr, Map constM LOG.warn("expression {} translate to legacy expr failed. ", expr, e); return; } + if (staleExpr == null) { + // just return, it's a fail-safe + LOG.warn("expression {} translate to legacy expr failed. ", expr); + return; + } tExprMap.put(id, staleExpr.treeToThrift()); } else { for (int i = 0; i < expr.children().size(); i++) { From 837b5e83d0b30391c1847e58dfd84c40dcca6e7b Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:35:47 +0800 Subject: [PATCH 037/163] [fix](planner)cast expr should do nothing in compactForLiteral method (#34047) --- .../main/java/org/apache/doris/analysis/CastExpr.java | 5 +++++ .../suites/correctness_p0/test_cast_decimal.groovy | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 739901b79295fe..c73afac14c8502 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -592,4 +592,9 @@ public void setNotFold(boolean notFold) { public boolean isNotFold() { return this.notFold; } + + @Override + protected void compactForLiteral(Type type) { + // do nothing + } } diff --git a/regression-test/suites/correctness_p0/test_cast_decimal.groovy b/regression-test/suites/correctness_p0/test_cast_decimal.groovy index 88859ea1d529fd..21a1ab6d0c3ee4 100644 --- a/regression-test/suites/correctness_p0/test_cast_decimal.groovy +++ b/regression-test/suites/correctness_p0/test_cast_decimal.groovy @@ -34,4 +34,15 @@ suite("test_cast_decimal") { sql """select cast(32123.34212456734 as decimal(3,2));""" contains "CAST(32123.34212456734 AS DECIMALV3(3, 2))" } + + sql """drop table if exists test_ttt""" + sql """create table test_ttt(big_key bigint)DISTRIBUTED BY HASH(big_key) BUCKETS 1 PROPERTIES ("replication_num" = "1");""" + sql """set enable_nereids_planner=false;""" + sql """set enable_fold_constant_by_be = false; """ + sql """SELECT 1 + FROM test_ttt e1 + HAVING truncate(100, 2) < -2308.57 + AND cast(round(round(465.56, min(-5.987)), 2) AS DECIMAL) in + (SELECT cast(truncate(round(8990.65 - 4556.2354, 2.4652), 2)AS DECIMAL) + FROM test_ttt r2);""" } From 05e734a8e6b033d11caf9d4f03e624554e23ec7c Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:36:05 +0800 Subject: [PATCH 038/163] [fix](nereids)move ReplaceVariableByLiteral rule to analyze phase (#33997) --- .../doris/nereids/jobs/executor/Analyzer.java | 12 ++++++ .../rules/analysis/VariableToLiteral.java | 39 +++++++++++++++++++ .../expression/ExpressionNormalization.java | 2 - .../suites/nereids_p0/test_user_var.groovy | 35 +++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index a0431e066beee8..ac0a44210717fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -44,6 +44,7 @@ import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; import org.apache.doris.nereids.rules.analysis.SubqueryToApply; +import org.apache.doris.nereids.rules.analysis.VariableToLiteral; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; @@ -157,6 +158,17 @@ private static List buildAnalyzeJobs(Optional c new NormalizeRepeat() ), bottomUp(new AdjustAggregateNullableForEmptySet()), + // consider sql with user defined var @t_zone + // set @t_zone='GMT'; + // SELECT + // DATE_FORMAT(convert_tz(dt, time_zone, @t_zone),'%Y-%m-%d') day + // FROM + // t + // GROUP BY + // 1; + // @t_zone must be replaced as 'GMT' before EliminateGroupByConstant and NormalizeAggregate rule. + // So need run VariableToLiteral rule before the two rules. + topDown(new VariableToLiteral()), // run CheckAnalysis before EliminateGroupByConstant in order to report error message correctly like bellow // select SUM(lo_tax) FROM lineorder group by 1; // errCode = 2, detailMessage = GROUP BY expression must not contain aggregate functions: sum(lo_tax) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java new file mode 100644 index 00000000000000..c7ba1bfe6a7af3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteRule; +import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.rules.expression.rules.ReplaceVariableByLiteral; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * replace Variable To Literal + */ +public class VariableToLiteral extends ExpressionRewrite { + public static final List NORMALIZE_REWRITE_RULES = + ImmutableList.of(bottomUp(ReplaceVariableByLiteral.INSTANCE)); + + public VariableToLiteral() { + super(new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java index adf0cb90a958c1..f63ab2eee5f942 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.rules.expression.rules.InPredicateDedup; import org.apache.doris.nereids.rules.expression.rules.InPredicateToEqualToRule; import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule; -import org.apache.doris.nereids.rules.expression.rules.ReplaceVariableByLiteral; import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule; import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticRule; import org.apache.doris.nereids.rules.expression.rules.SimplifyCastRule; @@ -43,7 +42,6 @@ public class ExpressionNormalization extends ExpressionRewrite { // from_unixtime(timestamp, 'yyyyMMdd') to 'yyyyMMdd' public static final List NORMALIZE_REWRITE_RULES = ImmutableList.of( bottomUp( - ReplaceVariableByLiteral.INSTANCE, SupportJavaDateFormatter.INSTANCE, NormalizeBinaryPredicatesRule.INSTANCE, InPredicateDedup.INSTANCE, diff --git a/regression-test/suites/nereids_p0/test_user_var.groovy b/regression-test/suites/nereids_p0/test_user_var.groovy index d5eb164e3f9a7f..8a7db3736822eb 100644 --- a/regression-test/suites/nereids_p0/test_user_var.groovy +++ b/regression-test/suites/nereids_p0/test_user_var.groovy @@ -31,4 +31,39 @@ suite("test_user_var") { qt_boolean 'select @d1, @d2;' qt_null_literal 'select @f1, @f2;' qt_function 'select @func_1' + + multi_sql( + """ + drop table if exists dwd_login_ttt; + + CREATE TABLE `dwd_login_ttt` ( + `game_code` varchar(100) NOT NULL DEFAULT "-" , + `plat_code` varchar(100) NOT NULL DEFAULT "-" , + `userid` varchar(255) NULL DEFAULT "-" , + `dt` datetime NOT NULL, + `time_zone` varchar(100) NULL + ) ENGINE=OLAP + UNIQUE KEY(`game_code`, `plat_code`) + DISTRIBUTED BY HASH(`game_code`) BUCKETS 16 + PROPERTIES("replication_num" = "1"); + + drop view if exists dwd_login_ttt_view; + + create view dwd_login_ttt_view as + SELECT game_code,plat_code,time_zone,DATE_FORMAT(convert_tz(dt,time_zone,@t_zone),'%Y-%m-%d') day,count(distinct userid) + from dwd_login_ttt + where dt>=convert_tz(@t_day,'Asia/Shanghai',@t_zone) + and dt= '2024-01-31 16:00:00'" + } } \ No newline at end of file From e6801105349b262f4ca5eef7812d764a124dd57c Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:38:10 +0800 Subject: [PATCH 039/163] [fix](planner)date_add function should accept date type as its param (#34035) --- .../apache/doris/analysis/FunctionCallExpr.java | 16 ++++++++++++++++ gensrc/script/doris_builtins_functions.py | 2 ++ .../correctness/test_date_function_const.groovy | 5 ++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 9cfce9e67de8ad..3978c3802bc1e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1658,6 +1658,22 @@ && collectChildReturnTypes()[0].isDecimalV3()) { } fn = getBuiltinFunction(fnName.getFunction(), argTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + } else if (fnName.getFunction().equalsIgnoreCase("date_add") + || fnName.getFunction().equalsIgnoreCase("days_add") + || fnName.getFunction().equalsIgnoreCase("adddate") + || fnName.getFunction().equalsIgnoreCase("date_sub") + || fnName.getFunction().equalsIgnoreCase("days_sub") + || fnName.getFunction().equalsIgnoreCase("subdate")) { + Type[] childTypes = collectChildReturnTypes(); + argTypes[0] = childTypes[0]; + argTypes[1] = childTypes[1]; + if (childTypes[1] == Type.TINYINT || childTypes[1] == Type.SMALLINT) { + // be only support second param as int type + uncheckedCastChild(Type.INT, 1); + argTypes[1] = Type.INT; + } + fn = getBuiltinFunction(fnName.getFunction(), argTypes, + Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); } else { // now first find table function in table function sets if (isTableFnCall) { diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 841799f0a13b85..b7912d08904775 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1080,6 +1080,8 @@ [['weeks_sub'], 'DATEV2', ['DATEV2', 'INT'], ''], [['days_add', 'date_add', 'adddate'], 'DATEV2', ['DATEV2', 'INT'], ''], [['days_sub', 'date_sub', 'subdate'], 'DATEV2', ['DATEV2', 'INT'], ''], + [['days_add', 'date_add', 'adddate'], 'DATE', ['DATE', 'INT'], ''], + [['days_sub', 'date_sub', 'subdate'], 'DATE', ['DATE', 'INT'], ''], [['hours_add'], 'DATETIMEV2', ['DATEV2', 'INT'], ''], [['hours_sub'], 'DATETIMEV2', ['DATEV2', 'INT'], ''], [['minutes_add'], 'DATETIMEV2', ['DATEV2', 'INT'], ''], diff --git a/regression-test/suites/correctness/test_date_function_const.groovy b/regression-test/suites/correctness/test_date_function_const.groovy index 85d8e5ba711370..df05738b1bf674 100644 --- a/regression-test/suites/correctness/test_date_function_const.groovy +++ b/regression-test/suites/correctness/test_date_function_const.groovy @@ -57,5 +57,8 @@ suite("test_date_function_const") { qt_select10 """ select hours_add(cast('2023-03-30 22:23:45.23452' as datetimev2(6)),8) """ - + explain { + sql("""select date_add(CURRENT_DATE(),-2);""") + notContains("00:00:00") + } } \ No newline at end of file From 04cfa16208e270ea84b4c7e4feb51c90766a0af2 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Fri, 26 Apr 2024 09:53:06 +0800 Subject: [PATCH 040/163] [chore](ci) adjust cloud_p* be.conf (#34132) --- regression-test/pipeline/cloud_p0/conf/be_custom.conf | 1 - regression-test/pipeline/cloud_p1/conf/be_custom.conf | 1 - 2 files changed, 2 deletions(-) diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf b/regression-test/pipeline/cloud_p0/conf/be_custom.conf index 9f85d1c98fac37..a84891beda8e62 100644 --- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf @@ -7,7 +7,6 @@ push_worker_count_high_priority = 2 streaming_load_max_mb = 107374182400 clear_file_cache=true enable_file_cache=true -mem_limit=50% #disable_storage_page_cache = true enable_file_cache_query_limit=true file_cache_max_file_segment_size=1048576 diff --git a/regression-test/pipeline/cloud_p1/conf/be_custom.conf b/regression-test/pipeline/cloud_p1/conf/be_custom.conf index 1f4104304fcaf8..ae9efacf4883bb 100644 --- a/regression-test/pipeline/cloud_p1/conf/be_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/be_custom.conf @@ -7,7 +7,6 @@ push_worker_count_high_priority = 2 streaming_load_max_mb = 107374182400 clear_file_cache=true enable_file_cache=true -mem_limit=50% #disable_storage_page_cache = true enable_file_cache_query_limit=true file_cache_max_file_segment_size=1048576 From 09ebfafc86876a3462e7e629c9dbecefc370d2e2 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Fri, 26 Apr 2024 10:26:16 +0800 Subject: [PATCH 041/163] [chore](variable) deprecated GROUP_BY_AND_HAVING_USE_ALIAS_FIRST (#34133) this variable intro by #15748 for backward compatibility. currently, it is used very infrequently. Nereids do not support it anymore. So, we tag it as deprecated, and will remove it in the future. --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 2037f5fa1efb27..579ad21c9ab1c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1338,7 +1338,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { // Default value is false, which means the group by and having clause // should first use column name not alias. According to mysql. - @VariableMgr.VarAttr(name = GROUP_BY_AND_HAVING_USE_ALIAS_FIRST) + @VariableMgr.VarAttr(name = GROUP_BY_AND_HAVING_USE_ALIAS_FIRST, varType = VariableAnnotation.DEPRECATED) public boolean groupByAndHavingUseAliasFirst = false; // Whether disable block file cache. Block cache only works when FE's query options sets disableFileCache false From d4cdd289d37dd93719d344b19cc3ca3ab66af8cd Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:45:35 +0800 Subject: [PATCH 042/163] Support high priority column stats auto collection. (#33703) * fix visible column (#33023) * Collect high priority columns. (#31235) * High priority queue and map. (#31509) * Support column level health value. (#31794) * Support follower sync query columns to master. (#31859) * Support show auto analyze pending jobs. (#31926) * Check column health value earlier, show job priority. (#32064) * support window (#32094) * Refactor. (#32273) * refactor2 (#32278) * Unit test (#32398) * Support auto analyze mv (#32433) * Fix bug (#32454) * Support identical column name in different index. (#32957) * Fix visible column * Use future to block auto analyze before job finish. (#33083) * Fix ut. (#33147) * Fix ut (#33161) * fix p0 (#33210) * Improve failover logic. (#33382) * Improve waiting empty table logic. (#33472) * Fix pipeline (#33671) --- .../java/org/apache/doris/common/Config.java | 2 +- fe/fe-core/src/main/cup/sql_parser.cup | 4 + .../doris/analysis/AnalyzeProperties.java | 2 + .../doris/analysis/ShowAnalyzeStmt.java | 1 + .../analysis/ShowAutoAnalyzeJobsStmt.java | 210 +++++++ .../doris/analysis/ShowColumnStatsStmt.java | 4 + .../java/org/apache/doris/catalog/Env.java | 27 +- .../org/apache/doris/catalog/OlapTable.java | 38 +- .../java/org/apache/doris/catalog/Table.java | 16 +- .../org/apache/doris/catalog/TableIf.java | 13 +- .../doris/datasource/ExternalTable.java | 30 +- .../doris/datasource/InternalCatalog.java | 6 +- .../doris/nereids/jobs/executor/Rewriter.java | 4 +- .../apache/doris/nereids/rules/RuleType.java | 1 + .../expression/QueryColumnCollector.java | 215 ++++++++ .../org/apache/doris/qe/SessionVariable.java | 7 + .../org/apache/doris/qe/ShowExecutor.java | 34 ++ .../doris/service/FrontendServiceImpl.java | 8 + .../apache/doris/statistics/AnalysisInfo.java | 27 +- .../doris/statistics/AnalysisInfoBuilder.java | 29 +- .../apache/doris/statistics/AnalysisJob.java | 8 +- .../doris/statistics/AnalysisManager.java | 165 ++++-- .../statistics/AnalysisTaskExecutor.java | 7 +- .../statistics/AutoAnalysisPendingJob.java | 52 ++ .../doris/statistics/BaseAnalysisTask.java | 36 +- .../apache/doris/statistics/ColStatsMeta.java | 16 +- .../statistics/ExternalAnalysisTask.java | 9 +- .../statistics/FollowerColumnSender.java | 151 ++++++ .../doris/statistics/HistogramTask.java | 5 - .../apache/doris/statistics/JobPriority.java | 25 + .../doris/statistics/OlapAnalysisTask.java | 27 +- .../apache/doris/statistics/QueryColumn.java | 66 +++ .../doris/statistics/StatisticConstants.java | 4 +- .../statistics/StatisticsAutoCollector.java | 270 +++++---- .../doris/statistics/StatisticsCollector.java | 79 --- .../statistics/StatisticsJobAppender.java | 204 +++++++ .../statistics/StatisticsRepository.java | 4 +- .../doris/statistics/TableStatsMeta.java | 22 +- .../doris/statistics/util/StatisticsUtil.java | 83 ++- .../doris/statistics/AnalysisJobTest.java | 8 +- .../doris/statistics/AnalysisManagerTest.java | 371 ++++++++++--- .../statistics/AnalysisTaskExecutorTest.java | 5 +- .../apache/doris/statistics/AnalyzeTest.java | 6 +- .../statistics/FollowerColumnSenderTest.java | 88 +++ .../StatisticsAutoCollectorTest.java | 512 +++--------------- .../statistics/StatisticsJobAppenderTest.java | 281 ++++++++++ .../doris/statistics/TableStatsMetaTest.java | 14 +- .../statistics/util/StatisticsUtilTest.java | 149 +++++ gensrc/thrift/FrontendService.thrift | 13 + .../hive/test_hive_statistic_auto.groovy | 2 +- .../suites/statistics/analyze_stats.groovy | 2 +- 51 files changed, 2422 insertions(+), 940 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/statistics/FollowerColumnSenderTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c40e4c4bec32fc..e2861310faa592 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1613,7 +1613,7 @@ public class Config extends ConfigBase { "This parameter controls the time interval for automatic collection jobs to check the health of table" + "statistics and trigger automatic collection" }) - public static int auto_check_statistics_in_minutes = 5; + public static int auto_check_statistics_in_minutes = 1; /** * If set to TRUE, the compaction slower replica will be skipped when select get queryable replicas diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 4efd1cf61d4ded..fc0b116bbea967 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -4568,6 +4568,10 @@ show_param ::= {: RESULT = new ShowAnalyzeStmt(tbl, parser.where, true); :} + | KW_AUTO KW_JOBS opt_table_name:tbl opt_wild_where + {: + RESULT = new ShowAutoAnalyzeJobsStmt(tbl, parser.where); + :} | KW_ANALYZE KW_TASK KW_STATUS INTEGER_LITERAL:jobId {: RESULT = new ShowAnalyzeTaskStatus(jobId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java index 94083989ca7b83..f78c63ebea1a83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java @@ -44,6 +44,7 @@ public class AnalyzeProperties { public static final String PROPERTY_PERIOD_SECONDS = "period.seconds"; public static final String PROPERTY_FORCE_FULL = "force.full"; public static final String PROPERTY_PARTITION_COLUMN_FROM_SQL = "partition.column.from.sql"; + public static final String PROPERTY_USE_AUTO_ANALYZER = "use.auto.analyzer"; public static final AnalyzeProperties DEFAULT_PROP = new AnalyzeProperties(new HashMap() { { @@ -72,6 +73,7 @@ public class AnalyzeProperties { .add(PROPERTY_PERIOD_CRON) .add(PROPERTY_FORCE_FULL) .add(PROPERTY_PARTITION_COLUMN_FROM_SQL) + .add(PROPERTY_USE_AUTO_ANALYZER) .build(); public AnalyzeProperties(Map properties) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java index 9ccfd956ca5d84..f660d6eeb3c6b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java @@ -62,6 +62,7 @@ public class ShowAnalyzeStmt extends ShowStmt { .add("schedule_type") .add("start_time") .add("end_time") + .add("priority") .build(); private long jobId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java new file mode 100644 index 00000000000000..560387fa5bc11c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAutoAnalyzeJobsStmt.java @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.statistics.JobPriority; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; + +/** + * ShowAutoAnalyzeJobsStmt is used to show pending auto analysis jobs. + * syntax: + * SHOW AUTO ANALYZE JOBS + * [TABLE] + * [ + * WHERE + * [PRIORITY = ["HIGH"|"MID"|"LOW"]] + * ] + */ +public class ShowAutoAnalyzeJobsStmt extends ShowStmt { + private static final String PRIORITY = "priority"; + private static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("catalog_name") + .add("db_name") + .add("tbl_name") + .add("col_list") + .add("priority") + .build(); + + private final TableName tableName; + private final Expr whereClause; + + public ShowAutoAnalyzeJobsStmt(TableName tableName, Expr whereClause) { + this.tableName = tableName; + this.whereClause = whereClause; + } + + // extract from predicate + private String jobPriority; + + public String getPriority() { + Preconditions.checkArgument(isAnalyzed(), + "The stateValue must be obtained after the parsing is complete"); + return jobPriority; + } + + public Expr getWhereClause() { + Preconditions.checkArgument(isAnalyzed(), + "The whereClause must be obtained after the parsing is complete"); + return whereClause; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + if (!ConnectContext.get().getSessionVariable().enableStats) { + throw new UserException("Analyze function is forbidden, you should add `enable_stats=true`" + + "in your FE conf file"); + } + super.analyze(analyzer); + if (tableName != null) { + tableName.analyze(analyzer); + String catalogName = tableName.getCtl(); + String dbName = tableName.getDb(); + String tblName = tableName.getTbl(); + checkShowAnalyzePriv(catalogName, dbName, tblName); + } + + // analyze where clause if not null + if (whereClause != null) { + analyzeSubPredicate(whereClause); + } + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TITLE_NAMES) { + builder.addColumn(new Column(title, ScalarType.createVarchar(128))); + } + return builder.build(); + } + + @Override + public RedirectStatus getRedirectStatus() { + return RedirectStatus.FORWARD_NO_SYNC; + } + + private void checkShowAnalyzePriv(String catalogName, String dbName, String tblName) throws AnalysisException { + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), catalogName, dbName, tblName, PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException( + ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, + "SHOW ANALYZE", + ConnectContext.get().getQualifiedUser(), + ConnectContext.get().getRemoteIP(), + dbName + ": " + tblName); + } + } + + private void analyzeSubPredicate(Expr subExpr) throws AnalysisException { + if (subExpr == null) { + return; + } + + boolean valid = true; + + CHECK: { + if (subExpr instanceof BinaryPredicate) { + BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr; + if (binaryPredicate.getOp() != BinaryPredicate.Operator.EQ) { + valid = false; + break CHECK; + } + } else { + valid = false; + break CHECK; + } + + // left child + if (!(subExpr.getChild(0) instanceof SlotRef)) { + valid = false; + break CHECK; + } + String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName(); + if (!PRIORITY.equalsIgnoreCase(leftKey)) { + valid = false; + break CHECK; + } + + // right child + if (!(subExpr.getChild(1) instanceof StringLiteral)) { + valid = false; + break CHECK; + } + + String value = subExpr.getChild(1).getStringValue(); + if (Strings.isNullOrEmpty(value)) { + valid = false; + break CHECK; + } + + jobPriority = value.toUpperCase(); + try { + JobPriority.valueOf(jobPriority); + } catch (Exception e) { + valid = false; + } + } + + if (!valid) { + throw new AnalysisException("Where clause should looks like: " + + "PRIORITY = \"HIGH|MID|LOW\""); + } + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("SHOW AUTO ANALYZE"); + + if (tableName != null) { + sb.append(" "); + sb.append(tableName.toSql()); + } + + if (whereClause != null) { + sb.append(" "); + sb.append("WHERE"); + sb.append(" "); + sb.append(whereClause.toSql()); + } + + return sb.toString(); + } + + @Override + public String toString() { + return toSql(); + } + + public TableName getTableName() { + return tableName; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java index 18bb916b8bdfce..cfe2d426f7b166 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java @@ -62,6 +62,8 @@ public class ShowColumnStatsStmt extends ShowStmt { .add("trigger") .add("query_times") .add("updated_time") + .add("update_rows") + .add("last_analyze_row_count") .build(); private final TableName tableName; @@ -162,6 +164,8 @@ public ShowResultSet constructResultSet(List, ColumnSt row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.jobType)); row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.queriedTimes)); row.add(String.valueOf(p.second.updatedTime)); + row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.updatedRows)); + row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.rowCount)); result.add(row); }); return new ShowResultSet(getMetaData(), result); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 0ad32f76566768..275c4d1ff42171 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -244,9 +244,11 @@ import org.apache.doris.service.ExecuteEnv; import org.apache.doris.service.FrontendOptions; import org.apache.doris.statistics.AnalysisManager; +import org.apache.doris.statistics.FollowerColumnSender; import org.apache.doris.statistics.StatisticsAutoCollector; import org.apache.doris.statistics.StatisticsCache; import org.apache.doris.statistics.StatisticsCleaner; +import org.apache.doris.statistics.StatisticsJobAppender; import org.apache.doris.statistics.query.QueryStats; import org.apache.doris.system.Backend; import org.apache.doris.system.Frontend; @@ -524,6 +526,10 @@ public class Env { private StatisticsAutoCollector statisticsAutoCollector; + private StatisticsJobAppender statisticsJobAppender; + + private FollowerColumnSender followerColumnSender; + private HiveTransactionMgr hiveTransactionMgr; private TopicPublisherThread topicPublisherThread; @@ -756,6 +762,7 @@ public Env(boolean isCheckpointCatalog) { this.analysisManager = new AnalysisManager(); this.statisticsCleaner = new StatisticsCleaner(); this.statisticsAutoCollector = new StatisticsAutoCollector(); + this.statisticsJobAppender = new StatisticsJobAppender(); this.globalFunctionMgr = new GlobalFunctionMgr(); this.workloadGroupMgr = new WorkloadGroupMgr(); this.workloadSchedPolicyMgr = new WorkloadSchedPolicyMgr(); @@ -1058,13 +1065,6 @@ public void initialize(String[] args) throws Exception { // If not using bdb, we need to notify the FE type transfer manually. notifyNewFETypeTransfer(FrontendNodeType.MASTER); } - if (statisticsCleaner != null) { - statisticsCleaner.start(); - } - if (statisticsAutoCollector != null) { - statisticsAutoCollector.start(); - } - queryCancelWorker.start(); } @@ -1715,6 +1715,10 @@ protected void startMasterOnlyDaemonThreads() { topicPublisherThread.addToTopicPublisherList(wpPublisher); topicPublisherThread.start(); + // auto analyze related threads. + statisticsCleaner.start(); + statisticsAutoCollector.start(); + statisticsJobAppender.start(); } // start threads that should run on all FE @@ -1777,6 +1781,11 @@ private void transferToNonMaster(FrontendNodeType newType) { if (analysisManager != null) { analysisManager.getStatisticsCache().preHeat(); } + + if (followerColumnSender == null) { + followerColumnSender = new FollowerColumnSender(); + followerColumnSender.start(); + } } // Set global variable 'lower_case_table_names' only when the cluster is initialized. @@ -6113,6 +6122,10 @@ public NereidsSqlCacheManager getSqlCacheManager() { return sqlCacheManager; } + public StatisticsJobAppender getStatisticsJobAppender() { + return statisticsJobAppender; + } + public void alterMTMVRefreshInfo(AlterMTMVRefreshInfo info) { AlterMTMV alter = new AlterMTMV(info.getMvName(), info.getRefreshInfo(), MTMVAlterOpType.ALTER_REFRESH_INFO); this.alter.processAlterMTMV(alter, false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 3932496f661c5a..a4c79d91890c1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -64,7 +64,6 @@ import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.HistogramTask; import org.apache.doris.statistics.OlapAnalysisTask; -import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; @@ -802,9 +801,20 @@ public List getSchemaByIndexId(Long indexId, boolean full) { } @Override - public List getSchemaAllIndexes(boolean full) { + public Set getSchemaAllIndexes(boolean full) { + Set columns = Sets.newHashSet(); + for (Long indexId : indexIdToMeta.keySet()) { + columns.addAll(getSchemaByIndexId(indexId, full)); + } + return columns; + } + + public List getMvColumns(boolean full) { List columns = Lists.newArrayList(); for (Long indexId : indexIdToMeta.keySet()) { + if (indexId == baseIndexId) { + continue; + } columns.addAll(getSchemaByIndexId(indexId, full)); } return columns; @@ -1323,29 +1333,9 @@ public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) { } } - public boolean needReAnalyzeTable(TableStatsMeta tblStats) { - if (tblStats == null) { - return true; - } - if (!tblStats.analyzeColumns().containsAll(getColumnIndexPairs(getSchemaAllIndexes(false) - .stream() - .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) - .map(Column::getName) - .collect(Collectors.toSet())))) { - return true; - } - long rowCount = getRowCount(); - if (rowCount > 0 && tblStats.rowCount == 0) { - return true; - } - long updateRows = tblStats.updatedRows.get(); - int tblHealth = StatisticsUtil.getTableHealth(rowCount, updateRows); - return tblHealth < StatisticsUtil.getTableStatsHealthThreshold(); - } - @Override - public List> getColumnIndexPairs(Set columns) { - List> ret = Lists.newArrayList(); + public Set> getColumnIndexPairs(Set columns) { + Set> ret = Sets.newHashSet(); // Check the schema of all indexes for each given column name, // If the column name exists in the index, add the pair to return list. for (String column : columns) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 136a0e04f2c253..9de006cebb5c46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -34,13 +34,13 @@ import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.ColumnStatistic; -import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.thrift.TTableDescriptor; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; @@ -391,11 +391,6 @@ public List getBaseSchema() { return getBaseSchema(Util.showHiddenColumns()); } - @Override - public List getSchemaAllIndexes(boolean full) { - return getBaseSchema(); - } - public List getBaseSchema(boolean full) { if (full) { return fullSchema; @@ -623,11 +618,6 @@ public Optional getColumnStatistic(String colName) { public void analyze(String dbName) {} - @Override - public boolean needReAnalyzeTable(TableStatsMeta tblStats) { - return true; - } - @Override public List getChunkSizes() { throw new NotImplementedException("getChunkSized not implemented"); @@ -639,8 +629,8 @@ public long fetchRowCount() { } @Override - public List> getColumnIndexPairs(Set columns) { - return Lists.newArrayList(); + public Set> getColumnIndexPairs(Set columns) { + return Sets.newHashSet(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 1ba8ee30766667..c5039660e6eeae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -31,7 +31,6 @@ import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.ColumnStatistic; -import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.thrift.TTableDescriptor; import com.google.common.collect.ImmutableList; @@ -120,7 +119,11 @@ default boolean tryWriteLockIfExist(long timeout, TimeUnit unit) { List getBaseSchema(); - List getSchemaAllIndexes(boolean full); + default Set getSchemaAllIndexes(boolean full) { + Set ret = Sets.newHashSet(); + ret.addAll(getBaseSchema()); + return ret; + } default List getBaseSchemaOrEmpty() { try { @@ -188,13 +191,11 @@ default long getRowCountForNereids() { Optional getColumnStatistic(String colName); - boolean needReAnalyzeTable(TableStatsMeta tblStats); - /** * @param columns Set of column names. - * @return List of pairs. Each pair is . For external table, index name is table name. + * @return Set of pairs. Each pair is . For external table, index name is table name. */ - List> getColumnIndexPairs(Set columns); + Set> getColumnIndexPairs(Set columns); // Get all the chunk sizes of this table. Now, only HMS external table implemented this interface. // For HMS external table, the return result is a list of all the files' size. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 952b5c64cf8fd5..11226fc9d78aa9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -33,11 +33,10 @@ import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.ColumnStatistic; -import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.TTableDescriptor; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; import lombok.Getter; import org.apache.commons.lang3.NotImplementedException; @@ -51,7 +50,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * External table represent tables that are not self-managed by Doris. @@ -151,11 +149,6 @@ public List getBaseSchema() { return getFullSchema(); } - @Override - public List getSchemaAllIndexes(boolean full) { - return getBaseSchema(); - } - @Override public List getBaseSchema(boolean full) { return getFullSchema(); @@ -331,25 +324,8 @@ public void gsonPostProcess() throws IOException { } @Override - public boolean needReAnalyzeTable(TableStatsMeta tblStats) { - if (tblStats == null) { - return true; - } - if (!tblStats.analyzeColumns().containsAll(getColumnIndexPairs( - getBaseSchema() - .stream() - .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) - .map(Column::getName) - .collect(Collectors.toSet())))) { - return true; - } - return System.currentTimeMillis() - - tblStats.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis(); - } - - @Override - public List> getColumnIndexPairs(Set columns) { - List> ret = Lists.newArrayList(); + public Set> getColumnIndexPairs(Set columns) { + Set> ret = Sets.newHashSet(); for (String column : columns) { Column col = getColumn(column); if (col == null || StatisticsUtil.isUnsupportedType(col.getType())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index dd52fad4f7f68c..7bd0040395a174 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -3206,7 +3206,6 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti rowsToTruncate += partition.getBaseIndex().getRowCount(); } } else { - rowsToTruncate = olapTable.getRowCount(); for (Partition partition : olapTable.getPartitions()) { // If need absolutely correct, should check running txn here. // But if the txn is in prepare state, cann't known which partitions had load data. @@ -3215,6 +3214,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti } origPartitions.put(partition.getName(), partition.getId()); partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo()); + rowsToTruncate += partition.getBaseIndex().getRowCount(); } } // if table currently has no partitions, this sql like empty command and do nothing, should return directly. @@ -3375,10 +3375,8 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti if (truncateEntireTable) { // Drop the whole table stats after truncate the entire table Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable); - } else { - // Update the updated rows in table stats after truncate some partitions. - Env.getCurrentEnv().getAnalysisManager().updateUpdatedRows(updateRecords); } + Env.getCurrentEnv().getAnalysisManager().updateUpdatedRows(updateRecords); LOG.info("finished to truncate table {}, partitions: {}", tblRef.getName().toSql(), tblRef.getPartitionNames()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index c4e67d6bc1260f..51d2f4f44d0d55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionNormalizationAndOptimization; import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.expression.QueryColumnCollector; import org.apache.doris.nereids.rules.rewrite.AddDefaultLimit; import org.apache.doris.nereids.rules.rewrite.AdjustConjunctsReturnType; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; @@ -417,7 +418,8 @@ public class Rewriter extends AbstractBatchJobExecutor { new CollectFilterAboveConsumer(), new CollectProjectAboveConsumer() ) - ) + ), + topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new)) ); private static final List WHOLE_TREE_REWRITE_JOBS diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 10004953cdf195..f1a797f4e2bc75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -311,6 +311,7 @@ public enum RuleType { LEADING_JOIN(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), + COLLECT_COLUMNS(RuleTypeClass.REWRITE), // topn opts DEFER_MATERIALIZE_TOP_N_RESULT(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java new file mode 100644 index 00000000000000..ebf361de1d3a9a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/QueryColumnCollector.java @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.expression.QueryColumnCollector.CollectorContext; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.AnalysisManager; +import org.apache.doris.statistics.util.StatisticsUtil; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Used to collect query column. + */ +public class QueryColumnCollector extends DefaultPlanRewriter implements CustomRewriter { + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getSessionVariable().internalSession) { + return plan; + } + CollectorContext context = new CollectorContext(); + plan.accept(this, context); + if (StatisticsUtil.enableAutoAnalyze()) { + context.midPriority.removeAll(context.highPriority); + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + analysisManager.updateHighPriorityColumn(context.highPriority); + analysisManager.updateMidPriorityColumn(context.midPriority); + } + return plan; + } + + /** + * Context. + */ + public static class CollectorContext { + public Map projects = new HashMap<>(); + + public Set highPriority = new HashSet<>(); + + public Set midPriority = new HashSet<>(); + } + + @Override + public Plan visitLogicalProject(LogicalProject project, CollectorContext context) { + project.child().accept(this, context); + List projects = project.getOutputs(); + List slots = project.computeOutput(); + for (int i = 0; i < slots.size(); i++) { + context.projects.put(slots.get(i), projects.get(i)); + } + if (project.child() instanceof LogicalCatalogRelation + || project.child() instanceof LogicalFilter + && ((LogicalFilter) project.child()).child() instanceof LogicalCatalogRelation) { + Set allUsed = project.getExpressions() + .stream().flatMap(e -> e.>collect(n -> n instanceof SlotReference).stream()) + .collect(Collectors.toSet()); + LogicalCatalogRelation scan = project.child() instanceof LogicalCatalogRelation + ? (LogicalCatalogRelation) project.child() + : (LogicalCatalogRelation) project.child().child(0); + List outputOfScan = scan.getOutput(); + for (Slot slot : outputOfScan) { + if (!allUsed.contains(slot)) { + context.midPriority.remove(slot); + } + } + } + return project; + } + + @Override + public Plan visitLogicalJoin(LogicalJoin join, CollectorContext context) { + join.child(0).accept(this, context); + join.child(1).accept(this, context); + context.highPriority.addAll( + (join.isMarkJoin() ? join.getLeftConditionSlot() : join.getConditionSlot()) + .stream().flatMap(s -> backtrace(s, context).stream()) + .collect(Collectors.toSet()) + ); + return join; + } + + @Override + public Plan visitLogicalAggregate(LogicalAggregate aggregate, CollectorContext context) { + aggregate.child(0).accept(this, context); + context.highPriority.addAll(aggregate.getGroupByExpressions() + .stream() + .flatMap(e -> e.>collect(n -> n instanceof SlotReference).stream()) + .flatMap(s -> backtrace(s, context).stream()) + .collect(Collectors.toSet())); + return aggregate; + } + + @Override + public Plan visitLogicalHaving(LogicalHaving having, CollectorContext context) { + having.child(0).accept(this, context); + context.highPriority.addAll( + having.getExpressions().stream() + .flatMap(e -> e.>collect(n -> n instanceof SlotReference).stream()) + .flatMap(s -> backtrace(s, context).stream()) + .collect(Collectors.toSet())); + return having; + } + + @Override + public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, CollectorContext context) { + List slots = olapScan.getOutput(); + context.midPriority.addAll(slots); + return olapScan; + } + + @Override + public Plan visitLogicalFileScan(LogicalFileScan fileScan, CollectorContext context) { + List slots = fileScan.getOutput(); + context.midPriority.addAll(slots); + return fileScan; + } + + @Override + public Plan visitLogicalFilter(LogicalFilter filter, CollectorContext context) { + filter.child(0).accept(this, context); + context.highPriority.addAll(filter + .getExpressions() + .stream() + .flatMap(e -> e.>collect(n -> n instanceof SlotReference).stream()) + .flatMap(s -> backtrace(s, context).stream()) + .collect(Collectors.toSet())); + return filter; + } + + @Override + public Plan visitLogicalWindow(LogicalWindow window, CollectorContext context) { + window.child(0).accept(this, context); + context.highPriority.addAll(window + .getWindowExpressions() + .stream() + .flatMap(e -> e.>collect(n -> n instanceof SlotReference).stream()) + .flatMap(s -> backtrace(s, context).stream()) + .collect(Collectors.toSet())); + return window; + } + + private Set backtrace(Slot slot, CollectorContext context) { + return backtrace(slot, new HashSet<>(), context); + } + + private Set backtrace(Slot slot, Set path, CollectorContext context) { + if (path.contains(slot)) { + return Collections.emptySet(); + } + path.add(slot); + if (slot instanceof SlotReference) { + SlotReference slotReference = (SlotReference) slot; + Optional col = slotReference.getColumn(); + Optional table = slotReference.getTable(); + if (col.isPresent() && table.isPresent()) { + return Collections.singleton(slot); + } + } + NamedExpression namedExpression = context.projects.get(slot); + if (namedExpression == null) { + return Collections.emptySet(); + } + Set slotReferences + = namedExpression.>collect(n -> n instanceof SlotReference); + Set refCol = new HashSet<>(); + for (SlotReference slotReference : slotReferences) { + refCol.addAll(backtrace(slotReference, path, context)); + } + return refCol; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 579ad21c9ab1c5..433997035c4f74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -480,6 +480,8 @@ public class SessionVariable implements Serializable, Writable { public static final String FORCE_SAMPLE_ANALYZE = "force_sample_analyze"; + public static final String ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG = "enable_auto_analyze_internal_catalog"; + public static final String AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = "auto_analyze_table_width_threshold"; public static final String FASTER_FLOAT_CONVERT = "faster_float_convert"; @@ -1551,6 +1553,11 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { flag = VariableMgr.GLOBAL) public boolean forceSampleAnalyze = Config.force_sample_analyze; + @VariableMgr.VarAttr(name = ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG, + description = {"临时参数，收否自动收集所有内表", "Temp variable， enable to auto collect all OlapTable."}, + flag = VariableMgr.GLOBAL) + public boolean enableAutoAnalyzeInternalCatalog = true; + @VariableMgr.VarAttr(name = AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD, description = {"参与自动收集的最大表宽度，列数多于这个参数的表不参与自动收集", "Maximum table width to enable auto analyze, " diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index f56a9993459428..e6fbef298890b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -30,6 +30,7 @@ import org.apache.doris.analysis.ShowAnalyzeStmt; import org.apache.doris.analysis.ShowAnalyzeTaskStatus; import org.apache.doris.analysis.ShowAuthorStmt; +import org.apache.doris.analysis.ShowAutoAnalyzeJobsStmt; import org.apache.doris.analysis.ShowBackendsStmt; import org.apache.doris.analysis.ShowBackupStmt; import org.apache.doris.analysis.ShowBrokerStmt; @@ -213,6 +214,7 @@ import org.apache.doris.qe.help.HelpTopic; import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.AnalysisInfo; +import org.apache.doris.statistics.AutoAnalysisPendingJob; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.Histogram; import org.apache.doris.statistics.ResultRow; @@ -455,6 +457,8 @@ public ShowResultSet execute() throws AnalysisException { handleShowCreateCatalog(); } else if (stmt instanceof ShowAnalyzeStmt) { handleShowAnalyze(); + } else if (stmt instanceof ShowAutoAnalyzeJobsStmt) { + handleShowAutoAnalyzePendingJobs(); } else if (stmt instanceof ShowTabletsBelongStmt) { handleShowTabletsBelong(); } else if (stmt instanceof AdminCopyTabletStmt) { @@ -2855,6 +2859,7 @@ private void handleShowAnalyze() { java.time.ZoneId.systemDefault()); row.add(startTime.format(formatter)); row.add(endTime.format(formatter)); + row.add(analysisInfo.priority.name()); resultRows.add(row); } catch (Exception e) { LOG.warn("Failed to get analyze info for table {}.{}.{}, reason: {}", @@ -2865,6 +2870,35 @@ private void handleShowAnalyze() { resultSet = new ShowResultSet(showStmt.getMetaData(), resultRows); } + private void handleShowAutoAnalyzePendingJobs() { + ShowAutoAnalyzeJobsStmt showStmt = (ShowAutoAnalyzeJobsStmt) stmt; + List jobs = Env.getCurrentEnv().getAnalysisManager().showAutoPendingJobs(showStmt); + List> resultRows = Lists.newArrayList(); + for (AutoAnalysisPendingJob job : jobs) { + try { + List row = new ArrayList<>(); + CatalogIf> c = StatisticsUtil.findCatalog(job.catalogName); + row.add(c.getName()); + Optional> databaseIf = c.getDb(job.dbName); + row.add(databaseIf.isPresent() ? databaseIf.get().getFullName() : "DB may get deleted"); + if (databaseIf.isPresent()) { + Optional table = databaseIf.get().getTable(job.tableName); + row.add(table.isPresent() ? table.get().getName() : "Table may get deleted"); + } else { + row.add("DB may get deleted"); + } + row.add(job.getColumnNames()); + row.add(String.valueOf(job.priority)); + resultRows.add(row); + } catch (Exception e) { + LOG.warn("Failed to get pending jobs for table {}.{}.{}, reason: {}", + job.catalogName, job.dbName, job.tableName, e.getMessage()); + continue; + } + } + resultSet = new ShowResultSet(showStmt.getMetaData(), resultRows); + } + private void handleShowTabletsBelong() { ShowTabletsBelongStmt showStmt = (ShowTabletsBelongStmt) stmt; List> rows = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 6b60e01d1c02a5..c26712af0bd345 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -225,6 +225,7 @@ import org.apache.doris.thrift.TStreamLoadMultiTablePutResult; import org.apache.doris.thrift.TStreamLoadPutRequest; import org.apache.doris.thrift.TStreamLoadPutResult; +import org.apache.doris.thrift.TSyncQueryColumns; import org.apache.doris.thrift.TTableIndexQueryStats; import org.apache.doris.thrift.TTableMetadataNameIds; import org.apache.doris.thrift.TTableQueryStats; @@ -3775,4 +3776,11 @@ public TShowUserResult showUser(TShowUserRequest request) { result.setUserinfoList(userInfo); return result; } + + public TStatus syncQueryColumns(TSyncQueryColumns request) throws TException { + Env.getCurrentEnv().getAnalysisManager().mergeFollowerQueryColumns(request.highPriorityColumns, + request.midPriorityColumns); + return new TStatus(TStatusCode.OK); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index c167db2228d8cc..e0fd91d1100354 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -96,7 +96,7 @@ public enum ScheduleType { public final long tblId; // Pair - public final List> jobColumns; + public final Set> jobColumns; public final Set partitionNames; @@ -188,8 +188,11 @@ public enum ScheduleType { @SerializedName("endTime") public long endTime; - @SerializedName("emptyJob") - public final boolean emptyJob; + @SerializedName("rowCount") + public final long rowCount; + + @SerializedName("updateRows") + public final long updateRows; /** * * Used to store the newest partition version of tbl when creating this job. @@ -197,16 +200,21 @@ public enum ScheduleType { */ public final long tblUpdateTime; + @SerializedName("userInject") public final boolean userInject; + @SerializedName("priority") + public final JobPriority priority; + public AnalysisInfo(long jobId, long taskId, List taskIds, long catalogId, long dbId, long tblId, - List> jobColumns, Set partitionNames, String colName, Long indexId, + Set> jobColumns, Set partitionNames, String colName, Long indexId, JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType, int samplePercent, long sampleRows, int maxBucketNum, long periodTimeInMs, String message, long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType, boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition, boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull, - boolean usingSqlForPartitionColumn, long tblUpdateTime, boolean emptyJob, boolean userInject) { + boolean usingSqlForPartitionColumn, long tblUpdateTime, long rowCount, boolean userInject, + long updateRows, JobPriority priority) { this.jobId = jobId; this.taskId = taskId; this.taskIds = taskIds; @@ -242,8 +250,10 @@ public AnalysisInfo(long jobId, long taskId, List taskIds, long catalogId, this.forceFull = forceFull; this.usingSqlForPartitionColumn = usingSqlForPartitionColumn; this.tblUpdateTime = tblUpdateTime; - this.emptyJob = emptyJob; + this.rowCount = rowCount; this.userInject = userInject; + this.updateRows = updateRows; + this.priority = priority; } @Override @@ -285,7 +295,10 @@ public String toString() { } sj.add("forceFull: " + forceFull); sj.add("usingSqlForPartitionColumn: " + usingSqlForPartitionColumn); - sj.add("emptyJob: " + emptyJob); + sj.add("rowCount: " + rowCount); + sj.add("userInject: " + userInject); + sj.add("updateRows: " + updateRows); + sj.add("priority: " + priority.name()); return sj.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index 00cf9f7b1bc560..83da112d33a366 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -36,7 +36,7 @@ public class AnalysisInfoBuilder { private long catalogId; private long dbId; private long tblId; - private List> jobColumns; + private Set> jobColumns; private Set partitionNames; private String colName; private long indexId = -1L; @@ -62,8 +62,10 @@ public class AnalysisInfoBuilder { private boolean forceFull; private boolean usingSqlForPartitionColumn; private long tblUpdateTime; - private boolean emptyJob; + private long rowCount; private boolean userInject; + private long updateRows; + private JobPriority priority; public AnalysisInfoBuilder() { } @@ -101,8 +103,10 @@ public AnalysisInfoBuilder(AnalysisInfo info) { forceFull = info.forceFull; usingSqlForPartitionColumn = info.usingSqlForPartitionColumn; tblUpdateTime = info.tblUpdateTime; - emptyJob = info.emptyJob; + rowCount = info.rowCount; userInject = info.userInject; + updateRows = info.updateRows; + priority = info.priority; } public AnalysisInfoBuilder setJobId(long jobId) { @@ -135,7 +139,7 @@ public AnalysisInfoBuilder setTblId(long tblId) { return this; } - public AnalysisInfoBuilder setJobColumns(List> jobColumns) { + public AnalysisInfoBuilder setJobColumns(Set> jobColumns) { this.jobColumns = jobColumns; return this; } @@ -265,8 +269,8 @@ public AnalysisInfoBuilder setTblUpdateTime(long tblUpdateTime) { return this; } - public AnalysisInfoBuilder setEmptyJob(boolean emptyJob) { - this.emptyJob = emptyJob; + public AnalysisInfoBuilder setRowCount(long rowCount) { + this.rowCount = rowCount; return this; } @@ -275,12 +279,23 @@ public AnalysisInfoBuilder setUserInject(boolean userInject) { return this; } + public AnalysisInfoBuilder setUpdateRows(long updateRows) { + this.updateRows = updateRows; + return this; + } + + public AnalysisInfoBuilder setPriority(JobPriority priority) { + this.priority = priority; + return this; + } + public AnalysisInfo build() { return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId, tblId, jobColumns, partitionNames, colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent, sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType, externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount, - cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, emptyJob, userInject); + cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, rowCount, userInject, updateRows, + priority); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java index 5fd5e43be53f2b..0bc0a437898c71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java @@ -84,14 +84,12 @@ public synchronized void rowCountDone(BaseAnalysisTask task) { protected void markOneTaskDone() { if (queryingTask.isEmpty()) { try { - writeBuf(); - updateTaskState(AnalysisState.FINISHED, "Cost time in sec: " - + (System.currentTimeMillis() - start) / 1000); + flushBuffer(); } finally { deregisterJob(); } } else if (buf.size() >= StatisticsUtil.getInsertMergeCount()) { - writeBuf(); + flushBuffer(); } } @@ -115,7 +113,7 @@ public void updateTaskState(AnalysisState state, String msg) { } } - protected void writeBuf() { + protected void flushBuffer() { if (killed) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index a2b30cce2ac5f4..03314fe7748a13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -25,6 +25,7 @@ import org.apache.doris.analysis.DropStatsStmt; import org.apache.doris.analysis.KillAnalysisJobStmt; import org.apache.doris.analysis.ShowAnalyzeStmt; +import org.apache.doris.analysis.ShowAutoAnalyzeJobsStmt; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; @@ -48,6 +49,8 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.persist.AnalyzeDeletionLog; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; @@ -63,6 +66,7 @@ import org.apache.doris.system.Frontend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TInvalidateFollowerStatsCacheRequest; +import org.apache.doris.thrift.TQueryColumn; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -83,15 +87,19 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.NavigableMap; +import java.util.Objects; import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.StringJoiner; import java.util.TreeMap; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @@ -104,6 +112,14 @@ public class AnalysisManager implements Writable { private static final Logger LOG = LogManager.getLogger(AnalysisManager.class); + public static final int COLUMN_QUEUE_SIZE = 1000; + public final Queue highPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE); + public final Queue midPriorityColumns = new ArrayBlockingQueue<>(COLUMN_QUEUE_SIZE); + // Map>> + public final Map>> highPriorityJobs = new LinkedHashMap<>(); + public final Map>> midPriorityJobs = new LinkedHashMap<>(); + public final Map>> lowPriorityJobs = new LinkedHashMap<>(); + // Tracking running manually submitted async tasks, keep in mem only protected final ConcurrentMap> analysisJobIdToTaskMap = new ConcurrentHashMap<>(); @@ -154,13 +170,8 @@ public void createAnalyze(AnalyzeStmt analyzeStmt, boolean proxy) throws DdlExce } } - public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throws DdlException, AnalysisException { + public void createAnalysisJobs(AnalyzeDBStmt analyzeDBStmt, boolean proxy) throws AnalysisException { DatabaseIf db = analyzeDBStmt.getDb(); - // Using auto analyzer if user specifies. - if (analyzeDBStmt.getAnalyzeProperties().getProperties().containsKey("use.auto.analyzer")) { - Env.getCurrentEnv().getStatisticsAutoCollector().analyzeDb(db); - return; - } List analysisInfos = buildAnalysisInfosForDB(db, analyzeDBStmt.getAnalyzeProperties()); if (!analyzeDBStmt.isSync()) { sendJobId(analysisInfos, proxy); @@ -178,9 +189,8 @@ public List buildAnalysisInfosForDB(DatabaseIf db, Analyz if (table instanceof View) { continue; } - TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(), - table.getName()); - // columnNames null means to add all visitable columns. + TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(), table.getName()); + // columnNames null means to add all visible columns. // Will get all the visible columns in analyzeTblStmt.check() AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeProperties, tableName, null, db.getId(), table); @@ -208,6 +218,13 @@ public List buildAnalysisInfosForDB(DatabaseIf db, Analyz // Each analyze stmt corresponding to an analysis job. public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlException { + // Using auto analyzer if user specifies. + if ("true".equalsIgnoreCase(stmt.getAnalyzeProperties().getProperties().get("use.auto.analyzer"))) { + Env.getCurrentEnv().getStatisticsAutoCollector() + .processOneJob(stmt.getTable(), + stmt.getTable().getColumnIndexPairs(stmt.getColumnNames()), JobPriority.HIGH); + return; + } AnalysisInfo jobInfo = buildAndAssignJob(stmt); if (jobInfo == null) { return; @@ -219,8 +236,9 @@ public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlExce @VisibleForTesting protected AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws DdlException { AnalysisInfo jobInfo = buildAnalysisJobInfo(stmt); - if (jobInfo.jobColumns.isEmpty()) { + if (jobInfo.jobColumns == null || jobInfo.jobColumns.isEmpty()) { // No statistics need to be collected or updated + LOG.info("Job columns are empty, skip analyze table {}", stmt.getTblName().toString()); return null; } // Only OlapTable and Hive HMSExternalTable support sample analyze. @@ -295,7 +313,7 @@ private void sendJobId(List analysisInfos, boolean proxy) { // Make sure colName of job has all the column as this AnalyzeStmt specified, no matter whether it will be analyzed // or not. @VisibleForTesting - public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlException { + public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) { AnalysisInfoBuilder infoBuilder = new AnalysisInfoBuilder(); long jobId = Env.getCurrentEnv().getNextId(); TableIf table = stmt.getTable(); @@ -329,7 +347,6 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio infoBuilder.setAnalysisMode(analysisMode); infoBuilder.setAnalysisMethod(analysisMethod); infoBuilder.setScheduleType(scheduleType); - infoBuilder.setLastExecTimeInMs(0); infoBuilder.setCronExpression(cronExpression); infoBuilder.setForceFull(stmt.forceFull()); infoBuilder.setUsingSqlForPartitionColumn(stmt.usingSqlForPartitionColumn()); @@ -346,7 +363,7 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio long periodTimeInMs = stmt.getPeriodTimeInMs(); infoBuilder.setPeriodTimeInMs(periodTimeInMs); - List> jobColumns = table.getColumnIndexPairs(columnNames); + Set> jobColumns = table.getColumnIndexPairs(columnNames); infoBuilder.setJobColumns(jobColumns); StringJoiner stringJoiner = new StringJoiner(",", "[", "]"); for (Pair pair : jobColumns) { @@ -355,8 +372,10 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio infoBuilder.setColName(stringJoiner.toString()); infoBuilder.setTaskIds(Lists.newArrayList()); infoBuilder.setTblUpdateTime(table.getUpdateTime()); - infoBuilder.setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0 - && analysisMethod.equals(AnalysisMethod.SAMPLE)); + infoBuilder.setRowCount(StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0 : table.getRowCount()); + TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId()); + infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get()); + infoBuilder.setPriority(JobPriority.MANUAL); return infoBuilder.build(); } @@ -372,7 +391,7 @@ public void recordAnalysisJob(AnalysisInfo jobInfo) { public void createTaskForEachColumns(AnalysisInfo jobInfo, Map analysisTasks, boolean isSync) throws DdlException { - List> jobColumns = jobInfo.jobColumns; + Set> jobColumns = jobInfo.jobColumns; TableIf table = jobInfo.getTable(); for (Pair pair : jobColumns) { AnalysisInfoBuilder colTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo); @@ -505,7 +524,7 @@ public void updateTableStats(AnalysisInfo jobInfo) { } TableStatsMeta tableStats = findTableStatsStatus(tbl.getId()); if (tableStats == null) { - updateTableStatsStatus(new TableStatsMeta(jobInfo.emptyJob ? 0 : tbl.getRowCount(), jobInfo, tbl)); + updateTableStatsStatus(new TableStatsMeta(jobInfo.rowCount, jobInfo, tbl)); } else { tableStats.update(jobInfo, tbl); logCreateTableStats(tableStats); @@ -529,6 +548,39 @@ public void updateTableStatsForAlterStats(AnalysisInfo jobInfo, TableIf tbl) { } } + public List showAutoPendingJobs(ShowAutoAnalyzeJobsStmt stmt) { + TableName tblName = stmt.getTableName(); + String priority = stmt.getPriority(); + List result = Lists.newArrayList(); + if (priority == null || priority.isEmpty()) { + result.addAll(getPendingJobs(highPriorityJobs, JobPriority.HIGH, tblName)); + result.addAll(getPendingJobs(midPriorityJobs, JobPriority.MID, tblName)); + result.addAll(getPendingJobs(lowPriorityJobs, JobPriority.LOW, tblName)); + } else if (priority.equals(JobPriority.HIGH.name())) { + result.addAll(getPendingJobs(highPriorityJobs, JobPriority.HIGH, tblName)); + } else if (priority.equals(JobPriority.MID.name())) { + result.addAll(getPendingJobs(midPriorityJobs, JobPriority.MID, tblName)); + } else if (priority.equals(JobPriority.LOW.name())) { + result.addAll(getPendingJobs(lowPriorityJobs, JobPriority.LOW, tblName)); + } + return result; + } + + protected List getPendingJobs(Map>> jobMap, + JobPriority priority, TableName tblName) { + List result = Lists.newArrayList(); + synchronized (jobMap) { + for (Entry>> entry : jobMap.entrySet()) { + TableName table = entry.getKey(); + if (tblName == null || tblName.equals(table)) { + result.add(new AutoAnalysisPendingJob(table.getCtl(), + table.getDb(), table.getTbl(), entry.getValue(), priority)); + } + } + } + return result; + } + public List showAnalysisJob(ShowAnalyzeStmt stmt) { return findShowAnalyzeResult(stmt); } @@ -555,7 +607,7 @@ private List findShowAnalyzeResult(ShowAnalyzeStmt stmt) { public String getJobProgress(long jobId) { List tasks = findTasksByTaskIds(jobId); - if (tasks == null) { + if (tasks == null || tasks.isEmpty()) { return "N/A"; } int finished = 0; @@ -674,6 +726,7 @@ public void invalidateLocalStats(long catalogId, long dbId, long tableId, } tableStats.updatedTime = 0; tableStats.userInjected = false; + tableStats.rowCount = table.getRowCount(); } public void invalidateRemoteStats(long catalogId, long dbId, long tableId, @@ -751,7 +804,7 @@ private BaseAnalysisTask createTask(AnalysisInfo analysisInfo) throws DdlExcepti analysisInfo.dbId, analysisInfo.tblId); return table.createAnalysisTask(analysisInfo); } catch (Throwable t) { - LOG.warn("Failed to find table", t); + LOG.warn("Failed to create task.", t); throw new DdlException("Failed to create task", t); } } @@ -854,7 +907,7 @@ public List findTasks(long jobId) { public List findTasksByTaskIds(long jobId) { AnalysisInfo jobInfo = analysisJobInfoMap.get(jobId); if (jobInfo != null && jobInfo.taskIds != null) { - return jobInfo.taskIds.stream().map(analysisTaskInfoMap::get).filter(i -> i != null) + return jobInfo.taskIds.stream().map(analysisTaskInfoMap::get).filter(Objects::nonNull) .collect(Collectors.toList()); } return null; @@ -871,7 +924,7 @@ public void removeAll(List analysisInfos) { public void dropAnalyzeJob(DropAnalyzeJobStmt analyzeJobStmt) throws DdlException { AnalysisInfo jobInfo = analysisJobInfoMap.get(analyzeJobStmt.getJobId()); if (jobInfo == null) { - throw new DdlException(String.format("Analyze job [%d] not exists", jobInfo.jobId)); + throw new DdlException(String.format("Analyze job [%d] not exists", analyzeJobStmt.getJobId())); } checkPriv(jobInfo); long jobId = analyzeJobStmt.getJobId(); @@ -911,15 +964,12 @@ public static boolean needAbandon(AnalysisInfo analysisInfo) { if (analysisInfo == null) { return true; } - if (analysisInfo.scheduleType == null || analysisInfo.scheduleType == null || analysisInfo.jobType == null) { - return true; - } - if ((AnalysisState.PENDING.equals(analysisInfo.state) || AnalysisState.RUNNING.equals(analysisInfo.state)) - && ScheduleType.ONCE.equals(analysisInfo.scheduleType) - && JobType.MANUAL.equals(analysisInfo.jobType)) { + if (analysisInfo.scheduleType == null || analysisInfo.jobType == null) { return true; } - return false; + return (AnalysisState.PENDING.equals(analysisInfo.state) || AnalysisState.RUNNING.equals(analysisInfo.state)) + && ScheduleType.ONCE.equals(analysisInfo.scheduleType) + && JobType.MANUAL.equals(analysisInfo.jobType); } private static void readIdToTblStats(DataInput in, Map map) throws IOException { @@ -1075,17 +1125,66 @@ public void removeJob(long id) { /** * Only OlapTable and Hive HMSExternalTable can sample for now. - * @param table + * @param table Table to check * @return Return true if the given table can do sample analyze. False otherwise. */ public boolean canSample(TableIf table) { if (table instanceof OlapTable) { return true; } - if (table instanceof HMSExternalTable - && ((HMSExternalTable) table).getDlaType().equals(HMSExternalTable.DLAType.HIVE)) { - return true; + return table instanceof HMSExternalTable + && ((HMSExternalTable) table).getDlaType().equals(HMSExternalTable.DLAType.HIVE); + } + + + public void updateHighPriorityColumn(Set slotReferences) { + updateColumn(slotReferences, highPriorityColumns); + } + + public void updateMidPriorityColumn(Collection slotReferences) { + updateColumn(slotReferences, midPriorityColumns); + } + + protected void updateColumn(Collection slotReferences, Queue queue) { + for (Slot s : slotReferences) { + if (!(s instanceof SlotReference)) { + return; + } + Optional optionalColumn = ((SlotReference) s).getColumn(); + Optional optionalTable = ((SlotReference) s).getTable(); + if (optionalColumn.isPresent() && optionalTable.isPresent() + && !StatisticsUtil.isUnsupportedType(optionalColumn.get().getType())) { + TableIf table = optionalTable.get(); + DatabaseIf database = table.getDatabase(); + if (database != null) { + CatalogIf catalog = database.getCatalog(); + if (catalog != null) { + queue.offer(new QueryColumn(catalog.getId(), database.getId(), + table.getId(), optionalColumn.get().getName())); + if (LOG.isDebugEnabled()) { + LOG.debug("Offer column " + table.getName() + "(" + table.getId() + ")." + + optionalColumn.get().getName()); + } + } + } + } + } + } + + public void mergeFollowerQueryColumns(Collection highColumns, + Collection midColumns) { + LOG.info("Received {} high columns and {} mid columns", highColumns.size(), midColumns.size()); + for (TQueryColumn c : highColumns) { + if (!highPriorityColumns.offer(new QueryColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId), + Long.parseLong(c.tblId), c.colName))) { + break; + } + } + for (TQueryColumn c : midColumns) { + if (!midPriorityColumns.offer(new QueryColumn(Long.parseLong(c.catalogId), Long.parseLong(c.dbId), + Long.parseLong(c.tblId), c.colName))) { + break; + } } - return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java index 3bdccaca047954..d787794534a7c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java @@ -27,6 +27,7 @@ import java.util.Comparator; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -52,7 +53,7 @@ public AnalysisTaskExecutor(int simultaneouslyRunningTaskNum, int taskQueueSize) simultaneouslyRunningTaskNum, simultaneouslyRunningTaskNum, 0, TimeUnit.DAYS, new LinkedBlockingQueue<>(taskQueueSize), - new BlockedPolicy("Analysis Job Executor", Integer.MAX_VALUE), + new BlockedPolicy("Analysis Job Executor Block Policy", Integer.MAX_VALUE), "Analysis Job Executor", true); cancelExpiredTask(); } else { @@ -88,9 +89,9 @@ protected void tryToCancel() { } } - public void submitTask(BaseAnalysisTask task) { + public Future submitTask(BaseAnalysisTask task) { AnalysisTaskWrapper taskWrapper = new AnalysisTaskWrapper(this, task); - executors.submit(taskWrapper); + return executors.submit(taskWrapper); } public void putJob(AnalysisTaskWrapper wrapper) throws Exception { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java new file mode 100644 index 00000000000000..e349e4fcb3f2e8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AutoAnalysisPendingJob.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.common.Pair; + +import java.util.Set; +import java.util.StringJoiner; + +public class AutoAnalysisPendingJob { + + public final String catalogName; + public final String dbName; + public final String tableName; + public final Set> columns; + public final JobPriority priority; + + public AutoAnalysisPendingJob(String catalogName, String dbName, String tableName, + Set> columns, JobPriority priority) { + this.catalogName = catalogName; + this.dbName = dbName; + this.tableName = tableName; + this.columns = columns; + this.priority = priority; + } + + public String getColumnNames() { + if (columns == null) { + return ""; + } + StringJoiner stringJoiner = new StringJoiner(","); + for (Pair col : columns) { + stringJoiner.add(col.toString()); + } + return stringJoiner.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index f871e8761a5e55..d35e45987b8501 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -39,7 +39,6 @@ import java.text.MessageFormat; import java.util.Collections; -import java.util.concurrent.TimeUnit; public abstract class BaseAnalysisTask { @@ -48,7 +47,7 @@ public abstract class BaseAnalysisTask { public static final long LIMIT_SIZE = 1024 * 1024 * 1024; // 1GB public static final double LIMIT_FACTOR = 1.2; - protected static final String COLLECT_COL_STATISTICS = + protected static final String FULL_ANALYZE_TEMPLATE = "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, " + " ${catalogId} AS `catalog_id`, " + " ${dbId} AS `db_id`, " @@ -194,9 +193,9 @@ protected void init(AnalysisInfo info) { } } - public void execute() { + public void execute() throws Exception { prepareExecution(); - executeWithRetry(); + doExecute(); afterExecution(); } @@ -204,29 +203,6 @@ protected void prepareExecution() { setTaskStateToRunning(); } - protected void executeWithRetry() { - int retriedTimes = 0; - while (retriedTimes < StatisticConstants.ANALYZE_TASK_RETRY_TIMES) { - if (killed) { - break; - } - try { - doExecute(); - break; - } catch (Throwable t) { - if (killed) { - throw new RuntimeException(t); - } - LOG.warn("Failed to execute analysis task, retried times: {}", retriedTimes++, t); - if (retriedTimes >= StatisticConstants.ANALYZE_TASK_RETRY_TIMES) { - job.taskFailed(this, t.getMessage()); - throw new RuntimeException(t); - } - StatisticsUtil.sleep(TimeUnit.SECONDS.toMillis(2 ^ retriedTimes) * 10); - } - } - } - public abstract void doExecute() throws Exception; protected void afterExecution() {} @@ -284,9 +260,8 @@ protected String getNdvFunction(String totalRows) { // (https://github.com/postgres/postgres/blob/master/src/backend/commands/analyze.c) // (http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.93.8637&rep=rep1&type=pdf) // sample_row * count_distinct / ( sample_row - once_count + once_count * sample_row / total_row) - String fn = MessageFormat.format("{0} * {1} / ({0} - {2} + {2} * {0} / {3})", sampleRows, + return MessageFormat.format("{0} * {1} / ({0} - {2} + {2} * {0} / {3})", sampleRows, countDistinct, onceCount, totalRows); - return fn; } // Max value is not accurate while sample, so set it to NULL to avoid optimizer generate bad plan. @@ -336,6 +311,9 @@ protected void runQuery(String sql) { Env.getCurrentEnv().getStatisticsCache().syncColStats(colStatsData); queryId = DebugUtil.printId(stmtExecutor.getContext().queryId()); job.appendBuf(this, Collections.singletonList(colStatsData)); + } catch (Exception e) { + LOG.warn("Failed to execute sql {}", sql); + throw e; } finally { if (LOG.isDebugEnabled()) { LOG.debug("End cost time in millisec: " + (System.currentTimeMillis() - startTime) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java index 445641b2505610..7e317d67bd740f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java @@ -43,16 +43,20 @@ public class ColStatsMeta { @SerializedName("trigger") public JobType jobType; - public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, - AnalysisType analysisType, JobType jobType, long queriedTimes) { + @SerializedName("updatedRows") + public long updatedRows; + + @SerializedName("rowCount") + public long rowCount; + + public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod, AnalysisType analysisType, JobType jobType, + long queriedTimes, long rowCount, long updatedRows) { this.updatedTime = updatedTime; this.analysisMethod = analysisMethod; this.analysisType = analysisType; this.jobType = jobType; this.queriedTimes.addAndGet(queriedTimes); - } - - public void clear() { - updatedTime = 0; + this.updatedRows = updatedRows; + this.rowCount = rowCount; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java index 287941be526635..7d3c9af254800d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java @@ -73,6 +73,8 @@ protected void setTable(ExternalTable table) { */ private void getTableStats() { Map params = buildStatsParams(null); + Pair sampleInfo = getSampleInfo(); + params.put("scaleFactor", String.valueOf(sampleInfo.first)); List columnResult = StatisticsUtil.execStatisticQuery(new StringSubstitutor(params) .replace(ANALYZE_TABLE_COUNT_TEMPLATE)); @@ -98,7 +100,7 @@ protected void getColumnStats() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Will do full collection for column {}", col.getName()); } - sb.append(COLLECT_COL_STATISTICS); + sb.append(FULL_ANALYZE_TEMPLATE); } else { // Do sample analyze if (LOG.isDebugEnabled()) { @@ -254,9 +256,6 @@ protected boolean needLimit(long sizeToRead, double factor) { } target = columnSize * tableSample.getSampleValue(); } - if (sizeToRead > LIMIT_SIZE && sizeToRead > target * LIMIT_FACTOR) { - return true; - } - return false; + return sizeToRead > LIMIT_SIZE && sizeToRead > target * LIMIT_FACTOR; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java new file mode 100644 index 00000000000000..0e66c7f8a75be7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/FollowerColumnSender.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.MasterDaemon; +import org.apache.doris.ha.FrontendNodeType; +import org.apache.doris.statistics.util.StatisticsUtil; +import org.apache.doris.system.Frontend; +import org.apache.doris.thrift.FrontendService; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TQueryColumn; +import org.apache.doris.thrift.TSyncQueryColumns; + +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Queue; +import java.util.Set; + +public class FollowerColumnSender extends MasterDaemon { + + private static final Logger LOG = LogManager.getLogger(FollowerColumnSender.class); + + public static final long INTERVAL = 60000; + + public FollowerColumnSender() { + super("Follower Column Sender", INTERVAL); + } + + @Override + protected void runAfterCatalogReady() { + if (!StatisticsUtil.enableAutoAnalyze()) { + return; + } + if (Env.getCurrentEnv().isMaster()) { + return; + } + if (Env.isCheckpointThread()) { + return; + } + send(); + } + + protected void send() { + if (Env.getCurrentEnv().isMaster()) { + return; + } + Env currentEnv = Env.getCurrentEnv(); + AnalysisManager analysisManager = currentEnv.getAnalysisManager(); + if (analysisManager.highPriorityColumns.isEmpty() && analysisManager.midPriorityColumns.isEmpty()) { + return; + } + Set highs = getNeedAnalyzeColumns(analysisManager.highPriorityColumns); + Set mids = getNeedAnalyzeColumns(analysisManager.midPriorityColumns); + mids.removeAll(highs); + TSyncQueryColumns queryColumns = new TSyncQueryColumns(); + queryColumns.highPriorityColumns = new ArrayList<>(highs); + queryColumns.midPriorityColumns = new ArrayList<>(mids); + Frontend master = null; + try { + InetSocketAddress masterAddress = currentEnv.getHaProtocol().getLeader(); + for (Frontend fe : currentEnv.getFrontends(FrontendNodeType.FOLLOWER)) { + InetSocketAddress socketAddress = new InetSocketAddress(fe.getHost(), fe.getEditLogPort()); + if (socketAddress.equals(masterAddress)) { + master = fe; + break; + } + } + } catch (Exception e) { + LOG.warn("Failed to find master FE.", e); + return; + } + + if (master == null) { + LOG.warn("No master found in cluster."); + return; + } + TNetworkAddress address = new TNetworkAddress(master.getHost(), master.getRpcPort()); + FrontendService.Client client = null; + try { + client = ClientPool.frontendPool.borrowObject(address); + client.syncQueryColumns(queryColumns); + LOG.info("Send {} high priority columns and {} mid priority columns to master.", + highs.size(), mids.size()); + } catch (Throwable t) { + LOG.warn("Failed to sync stats to master: {}", address, t); + } finally { + if (client != null) { + ClientPool.frontendPool.returnObject(address, client); + } + } + } + + protected Set getNeedAnalyzeColumns(Queue columnQueue) { + Set ret = Sets.newHashSet(); + TableIf table; + int size = columnQueue.size(); + for (int i = 0; i < size; i++) { + QueryColumn column = columnQueue.poll(); + if (column == null) { + continue; + } + try { + table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId); + } catch (Exception e) { + LOG.warn("Failed to find table for column {}", column.colName, e); + continue; + } + if (StatisticsUtil.isUnsupportedType(table.getColumn(column.colName).getType())) { + continue; + } + Set> columnIndexPairs = table.getColumnIndexPairs( + Collections.singleton(column.colName)); + for (Pair pair : columnIndexPairs) { + if (StatisticsUtil.needAnalyzeColumn(table, pair)) { + ret.add(column.toThrift()); + break; + } + } + } + return ret; + } + + protected List convertSetToList(Set set) { + return new ArrayList<>(set); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index 60da8f4d2a0803..26ef561ddf6a27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -79,11 +79,6 @@ public void doExecute() throws Exception { tbl.getDatabase().getCatalog().getId(), tbl.getDatabase().getId(), tbl.getId(), -1, col.getName()); } - @Override - protected void afterExecution() { - // DO NOTHING - } - private String getSampleRateFunction() { if (info.analysisMethod == AnalysisMethod.FULL) { return "0"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java new file mode 100644 index 00000000000000..c3656b929279e6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JobPriority.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +public enum JobPriority { + HIGH, + MID, + LOW, + MANUAL; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index 60bfcab6157377..ce7982d4f1ad58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -37,7 +37,6 @@ import java.security.SecureRandom; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -65,11 +64,11 @@ public OlapAnalysisTask(AnalysisInfo info) { } public void doExecute() throws Exception { - List> columnList = info.jobColumns; - if (StatisticsUtil.isEmptyTable(tbl, info.analysisMethod) || columnList == null || columnList.isEmpty()) { + // For empty table, write empty result directly, no need to run SQL to collect stats. + if (info.rowCount == 0 && tableSample != null) { StatsId statsId = new StatsId(concatColumnStatsId(), info.catalogId, info.dbId, info.tblId, info.indexId, info.colName, null); - job.appendBuf(this, Arrays.asList(new ColStatsData(statsId))); + job.appendBuf(this, Collections.singletonList(new ColStatsData(statsId))); return; } if (tableSample != null) { @@ -84,7 +83,7 @@ public void doExecute() throws Exception { * 2. estimate partition stats * 3. insert col stats and partition stats */ - protected void doSample() throws Exception { + protected void doSample() { if (LOG.isDebugEnabled()) { LOG.debug("Will do sample collection for column {}", col.getName()); } @@ -209,7 +208,7 @@ protected ResultRow collectBasicStat(AutoCloseConnectContext context) { * 2. insert partition in batch * 3. calculate column stats based on partition stats */ - protected void doFull() throws Exception { + protected void doFull() { if (LOG.isDebugEnabled()) { LOG.debug("Will do full collection for column {}", col.getName()); } @@ -228,8 +227,7 @@ protected void doFull() throws Exception { params.put("tblName", String.valueOf(tbl.getName())); params.put("index", getIndex()); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); - String collectColStats = stringSubstitutor.replace(COLLECT_COL_STATISTICS); - runQuery(collectColStats); + runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE)); } protected String getIndex() { @@ -316,10 +314,7 @@ protected boolean needLimit() { return false; } // Partition column need to scan tablets from all partitions. - if (tbl.isPartitionColumn(col.getName())) { - return false; - } - return true; + return !tbl.isPartitionColumn(col.getName()); } /** @@ -382,12 +377,6 @@ protected boolean isSingleUniqueKey() { } protected String concatColumnStatsId() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append(info.tblId); - stringBuilder.append("-"); - stringBuilder.append(info.indexId); - stringBuilder.append("-"); - stringBuilder.append(info.colName); - return stringBuilder.toString(); + return info.tblId + "-" + info.indexId + "-" + info.colName; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java new file mode 100644 index 00000000000000..df91ea7f4c0582 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/QueryColumn.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.thrift.TQueryColumn; + +import java.util.Objects; + +public class QueryColumn { + + public final long catalogId; + public final long dbId; + public final long tblId; + public final String colName; + + public QueryColumn(long catalogId, long dbId, long tblId, String colName) { + this.catalogId = catalogId; + this.dbId = dbId; + this.tblId = tblId; + this.colName = colName; + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, dbId, tblId, colName); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof QueryColumn)) { + return false; + } + QueryColumn otherCriticalColumn = (QueryColumn) other; + return this.catalogId == otherCriticalColumn.catalogId + && this.dbId == otherCriticalColumn.dbId + && this.tblId == otherCriticalColumn.tblId + && this.colName.equals(otherCriticalColumn.colName); + } + + public TQueryColumn toThrift() { + TQueryColumn tQueryColumn = new TQueryColumn(); + tQueryColumn.catalogId = String.valueOf(catalogId); + tQueryColumn.dbId = String.valueOf(dbId); + tQueryColumn.tblId = String.valueOf(tblId); + tQueryColumn.colName = colName; + return tQueryColumn; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java index 74c7bd7c9db127..a5bd18946e8376 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java @@ -64,8 +64,6 @@ public class StatisticConstants { public static List SYSTEM_DBS = new ArrayList<>(); - public static int ANALYZE_TASK_RETRY_TIMES = 5; - public static final String DB_NAME = FeConstants.INTERNAL_DB_NAME; public static final String FULL_QUALIFIED_STATS_TBL_NAME = InternalCatalog.INTERNAL_CATALOG_NAME @@ -95,7 +93,7 @@ public class StatisticConstants { public static final int ANALYZE_TIMEOUT_IN_SEC = 43200; - public static final int TASK_QUEUE_CAP = 10; + public static final int TASK_QUEUE_CAP = 1; public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 100; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index 9ca971845b7e64..479610ccea25ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -17,17 +17,16 @@ package org.apache.doris.statistics; +import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; +import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.JobType; @@ -40,200 +39,199 @@ import java.time.LocalTime; import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.StringJoiner; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class StatisticsAutoCollector extends StatisticsCollector { +public class StatisticsAutoCollector extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(StatisticsAutoCollector.class); + protected final AnalysisTaskExecutor analysisTaskExecutor; + public StatisticsAutoCollector() { - super("Automatic Analyzer", - TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes), - new AnalysisTaskExecutor(Config.auto_analyze_simultaneously_running_task_num, - StatisticConstants.TASK_QUEUE_CAP)); + super("Automatic Analyzer", TimeUnit.MINUTES.toMillis(Config.auto_check_statistics_in_minutes)); + this.analysisTaskExecutor = new AnalysisTaskExecutor(Config.auto_analyze_simultaneously_running_task_num, + StatisticConstants.TASK_QUEUE_CAP); } @Override - protected void collect() { - if (canCollect()) { - analyzeAll(); + protected void runAfterCatalogReady() { + if (!Env.getCurrentEnv().isMaster()) { + return; } + if (!StatisticsUtil.statsTblAvailable()) { + LOG.info("Stats table not available, skip"); + return; + } + if (Env.isCheckpointThread()) { + return; + } + collect(); } - protected boolean canCollect() { - return StatisticsUtil.enableAutoAnalyze() - && StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId())); - } - - protected void analyzeAll() { - List catalogs = getCatalogsInOrder(); - for (CatalogIf ctl : catalogs) { - if (!canCollect()) { - analysisTaskExecutor.clear(); + protected void collect() { + while (canCollect()) { + Pair>>, JobPriority> job = getJob(); + if (job == null) { + // No more job to process, break and sleep. break; } - if (!ctl.enableAutoAnalyze()) { - continue; - } - List dbs = getDatabasesInOrder(ctl); - for (DatabaseIf databaseIf : dbs) { - if (!canCollect()) { - analysisTaskExecutor.clear(); - break; - } - if (StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) { - continue; - } - try { - analyzeDb(databaseIf); - } catch (Throwable t) { - LOG.warn("Failed to analyze database {}.{}", ctl.getName(), databaseIf.getFullName(), t); + try { + TableName tblName = job.first.getKey(); + TableIf table = StatisticsUtil.findTable(tblName.getCtl(), tblName.getDb(), tblName.getTbl()); + if (!supportAutoAnalyze(table)) { continue; } + processOneJob(table, job.first.getValue(), job.second); + } catch (Exception e) { + LOG.warn("Failed to analyze table {} with columns [{}]", job.first.getKey().getTbl(), + job.first.getValue().stream().map(Pair::toString).collect(Collectors.joining(",")), e); } } } - public List getCatalogsInOrder() { - return Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog().stream() - .sorted((c1, c2) -> (int) (c1.getId() - c2.getId())).collect(Collectors.toList()); - } - - public List> getDatabasesInOrder(CatalogIf catalog) { - return catalog.getAllDbs().stream() - .sorted((d1, d2) -> (int) (d1.getId() - d2.getId())).collect(Collectors.toList()); + protected boolean canCollect() { + return StatisticsUtil.enableAutoAnalyze() + && StatisticsUtil.inAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId())); } - public List getTablesInOrder(DatabaseIf db) { - return db.getTables().stream() - .sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList()); + protected Pair>>, JobPriority> getJob() { + AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); + Optional>>> job = fetchJobFromMap(manager.highPriorityJobs); + if (job.isPresent()) { + return Pair.of(job.get(), JobPriority.HIGH); + } + job = fetchJobFromMap(manager.midPriorityJobs); + if (job.isPresent()) { + return Pair.of(job.get(), JobPriority.MID); + } + job = fetchJobFromMap(manager.lowPriorityJobs); + return job.map(entry -> Pair.of(entry, JobPriority.LOW)).orElse(null); } - public void analyzeDb(DatabaseIf databaseIf) throws DdlException { - List analysisInfos = constructAnalysisInfo(databaseIf); - for (AnalysisInfo analysisInfo : analysisInfos) { - try { - if (!canCollect()) { - analysisTaskExecutor.clear(); - break; - } - analysisInfo = getNeedAnalyzeColumns(analysisInfo); - if (analysisInfo == null) { - continue; - } - createSystemAnalysisJob(analysisInfo); - } catch (Throwable t) { - analysisInfo.message = t.getMessage(); - LOG.warn("Failed to auto analyze table {}.{}, reason {}", - databaseIf.getFullName(), analysisInfo.tblId, analysisInfo.message, t); - continue; - } + protected Optional>>> fetchJobFromMap( + Map>> jobMap) { + synchronized (jobMap) { + Optional>>> first = jobMap.entrySet().stream().findFirst(); + first.ifPresent(entry -> jobMap.remove(entry.getKey())); + return first; } } - protected List constructAnalysisInfo(DatabaseIf db) { - List analysisInfos = new ArrayList<>(); - for (TableIf table : getTablesInOrder(db)) { - try { - if (skip(table)) { - continue; - } - createAnalyzeJobForTbl(db, analysisInfos, table); - } catch (Throwable t) { - LOG.warn("Failed to analyze table {}.{}.{}", - db.getCatalog().getName(), db.getFullName(), table.getName(), t); - continue; + protected void processOneJob(TableIf table, Set> columns, + JobPriority priority) throws DdlException { + // appendMvColumn(table, columns); + appendPartitionColumns(table, columns); + columns = columns.stream().filter(c -> StatisticsUtil.needAnalyzeColumn(table, c)).collect(Collectors.toSet()); + if (columns.isEmpty()) { + return; + } + AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority); + LOG.debug("Auto analyze job : {}", analyzeJob.toString()); + try { + executeSystemAnalysisJob(analyzeJob); + } catch (Exception e) { + StringJoiner stringJoiner = new StringJoiner(",", "[", "]"); + for (Pair pair : columns) { + stringJoiner.add(pair.toString()); } + LOG.warn("Fail to auto analyze table {}, columns [{}]", table.getName(), stringJoiner.toString()); } - return analysisInfos; } - // return true if skip auto analyze this time. - protected boolean skip(TableIf table) { - if (!(table instanceof OlapTable || table instanceof HMSExternalTable)) { - return true; + protected void appendPartitionColumns(TableIf table, Set> columns) throws DdlException { + if (!(table instanceof OlapTable)) { + return; } - // For now, only support Hive HMS table auto collection. - if (table instanceof HMSExternalTable - && !((HMSExternalTable) table).getDlaType().equals(HMSExternalTable.DLAType.HIVE)) { - return true; + AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); + TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId()); + if (tableStatsStatus != null && tableStatsStatus.newPartitionLoaded.get()) { + OlapTable olapTable = (OlapTable) table; + columns.addAll(olapTable.getColumnIndexPairs(olapTable.getPartitionColumnNames())); } - if (table.getDataSize(true) < StatisticsUtil.getHugeTableLowerBoundSizeInBytes() * 5) { - return false; + } + + protected void appendMvColumn(TableIf table, Set columns) { + if (!(table instanceof OlapTable)) { + return; } - TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId()); - // means it's never got analyzed or new partition loaded data. - if (tableStats == null || tableStats.newPartitionLoaded.get()) { + OlapTable olapTable = (OlapTable) table; + Set mvColumns = olapTable.getMvColumns(false).stream().map(Column::getName).collect(Collectors.toSet()); + columns.addAll(mvColumns); + } + + protected boolean supportAutoAnalyze(TableIf tableIf) { + if (tableIf == null) { return false; } - if (tableStats.userInjected) { - return true; - } - return System.currentTimeMillis() - - tableStats.updatedTime < StatisticsUtil.getHugeTableAutoAnalyzeIntervalInMillis(); + return tableIf instanceof OlapTable + || tableIf instanceof HMSExternalTable + && ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE); } - protected void createAnalyzeJobForTbl(DatabaseIf db, - List analysisInfos, TableIf table) { + protected AnalysisInfo createAnalyzeJobForTbl( + TableIf table, Set> jobColumns, JobPriority priority) { AnalysisMethod analysisMethod = table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes() ? AnalysisMethod.SAMPLE : AnalysisMethod.FULL; - AnalysisInfo jobInfo = new AnalysisInfoBuilder() + AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); + TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId()); + long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0 : table.getRowCount(); + StringJoiner stringJoiner = new StringJoiner(",", "[", "]"); + for (Pair pair : jobColumns) { + stringJoiner.add(pair.toString()); + } + return new AnalysisInfoBuilder() .setJobId(Env.getCurrentEnv().getNextId()) - .setCatalogId(db.getCatalog().getId()) - .setDBId(db.getId()) + .setCatalogId(table.getDatabase().getCatalog().getId()) + .setDBId(table.getDatabase().getId()) .setTblId(table.getId()) - .setColName(null) + .setColName(stringJoiner.toString()) + .setJobColumns(jobColumns) .setAnalysisType(AnalysisInfo.AnalysisType.FUNDAMENTALS) .setAnalysisMode(AnalysisInfo.AnalysisMode.INCREMENTAL) .setAnalysisMethod(analysisMethod) .setSampleRows(analysisMethod.equals(AnalysisMethod.SAMPLE) - ? StatisticsUtil.getHugeTableSampleRows() : -1) + ? StatisticsUtil.getHugeTableSampleRows() : -1) .setScheduleType(ScheduleType.AUTOMATIC) .setState(AnalysisState.PENDING) .setTaskIds(new ArrayList<>()) .setLastExecTimeInMs(System.currentTimeMillis()) .setJobType(JobType.SYSTEM) .setTblUpdateTime(table.getUpdateTime()) - .setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0 - && analysisMethod.equals(AnalysisMethod.SAMPLE)) + .setRowCount(rowCount) + .setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get()) + .setPriority(priority) .build(); - analysisInfos.add(jobInfo); } + // Analysis job created by the system @VisibleForTesting - protected AnalysisInfo getNeedAnalyzeColumns(AnalysisInfo jobInfo) { - TableIf table = StatisticsUtil.findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId); - // Skip tables that are too wide. - if (table.getBaseSchema().size() > StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) { - return null; - } - - AnalysisManager analysisManager = Env.getServingEnv().getAnalysisManager(); - TableStatsMeta tblStats = analysisManager.findTableStatsStatus(table.getId()); - - List> needRunColumns = null; - if (table.needReAnalyzeTable(tblStats)) { - needRunColumns = table.getColumnIndexPairs(table.getSchemaAllIndexes(false) - .stream().map(Column::getName).collect(Collectors.toSet())); - } else if (table instanceof OlapTable && tblStats.newPartitionLoaded.get()) { - OlapTable olapTable = (OlapTable) table; - Set partitionNames = olapTable.getAllPartitions().stream() - .map(Partition::getName).collect(Collectors.toSet()); - needRunColumns = olapTable.getColumnIndexPairs(partitionNames); + protected void executeSystemAnalysisJob(AnalysisInfo jobInfo) + throws DdlException, ExecutionException, InterruptedException { + Map analysisTasks = new HashMap<>(); + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false); + if (StatisticsUtil.isExternalTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId) + && jobInfo.priority.equals(JobPriority.LOW)) { + analysisManager.createTableLevelTaskForExternalTable(jobInfo, analysisTasks, false); } - - if (needRunColumns == null || needRunColumns.isEmpty()) { - return null; + Env.getCurrentEnv().getAnalysisManager().constructJob(jobInfo, analysisTasks.values()); + Env.getCurrentEnv().getAnalysisManager().registerSysJob(jobInfo, analysisTasks); + Future[] futures = new Future[analysisTasks.values().size()]; + int i = 0; + for (BaseAnalysisTask task : analysisTasks.values()) { + futures[i++] = analysisTaskExecutor.submitTask(task); } - StringJoiner stringJoiner = new StringJoiner(",", "[", "]"); - for (Pair pair : needRunColumns) { - stringJoiner.add(pair.toString()); + for (Future future : futures) { + future.get(); } - return new AnalysisInfoBuilder(jobInfo) - .setColName(stringJoiner.toString()).setJobColumns(needRunColumns).build(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java deleted file mode 100644 index ec187fe893af49..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.statistics; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.MasterDaemon; -import org.apache.doris.statistics.util.StatisticsUtil; - -import org.apache.hudi.common.util.VisibleForTesting; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.HashMap; -import java.util.Map; - -public abstract class StatisticsCollector extends MasterDaemon { - - private static final Logger LOG = LogManager.getLogger(StatisticsCollector.class); - - protected final AnalysisTaskExecutor analysisTaskExecutor; - - public StatisticsCollector(String name, long intervalMs, AnalysisTaskExecutor analysisTaskExecutor) { - super(name, intervalMs); - this.analysisTaskExecutor = analysisTaskExecutor; - } - - @Override - protected void runAfterCatalogReady() { - if (!Env.getCurrentEnv().isMaster()) { - return; - } - if (!StatisticsUtil.statsTblAvailable()) { - LOG.info("Stats table not available, skip"); - return; - } - if (Env.isCheckpointThread()) { - return; - } - collect(); - } - - protected abstract void collect(); - - // Analysis job created by the system - @VisibleForTesting - protected void createSystemAnalysisJob(AnalysisInfo jobInfo) - throws DdlException { - if (jobInfo.jobColumns.isEmpty()) { - // No statistics need to be collected or updated - return; - } - Map analysisTasks = new HashMap<>(); - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false); - if (StatisticsUtil.isExternalTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId)) { - analysisManager.createTableLevelTaskForExternalTable(jobInfo, analysisTasks, false); - } - Env.getCurrentEnv().getAnalysisManager().constructJob(jobInfo, analysisTasks.values()); - Env.getCurrentEnv().getAnalysisManager().registerSysJob(jobInfo, analysisTasks); - analysisTasks.values().forEach(analysisTaskExecutor::submitTask); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java new file mode 100644 index 00000000000000..74484a06afa99f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.MasterDaemon; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.statistics.util.StatisticsUtil; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class StatisticsJobAppender extends MasterDaemon { + + private static final Logger LOG = LogManager.getLogger(StatisticsJobAppender.class); + + public static final long INTERVAL = 1000; + public static final int JOB_MAP_SIZE = 1000; + public static final int TABLE_BATCH_SIZE = 100; + + private long currentDbId = 0; + private long currentTableId = 0; + private long lastRoundFinishTime = 0; + private final long lowJobIntervalMs = TimeUnit.MINUTES.toMillis(1); + + public StatisticsJobAppender() { + super("Statistics Job Appender", INTERVAL); + } + + @Override + protected void runAfterCatalogReady() { + if (!StatisticsUtil.enableAutoAnalyze()) { + return; + } + if (!Env.getCurrentEnv().isMaster()) { + return; + } + if (Env.isCheckpointThread()) { + return; + } + appendJobs(); + } + + protected void appendJobs() { + AnalysisManager manager = Env.getCurrentEnv().getAnalysisManager(); + appendColumnsToJobs(manager.highPriorityColumns, manager.highPriorityJobs); + appendColumnsToJobs(manager.midPriorityColumns, manager.midPriorityJobs); + if (StatisticsUtil.enableAutoAnalyzeInternalCatalog()) { + appendToLowJobs(manager.lowPriorityJobs); + } + } + + protected void appendColumnsToJobs(Queue columnQueue, Map>> jobs) { + int size = columnQueue.size(); + int processed = 0; + for (int i = 0; i < size; i++) { + QueryColumn column = columnQueue.poll(); + if (column == null) { + continue; + } + TableIf table; + try { + table = StatisticsUtil.findTable(column.catalogId, column.dbId, column.tblId); + } catch (Exception e) { + LOG.warn("Fail to find table {}.{}.{} for column {}", + column.catalogId, column.dbId, column.tblId, column.colName, e); + continue; + } + if (StatisticConstants.SYSTEM_DBS.contains(table.getDatabase().getFullName())) { + continue; + } + Column col = table.getColumn(column.colName); + if (col == null || !col.isVisible() || StatisticsUtil.isUnsupportedType(col.getType())) { + continue; + } + Set> columnIndexPairs = table.getColumnIndexPairs( + Collections.singleton(column.colName)).stream() + .filter(p -> StatisticsUtil.needAnalyzeColumn(table, p)) + .collect(Collectors.toSet()); + if (columnIndexPairs.isEmpty()) { + continue; + } + TableName tableName = new TableName(table.getDatabase().getCatalog().getName(), + table.getDatabase().getFullName(), table.getName()); + synchronized (jobs) { + // If job map reach the upper limit, stop putting new jobs. + if (!jobs.containsKey(tableName) && jobs.size() >= JOB_MAP_SIZE) { + LOG.info("High or mid job map full."); + break; + } + if (jobs.containsKey(tableName)) { + jobs.get(tableName).addAll(columnIndexPairs); + } else { + jobs.put(tableName, columnIndexPairs); + } + } + processed++; + } + if (size > 0 && LOG.isDebugEnabled()) { + LOG.debug("{} of {} columns append to jobs", processed, size); + } + } + + protected void appendToLowJobs(Map>> jobs) { + if (System.currentTimeMillis() - lastRoundFinishTime < lowJobIntervalMs) { + return; + } + InternalCatalog catalog = Env.getCurrentInternalCatalog(); + List sortedDbs = catalog.getDbIds().stream().sorted().collect(Collectors.toList()); + int processed = 0; + for (long dbId : sortedDbs) { + if (dbId < currentDbId || catalog.getDbNullable(dbId) == null + || StatisticConstants.SYSTEM_DBS.contains(catalog.getDbNullable(dbId).getFullName())) { + continue; + } + currentDbId = dbId; + Optional db = catalog.getDb(dbId); + if (!db.isPresent()) { + continue; + } + List tables = db.get().getTables().stream() + .sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList()); + for (Table t : tables) { + if (!(t instanceof OlapTable) || t.getId() <= currentTableId) { + continue; + } + if (t.getBaseSchema().size() > StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) { + continue; + } + Set> columnIndexPairs = t.getColumnIndexPairs( + t.getSchemaAllIndexes(false).stream() + .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) + .map(Column::getName).collect(Collectors.toSet())) + .stream().filter(p -> StatisticsUtil.needAnalyzeColumn(t, p)) + .collect(Collectors.toSet()); + if (columnIndexPairs.isEmpty()) { + continue; + } + TableName tableName = new TableName(t.getDatabase().getCatalog().getName(), + t.getDatabase().getFullName(), t.getName()); + synchronized (jobs) { + // If job map reach the upper limit, stop adding new jobs. + if (!jobs.containsKey(tableName) && jobs.size() >= JOB_MAP_SIZE) { + LOG.info("Low job map full."); + return; + } + if (jobs.containsKey(tableName)) { + jobs.get(tableName).addAll(columnIndexPairs); + } else { + jobs.put(tableName, columnIndexPairs); + } + } + currentTableId = t.getId(); + if (++processed >= TABLE_BATCH_SIZE) { + return; + } + } + } + // All tables have been processed once, reset for the next loop. + if (LOG.isDebugEnabled()) { + LOG.debug("All low priority internal tables are appended once."); + } + currentDbId = 0; + currentTableId = 0; + lastRoundFinishTime = System.currentTimeMillis(); + } + + // For unit test only. + public void setLastRoundFinishTime(long value) { + lastRoundFinishTime = value; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index 87ce90c5300a2b..8ec5582af5c93f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -29,8 +29,8 @@ import org.apache.doris.statistics.util.DBObjects; import org.apache.doris.statistics.util.StatisticsUtil; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -337,7 +337,7 @@ public static void alterColumnStatistics(AlterColumnStatsStmt alterColumnStatsSt AnalysisInfo mockedJobInfo = new AnalysisInfoBuilder() .setTblUpdateTime(System.currentTimeMillis()) .setColName("") - .setJobColumns(Lists.newArrayList()) + .setJobColumns(Sets.newHashSet()) .setUserInject(true) .setJobType(AnalysisInfo.JobType.MANUAL) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index 3b9b1e2bead005..a5073a922143f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -25,7 +25,9 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.JobType; +import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.annotations.VisibleForTesting; import com.google.gson.annotations.SerializedName; @@ -129,30 +131,34 @@ public void update(AnalysisInfo analyzedJob, TableIf tableIf) { for (Pair colPair : analyzedJob.jobColumns) { ColStatsMeta colStatsMeta = colToColStatsMeta.get(colPair); if (colStatsMeta == null) { - colToColStatsMeta.put(colPair, new ColStatsMeta(updatedTime, - analyzedJob.analysisMethod, analyzedJob.analysisType, analyzedJob.jobType, 0)); + colToColStatsMeta.put(colPair, new ColStatsMeta(updatedTime, analyzedJob.analysisMethod, + analyzedJob.analysisType, analyzedJob.jobType, 0, analyzedJob.rowCount, + analyzedJob.updateRows)); } else { colStatsMeta.updatedTime = updatedTime; colStatsMeta.analysisType = analyzedJob.analysisType; colStatsMeta.analysisMethod = analyzedJob.analysisMethod; colStatsMeta.jobType = analyzedJob.jobType; + colStatsMeta.updatedRows = analyzedJob.updateRows; + colStatsMeta.rowCount = analyzedJob.rowCount; } } jobType = analyzedJob.jobType; if (tableIf != null) { if (tableIf instanceof OlapTable) { - rowCount = analyzedJob.emptyJob ? 0 : tableIf.getRowCount(); + rowCount = analyzedJob.rowCount; } - if (analyzedJob.emptyJob) { + if (rowCount == 0 && AnalysisMethod.SAMPLE.equals(analyzedJob.analysisMethod)) { return; } if (analyzedJob.jobColumns.containsAll( tableIf.getColumnIndexPairs( - tableIf.getSchemaAllIndexes(false).stream().map(Column::getName).collect(Collectors.toSet())))) { - updatedRows.set(0); + tableIf.getSchemaAllIndexes(false).stream() + .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) + .map(Column::getName).collect(Collectors.toSet())))) { newPartitionLoaded.set(false); - } - if (tableIf instanceof OlapTable) { + userInjected = false; + } else if (tableIf instanceof OlapTable) { PartitionInfo partitionInfo = ((OlapTable) tableIf).getPartitionInfo(); if (partitionInfo != null && analyzedJob.jobColumns .containsAll(tableIf.getColumnIndexPairs(partitionInfo.getPartitionColumns().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 44ff7dafe64cdf..12e4e8ff63abce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -56,6 +56,7 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.qe.AutoCloseConnectContext; @@ -65,11 +66,14 @@ import org.apache.doris.qe.StmtExecutor; import org.apache.doris.qe.VariableMgr; import org.apache.doris.statistics.AnalysisInfo; +import org.apache.doris.statistics.AnalysisManager; +import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.ColumnStatisticBuilder; import org.apache.doris.statistics.Histogram; import org.apache.doris.statistics.ResultRow; import org.apache.doris.statistics.StatisticConstants; +import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.system.Frontend; import com.google.common.base.Preconditions; @@ -790,6 +794,16 @@ public static boolean enableAutoAnalyze() { return false; } + public static boolean enableAutoAnalyzeInternalCatalog() { + try { + return findConfigFromGlobalSessionVar( + SessionVariable.ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG).enableAutoAnalyzeInternalCatalog; + } catch (Exception e) { + LOG.warn("Fail to get value of enable auto analyze internal catalog, return false by default", e); + } + return true; + } + public static int getInsertMergeCount() { try { return findConfigFromGlobalSessionVar(SessionVariable.STATS_INSERT_MERGE_ITEM_COUNT) @@ -898,7 +912,7 @@ public static boolean isMvColumn(TableIf table, String columnName) { } public static boolean isEmptyTable(TableIf table, AnalysisInfo.AnalysisMethod method) { - int waitRowCountReportedTime = 90; + int waitRowCountReportedTime = 75; if (!(table instanceof OlapTable) || method.equals(AnalysisInfo.AnalysisMethod.FULL)) { return false; } @@ -921,4 +935,71 @@ public static boolean isEmptyTable(TableIf table, AnalysisInfo.AnalysisMethod me return true; } + public static boolean needAnalyzeColumn(TableIf table, Pair column) { + if (column == null) { + return false; + } + AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); + TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId()); + // Table never been analyzed, need analyze. + if (tableStatsStatus == null) { + return true; + } + // User injected column stats, don't do auto analyze, avoid overwrite user injected stats. + if (tableStatsStatus.userInjected) { + return false; + } + ColStatsMeta columnStatsMeta = tableStatsStatus.findColumnStatsMeta(column.first, column.second); + // Column never been analyzed, need analyze. + if (columnStatsMeta == null) { + return true; + } + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + // 0. Check new partition first time loaded flag. + if (olapTable.isPartitionColumn(column.second) && tableStatsStatus.newPartitionLoaded.get()) { + return true; + } + // 1. Check row count. + // TODO: One conner case. Last analyze row count is 0, but actually it's not 0 because isEmptyTable waiting. + long currentRowCount = olapTable.getRowCount(); + long lastAnalyzeRowCount = columnStatsMeta.rowCount; + // 1.1 Empty table -> non-empty table. Need analyze. + if (currentRowCount != 0 && lastAnalyzeRowCount == 0) { + return true; + } + // 1.2 Non-empty table -> empty table. Need analyze; + if (currentRowCount == 0 && lastAnalyzeRowCount != 0) { + return true; + } + // 1.3 Table is still empty. Not need to analyze. lastAnalyzeRowCount == 0 is always true here. + if (currentRowCount == 0) { + return false; + } + // 1.4 If row count changed more than the threshold, need analyze. + // lastAnalyzeRowCount == 0 is always false here. + double changeRate = + ((double) Math.abs(currentRowCount - lastAnalyzeRowCount) / lastAnalyzeRowCount) * 100.0; + if (changeRate > StatisticsUtil.getTableStatsHealthThreshold()) { + return true; + } + // 2. Check update rows. + long currentUpdatedRows = tableStatsStatus.updatedRows.get(); + long lastAnalyzeUpdateRows = columnStatsMeta.updatedRows; + changeRate = ((double) Math.abs(currentUpdatedRows - lastAnalyzeUpdateRows) / lastAnalyzeRowCount) * 100.0; + return changeRate > StatisticsUtil.getTableStatsHealthThreshold(); + } else { + // Now, we only support Hive external table auto analyze. + if (!(table instanceof HMSExternalTable)) { + return false; + } + HMSExternalTable hmsTable = (HMSExternalTable) table; + if (!hmsTable.getDlaType().equals(DLAType.HIVE)) { + return false; + } + // External is hard to calculate change rate, use time interval to control analyze frequency. + return System.currentTimeMillis() + - tableStatsStatus.updatedTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis(); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java index 1bf2041bb4f12c..8a163523eebabb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java @@ -80,7 +80,7 @@ public void testAppendBufTest2(@Mocked AnalysisInfo analysisInfo, @Mocked OlapAn new MockUp() { @Mock - protected void writeBuf() { + protected void flushBuffer() { writeBufInvokeTimes.incrementAndGet(); } @@ -111,7 +111,7 @@ public void testAppendBufTest3(@Mocked AnalysisInfo analysisInfo, @Mocked OlapAn new MockUp() { @Mock - protected void writeBuf() { + protected void flushBuffer() { writeBufInvokeTimes.incrementAndGet(); } @@ -184,7 +184,7 @@ protected void executeWithExceptionOnFail(StmtExecutor stmtExecutor) throws Exce protected void syncLoadStats() { } }; - job.writeBuf(); + job.flushBuffer(); Assertions.assertEquals(0, job.queryFinished.size()); } @@ -210,7 +210,7 @@ protected void syncLoadStats() { job.buf.add(new ColStatsData()); job.queryFinished = new HashSet<>(); job.queryFinished.add(task2); - job.writeBuf(); + job.flushBuffer(); Assertions.assertEquals(0, job.queryFinished.size()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java index 674456b0b46891..9c6580ee5cc2af 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java @@ -21,20 +21,31 @@ import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.PartitionNames; import org.apache.doris.analysis.ShowAnalyzeStmt; +import org.apache.doris.analysis.ShowAutoAnalyzeJobsStmt; +import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Table; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.AnalysisInfo.ScheduleType; import org.apache.doris.statistics.util.StatisticsUtil; +import org.apache.doris.thrift.TQueryColumn; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import mockit.Expectations; import mockit.Injectable; import mockit.Mock; @@ -46,8 +57,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; // CHECKSTYLE OFF @@ -110,7 +123,7 @@ public String toString() { // test build sync job @Test public void testBuildAndAssignJob1() throws Exception { - AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setJobColumns(new ArrayList<>()).build(); + AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setJobColumns(new HashSet<>()).build(); new MockUp() { @Mock @@ -187,7 +200,7 @@ public void updateTableStats(AnalysisInfo jobInfo) { // test build async job @Test public void testBuildAndAssignJob2(@Injectable OlapAnalysisTask analysisTask) throws Exception { - AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setJobColumns(new ArrayList<>()) + AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setJobColumns(new HashSet<>()) .setScheduleType(ScheduleType.PERIOD) .build(); new MockUp() { @@ -261,69 +274,6 @@ public void logCreateAnalysisJob(AnalysisInfo analysisJob) { }; } - @Test - public void testReAnalyze() { - new MockUp() { - - final Column c = new Column("col1", PrimitiveType.INT); - @Mock - public List getBaseSchema() { - return Lists.newArrayList(c); - } - - @Mock - public List getColumns() { return Lists.newArrayList(c); } - - @Mock - public List> getColumnIndexPairs(Set columns) { - List> jobList = Lists.newArrayList(); - jobList.add(Pair.of("1", "1")); - jobList.add(Pair.of("2", "2")); - jobList.add(Pair.of("3", "3")); - return jobList; - } - }; - OlapTable olapTable = new OlapTable(); - List> jobList = Lists.newArrayList(); - jobList.add(Pair.of("1", "1")); - jobList.add(Pair.of("2", "2")); - TableStatsMeta stats0 = new TableStatsMeta( - 0, new AnalysisInfoBuilder().setJobColumns(jobList) - .setColName("col1").build(), olapTable); - Assertions.assertTrue(olapTable.needReAnalyzeTable(stats0)); - - new MockUp() { - int count = 0; - int[] rowCount = new int[]{100, 100, 200, 200, 1, 1}; - - @Mock - public long getRowCount() { - return rowCount[count++]; - } - @Mock - public List> getColumnIndexPairs(Set columns) { - List> jobList = Lists.newArrayList(); - return jobList; - } - }; - TableStatsMeta stats1 = new TableStatsMeta( - 50, new AnalysisInfoBuilder().setJobColumns(new ArrayList<>()) - .setColName("col1").build(), olapTable); - stats1.updatedRows.addAndGet(50); - - Assertions.assertTrue(olapTable.needReAnalyzeTable(stats1)); - TableStatsMeta stats2 = new TableStatsMeta( - 190, new AnalysisInfoBuilder() - .setJobColumns(new ArrayList<>()).setColName("col1").build(), olapTable); - stats2.updatedRows.addAndGet(20); - Assertions.assertFalse(olapTable.needReAnalyzeTable(stats2)); - - TableStatsMeta stats3 = new TableStatsMeta(0, new AnalysisInfoBuilder() - .setJobColumns(new ArrayList<>()).setEmptyJob(true).setColName("col1").build(), olapTable); - Assertions.assertTrue(olapTable.needReAnalyzeTable(stats3)); - - } - @Test public void testRecordLimit1() { Config.analyze_record_limit = 2; @@ -399,4 +349,293 @@ public void testShowAutoTasks(@Injectable ShowAnalyzeStmt stmt) { Assertions.assertEquals(AnalysisState.FINISHED, analysisInfos.get(1).getState()); Assertions.assertEquals(AnalysisState.FAILED, analysisInfos.get(2).getState()); } + + @Test + public void testAddQuerySlotToQueue() throws DdlException { + AnalysisManager analysisManager = new AnalysisManager(); + InternalCatalog testCatalog = new InternalCatalog(); + Database db = new Database(100, "testDb"); + testCatalog.unprotectCreateDb(db); + Column column1 = new Column("placeholder", PrimitiveType.INT); + Column column2 = new Column("placeholder", PrimitiveType.INT); + Column column3 = new Column("test", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); + db.createTableWithLock(table, true, false); + + new MockUp
() { + @Mock + public DatabaseIf getDatabase() { + return db; + } + }; + + new MockUp() { + @Mock + public CatalogIf getCatalog() { + return testCatalog; + } + }; + + SlotReference slot1 = new SlotReference(new ExprId(1), "slot1", IntegerType.INSTANCE, true, + new ArrayList<>(), table, column1, Optional.empty(), null); + SlotReference slot2 = new SlotReference(new ExprId(2), "slot2", IntegerType.INSTANCE, true, + new ArrayList<>(), table, column2, Optional.empty(), null); + SlotReference slot3 = new SlotReference(new ExprId(3), "slot3", IntegerType.INSTANCE, true, + new ArrayList<>(), table, column3, Optional.empty(), null); + Set set1 = new HashSet<>(); + set1.add(slot1); + set1.add(slot2); + analysisManager.updateHighPriorityColumn(set1); + Assertions.assertEquals(2, analysisManager.highPriorityColumns.size()); + QueryColumn result = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("placeholder", result.colName); + Assertions.assertEquals(testCatalog.getId(), result.catalogId); + Assertions.assertEquals(db.getId(), result.dbId); + Assertions.assertEquals(table.getId(), result.tblId); + + result = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("placeholder", result.colName); + Assertions.assertEquals(testCatalog.getId(), result.catalogId); + Assertions.assertEquals(db.getId(), result.dbId); + Assertions.assertEquals(table.getId(), result.tblId); + Assertions.assertEquals(0, analysisManager.highPriorityColumns.size()); + Set set2 = new HashSet<>(); + set2.add(slot3); + for (int i = 0; i < AnalysisManager.COLUMN_QUEUE_SIZE / 2 - 1; i++) { + analysisManager.updateHighPriorityColumn(set1); + } + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE - 2, analysisManager.highPriorityColumns.size()); + analysisManager.updateHighPriorityColumn(set2); + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE - 1, analysisManager.highPriorityColumns.size()); + analysisManager.updateHighPriorityColumn(set2); + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE, analysisManager.highPriorityColumns.size()); + analysisManager.updateHighPriorityColumn(set2); + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE, analysisManager.highPriorityColumns.size()); + + for (int i = 0; i < AnalysisManager.COLUMN_QUEUE_SIZE - 2; i++) { + result = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("placeholder", result.colName); + Assertions.assertEquals(testCatalog.getId(), result.catalogId); + Assertions.assertEquals(db.getId(), result.dbId); + Assertions.assertEquals(table.getId(), result.tblId); + } + Assertions.assertEquals(2, analysisManager.highPriorityColumns.size()); + result = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("test", result.colName); + Assertions.assertEquals(testCatalog.getId(), result.catalogId); + Assertions.assertEquals(db.getId(), result.dbId); + Assertions.assertEquals(table.getId(), result.tblId); + + Assertions.assertEquals(1, analysisManager.highPriorityColumns.size()); + result = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("test", result.colName); + Assertions.assertEquals(testCatalog.getId(), result.catalogId); + Assertions.assertEquals(db.getId(), result.dbId); + Assertions.assertEquals(table.getId(), result.tblId); + + result = analysisManager.highPriorityColumns.poll(); + Assertions.assertNull(result); + } + + @Test + public void testMergeFollowerColumn() throws DdlException { + AnalysisManager analysisManager = new AnalysisManager(); + QueryColumn placeholder = new QueryColumn(1, 2, 3, "placeholder"); + QueryColumn high1 = new QueryColumn(10, 20, 30, "high1"); + QueryColumn high2 = new QueryColumn(11, 21, 31, "high2"); + QueryColumn mid1 = new QueryColumn(100, 200, 300, "mid1"); + QueryColumn mid2 = new QueryColumn(101, 201, 301, "mid2"); + List highColumns = new ArrayList<>(); + highColumns.add(high1.toThrift()); + highColumns.add(high2.toThrift()); + List midColumns = new ArrayList<>(); + midColumns.add(mid1.toThrift()); + midColumns.add(mid2.toThrift()); + for (int i = 0; i < AnalysisManager.COLUMN_QUEUE_SIZE - 1; i++) { + analysisManager.highPriorityColumns.offer(placeholder); + } + for (int i = 0; i < AnalysisManager.COLUMN_QUEUE_SIZE - 2; i++) { + analysisManager.midPriorityColumns.offer(placeholder); + } + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE - 1, analysisManager.highPriorityColumns.size()); + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE - 2, analysisManager.midPriorityColumns.size()); + analysisManager.mergeFollowerQueryColumns(highColumns, midColumns); + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE, analysisManager.highPriorityColumns.size()); + Assertions.assertEquals(AnalysisManager.COLUMN_QUEUE_SIZE, analysisManager.midPriorityColumns.size()); + for (int i = 0; i < AnalysisManager.COLUMN_QUEUE_SIZE - 1; i++) { + QueryColumn poll = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("placeholder", poll.colName); + Assertions.assertEquals(1, poll.catalogId); + Assertions.assertEquals(2, poll.dbId); + Assertions.assertEquals(3, poll.tblId); + } + QueryColumn poll = analysisManager.highPriorityColumns.poll(); + Assertions.assertEquals("high1", poll.colName); + Assertions.assertEquals(10, poll.catalogId); + Assertions.assertEquals(20, poll.dbId); + Assertions.assertEquals(30, poll.tblId); + Assertions.assertEquals(0, analysisManager.highPriorityColumns.size()); + + for (int i = 0; i < AnalysisManager.COLUMN_QUEUE_SIZE - 2; i++) { + QueryColumn pol2 = analysisManager.midPriorityColumns.poll(); + Assertions.assertEquals("placeholder", pol2.colName); + Assertions.assertEquals(1, pol2.catalogId); + Assertions.assertEquals(2, pol2.dbId); + Assertions.assertEquals(3, pol2.tblId); + } + QueryColumn pol2 = analysisManager.midPriorityColumns.poll(); + Assertions.assertEquals("mid1", pol2.colName); + Assertions.assertEquals(100, pol2.catalogId); + Assertions.assertEquals(200, pol2.dbId); + Assertions.assertEquals(300, pol2.tblId); + + pol2 = analysisManager.midPriorityColumns.poll(); + Assertions.assertEquals("mid2", pol2.colName); + Assertions.assertEquals(101, pol2.catalogId); + Assertions.assertEquals(201, pol2.dbId); + Assertions.assertEquals(301, pol2.tblId); + Assertions.assertEquals(0, analysisManager.midPriorityColumns.size()); + } + + @Test + public void testShowAutoJobs() { + AnalysisManager manager = new AnalysisManager(); + TableName high1 = new TableName("catalog1", "db1", "high1"); + TableName high2 = new TableName("catalog2", "db2", "high2"); + TableName mid1 = new TableName("catalog3", "db3", "mid1"); + TableName mid2 = new TableName("catalog4", "db4", "mid2"); + TableName low1 = new TableName("catalog5", "db5", "low1"); + + manager.highPriorityJobs.put(high1, new HashSet<>()); + manager.highPriorityJobs.get(high1).add(Pair.of("index1", "col1")); + manager.highPriorityJobs.get(high1).add(Pair.of("index2", "col2")); + manager.highPriorityJobs.put(high2, new HashSet<>()); + manager.highPriorityJobs.get(high2).add(Pair.of("index1", "col3")); + manager.midPriorityJobs.put(mid1, new HashSet<>()); + manager.midPriorityJobs.get(mid1).add(Pair.of("index1", "col4")); + manager.midPriorityJobs.put(mid2, new HashSet<>()); + manager.midPriorityJobs.get(mid2).add(Pair.of("index1", "col5")); + manager.lowPriorityJobs.put(low1, new HashSet<>()); + manager.lowPriorityJobs.get(low1).add(Pair.of("index1", "col6")); + manager.lowPriorityJobs.get(low1).add(Pair.of("index1", "col7")); + + new MockUp() { + @Mock + public boolean isAnalyzed() { + return true; + } + }; + ShowAutoAnalyzeJobsStmt stmt = new ShowAutoAnalyzeJobsStmt(null, null); + List autoAnalysisPendingJobs = manager.showAutoPendingJobs(stmt); + Assertions.assertEquals(5, autoAnalysisPendingJobs.size()); + AutoAnalysisPendingJob job = autoAnalysisPendingJobs.get(0); + Assertions.assertEquals("catalog1", job.catalogName); + Assertions.assertEquals("db1", job.dbName); + Assertions.assertEquals("high1", job.tableName); + Assertions.assertEquals(2, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col1"))); + Assertions.assertTrue(job.columns.contains(Pair.of("index2", "col2"))); + Assertions.assertEquals(JobPriority.HIGH, job.priority); + + job = autoAnalysisPendingJobs.get(1); + Assertions.assertEquals("catalog2", job.catalogName); + Assertions.assertEquals("db2", job.dbName); + Assertions.assertEquals("high2", job.tableName); + Assertions.assertEquals(1, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col3"))); + Assertions.assertEquals(JobPriority.HIGH, job.priority); + + job = autoAnalysisPendingJobs.get(2); + Assertions.assertEquals("catalog3", job.catalogName); + Assertions.assertEquals("db3", job.dbName); + Assertions.assertEquals("mid1", job.tableName); + Assertions.assertEquals(1, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col4"))); + Assertions.assertEquals(JobPriority.MID, job.priority); + + job = autoAnalysisPendingJobs.get(3); + Assertions.assertEquals("catalog4", job.catalogName); + Assertions.assertEquals("db4", job.dbName); + Assertions.assertEquals("mid2", job.tableName); + Assertions.assertEquals(1, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col5"))); + Assertions.assertEquals(JobPriority.MID, job.priority); + + job = autoAnalysisPendingJobs.get(4); + Assertions.assertEquals("catalog5", job.catalogName); + Assertions.assertEquals("db5", job.dbName); + Assertions.assertEquals("low1", job.tableName); + Assertions.assertEquals(2, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col6"))); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col7"))); + Assertions.assertEquals(JobPriority.LOW, job.priority); + + new MockUp() { + @Mock + public String getPriority() { + return JobPriority.HIGH.name().toUpperCase(); + } + }; + List highJobs = manager.showAutoPendingJobs(stmt); + Assertions.assertEquals(2, highJobs.size()); + job = highJobs.get(0); + Assertions.assertEquals("catalog1", job.catalogName); + Assertions.assertEquals("db1", job.dbName); + Assertions.assertEquals("high1", job.tableName); + Assertions.assertEquals(2, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col1"))); + Assertions.assertTrue(job.columns.contains(Pair.of("index2", "col2"))); + Assertions.assertEquals(JobPriority.HIGH, job.priority); + + job = highJobs.get(1); + Assertions.assertEquals("catalog2", job.catalogName); + Assertions.assertEquals("db2", job.dbName); + Assertions.assertEquals("high2", job.tableName); + Assertions.assertEquals(1, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col3"))); + Assertions.assertEquals(JobPriority.HIGH, job.priority); + + new MockUp() { + @Mock + public String getPriority() { + return JobPriority.MID.name().toUpperCase(); + } + }; + List midJobs = manager.showAutoPendingJobs(stmt); + Assertions.assertEquals(2, midJobs.size()); + job = midJobs.get(0); + Assertions.assertEquals("catalog3", job.catalogName); + Assertions.assertEquals("db3", job.dbName); + Assertions.assertEquals("mid1", job.tableName); + Assertions.assertEquals(1, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col4"))); + Assertions.assertEquals(JobPriority.MID, job.priority); + + job = midJobs.get(1); + Assertions.assertEquals("catalog4", job.catalogName); + Assertions.assertEquals("db4", job.dbName); + Assertions.assertEquals("mid2", job.tableName); + Assertions.assertEquals(1, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col5"))); + Assertions.assertEquals(JobPriority.MID, job.priority); + + new MockUp() { + @Mock + public String getPriority() { + return JobPriority.LOW.name().toUpperCase(); + } + }; + List lowJobs = manager.showAutoPendingJobs(stmt); + Assertions.assertEquals(1, lowJobs.size()); + job = lowJobs.get(0); + Assertions.assertEquals("catalog5", job.catalogName); + Assertions.assertEquals("db5", job.dbName); + Assertions.assertEquals("low1", job.tableName); + Assertions.assertEquals(2, job.columns.size()); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col6"))); + Assertions.assertTrue(job.columns.contains(Pair.of("index1", "col7"))); + Assertions.assertEquals(JobPriority.LOW, job.priority); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java index 5698f0e9b20e63..29e04b1ef4fd94 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java @@ -34,7 +34,7 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.utframe.TestWithFeService; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import mockit.Mock; import mockit.MockUp; import mockit.Mocked; @@ -44,6 +44,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -157,7 +158,7 @@ public void syncLoadColStats(long tableId, long idxId, String colName) { }; AnalysisTaskExecutor analysisTaskExecutor = new AnalysisTaskExecutor(1); - List> columns = Lists.newArrayList(); + Set> columns = Sets.newHashSet(); columns.add(Pair.of("col1", "t1")); AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setJobId(0).setTaskId(0) .setCatalogId(0).setDBId(0).setTblId(0) diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java index bf6ce32e155f42..250a796cee4cd5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java @@ -36,7 +36,7 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.utframe.TestWithFeService; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import mockit.Expectations; import mockit.Mock; import mockit.MockUp; @@ -48,6 +48,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; public class AnalyzeTest extends TestWithFeService { @@ -159,7 +160,7 @@ public void execSQLs(List partitionAnalysisSQLs, Map par @Mock protected void runQuery(String sql) {} }; - List> colList = Lists.newArrayList(); + Set> colList = Sets.newHashSet(); colList.add(Pair.of("col1", "index1")); AnalysisInfo analysisJobInfo = new AnalysisInfoBuilder().setJobId(0).setTaskId(0) .setCatalogId(0) @@ -171,6 +172,7 @@ protected void runQuery(String sql) {} .setAnalysisType(AnalysisType.FUNDAMENTALS) .setJobColumns(colList) .setState(AnalysisState.RUNNING) + .setRowCount(10) .build(); new OlapAnalysisTask(analysisJobInfo).doExecute(); new Expectations() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/FollowerColumnSenderTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/FollowerColumnSenderTest.java new file mode 100644 index 00000000000000..2a5ae531d1e6dd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/FollowerColumnSenderTest.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Pair; +import org.apache.doris.statistics.util.StatisticsUtil; +import org.apache.doris.thrift.TQueryColumn; + +import mockit.Mock; +import mockit.MockUp; +import org.eclipse.jetty.util.BlockingArrayQueue; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Queue; +import java.util.Set; + +public class FollowerColumnSenderTest { + + @Test + public void testGetNeedAnalyzeColumns() { + new MockUp() { + @Mock + public Column getColumn(String name) { + return new Column("col", PrimitiveType.INT); + } + + @Mock + public Set> getColumnIndexPairs(Set columns) { + return Collections.singleton(Pair.of("mockIndex", "mockCol")); + } + }; + + new MockUp() { + boolean[] result = {false, true, false, true, true}; + int i = 0; + @Mock + public boolean needAnalyzeColumn(TableIf table, Pair column) { + return result[i++]; + } + + @Mock + public TableIf findTable(long catalogId, long dbId, long tblId) { + return new OlapTable(); + } + }; + QueryColumn column1 = new QueryColumn(1, 2, 3, "col1"); + QueryColumn column2 = new QueryColumn(1, 2, 3, "col2"); + QueryColumn column3 = new QueryColumn(1, 2, 3, "col3"); + QueryColumn column4 = new QueryColumn(1, 2, 3, "col4"); + Queue queue = new BlockingArrayQueue<>(); + queue.add(column1); + queue.add(column2); + queue.add(column3); + queue.add(column4); + queue.add(column4); + Assertions.assertEquals(5, queue.size()); + + FollowerColumnSender sender = new FollowerColumnSender(); + Set needAnalyzeColumns = sender.getNeedAnalyzeColumns(queue); + Assertions.assertEquals(2, needAnalyzeColumns.size()); + Assertions.assertFalse(needAnalyzeColumns.contains(column1.toThrift())); + Assertions.assertTrue(needAnalyzeColumns.contains(column2.toThrift())); + Assertions.assertFalse(needAnalyzeColumns.contains(column3.toThrift())); + Assertions.assertTrue(needAnalyzeColumns.contains(column4.toThrift())); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index f7b75261cc54fa..6324624abac6a6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -17,457 +17,125 @@ package org.apache.doris.statistics; +import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.EnvFactory; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.Type; -import org.apache.doris.catalog.View; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; -import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.statistics.util.StatisticsUtil; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.jdbc.JdbcExternalTable; -import com.google.common.collect.Lists; -import mockit.Expectations; -import mockit.Injectable; import mockit.Mock; import mockit.MockUp; -import mockit.Mocked; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.time.LocalTime; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; public class StatisticsAutoCollectorTest { @Test - public void testAnalyzeAll(@Injectable AnalysisInfo analysisInfo) { - new MockUp() { - @Mock - public Collection getAllDbs() { - Database db1 = new Database(1, FeConstants.INTERNAL_DB_NAME); - Database db2 = new Database(2, "anyDB"); - List databaseIfs = new ArrayList<>(); - databaseIfs.add(db1); - databaseIfs.add(db2); - return databaseIfs; - } - }; - new MockUp() { - @Mock - public List constructAnalysisInfo(DatabaseIf db) { - return Arrays.asList(analysisInfo, analysisInfo); - } - - int count = 0; - - @Mock - public AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) { - return count++ == 0 ? null : jobInfo; - } - - @Mock - public void createSystemAnalysisJob(AnalysisInfo jobInfo) - throws DdlException { - - } - }; - - StatisticsAutoCollector saa = new StatisticsAutoCollector(); - saa.runAfterCatalogReady(); - new Expectations() { - { - try { - saa.createSystemAnalysisJob((AnalysisInfo) any); - times = 1; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - } - - @Test - public void testConstructAnalysisInfo( - @Injectable OlapTable o2, @Injectable View v) { - new MockUp() { - @Mock - public List
getTables() { - List
tableIfs = new ArrayList<>(); - tableIfs.add(o2); - tableIfs.add(v); - return tableIfs; - } - - @Mock - public String getFullName() { - return "anyDb"; - } - }; - - new MockUp() { - @Mock - public String getName() { - return "anytable"; - } - - @Mock - public List getSchemaAllIndexes(boolean full) { - List columns = new ArrayList<>(); - columns.add(new Column("c1", PrimitiveType.INT)); - columns.add(new Column("c2", PrimitiveType.HLL)); - return columns; - } - }; - StatisticsAutoCollector saa = new StatisticsAutoCollector(); - List analysisInfoList = saa.constructAnalysisInfo(new Database(1, "anydb")); - Assertions.assertEquals(1, analysisInfoList.size()); - Assertions.assertNull(analysisInfoList.get(0).colName); - } - - @Test - public void testSkipWideTable() { - - TableIf tableIf = new OlapTable(); - - new MockUp() { - @Mock - public List getBaseSchema() { - return Lists.newArrayList(new Column("col1", Type.INT), new Column("col2", Type.INT)); - } - - @Mock - public List> getColumnIndexPairs(Set columns) { - ArrayList> list = Lists.newArrayList(); - list.add(Pair.of("1", "1")); - return list; - } - }; - - new MockUp() { - int count = 0; - int[] thresholds = {1, 10}; - - @Mock - public TableIf findTable(long catalogName, long dbName, long tblName) { - return tableIf; - } - - @Mock - public int getAutoAnalyzeTableWidthThreshold() { - return thresholds[count++]; - } - }; - - AnalysisInfo analysisInfo = new AnalysisInfoBuilder().build(); - StatisticsAutoCollector statisticsAutoCollector = new StatisticsAutoCollector(); - Assertions.assertNull(statisticsAutoCollector.getNeedAnalyzeColumns(analysisInfo)); - Assertions.assertNotNull(statisticsAutoCollector.getNeedAnalyzeColumns(analysisInfo)); - } - - @Test - public void testLoop() { - AtomicBoolean timeChecked = new AtomicBoolean(); - AtomicBoolean switchChecked = new AtomicBoolean(); - new MockUp() { - - @Mock - public boolean inAnalyzeTime(LocalTime now) { - timeChecked.set(true); - return true; - } - - @Mock - public boolean enableAutoAnalyze() { - switchChecked.set(true); - return true; - } - }; - StatisticsAutoCollector autoCollector = new StatisticsAutoCollector(); - autoCollector.collect(); - Assertions.assertTrue(timeChecked.get() && switchChecked.get()); - - } - - @Test - public void checkAvailableThread() { - StatisticsAutoCollector autoCollector = new StatisticsAutoCollector(); - Assertions.assertEquals(Config.auto_analyze_simultaneously_running_task_num, - autoCollector.analysisTaskExecutor.executors.getMaximumPoolSize()); - } - - @Test - public void testSkip(@Mocked OlapTable olapTable, @Mocked TableStatsMeta stats, @Mocked TableIf anyOtherTable) { - new MockUp() { - - @Mock - public long getDataSize(boolean singleReplica) { - return StatisticsUtil.getHugeTableLowerBoundSizeInBytes() * 5 + 1000000000; - } - }; - - new MockUp() { - - @Mock - public TableStatsMeta findTableStatsStatus(long tblId) { - return stats; - } - }; - // A very huge table has been updated recently, so we should skip it this time - stats.updatedTime = System.currentTimeMillis() - 1000; - stats.newPartitionLoaded = new AtomicBoolean(); - stats.newPartitionLoaded.set(true); - StatisticsAutoCollector autoCollector = new StatisticsAutoCollector(); - // Test new partition loaded data for the first time. Not skip. - Assertions.assertFalse(autoCollector.skip(olapTable)); - stats.newPartitionLoaded.set(false); - // Assertions.assertTrue(autoCollector.skip(olapTable)); - // The update of this huge table is long time ago, so we shouldn't skip it this time - stats.updatedTime = System.currentTimeMillis() - - StatisticsUtil.getHugeTableAutoAnalyzeIntervalInMillis() - 10000; - Assertions.assertFalse(autoCollector.skip(olapTable)); - new MockUp() { - - @Mock - public TableStatsMeta findTableStatsStatus(long tblId) { - return null; - } - }; - // can't find table stats meta, which means this table never get analyzed, so we shouldn't skip it this time - Assertions.assertFalse(autoCollector.skip(olapTable)); - new MockUp() { - - @Mock - public TableStatsMeta findTableStatsStatus(long tblId) { - return stats; - } - }; - stats.userInjected = true; - Assertions.assertTrue(autoCollector.skip(olapTable)); - // this is not olap table nor external table, so we should skip it this time - Assertions.assertTrue(autoCollector.skip(anyOtherTable)); + public void testFetchJob() { + AnalysisManager manager = new AnalysisManager(); + TableName high1 = new TableName("catalog", "db", "high1"); + TableName high2 = new TableName("catalog", "db", "high2"); + TableName mid1 = new TableName("catalog", "db", "mid1"); + TableName mid2 = new TableName("catalog", "db", "mid2"); + TableName low1 = new TableName("catalog", "db", "low1"); + + manager.highPriorityJobs.put(high1, new HashSet<>()); + manager.highPriorityJobs.get(high1).add(Pair.of("index1", "col1")); + manager.highPriorityJobs.get(high1).add(Pair.of("index1", "col2")); + manager.highPriorityJobs.put(high2, new HashSet<>()); + manager.highPriorityJobs.get(high2).add(Pair.of("index1", "col3")); + manager.midPriorityJobs.put(mid1, new HashSet<>()); + manager.midPriorityJobs.get(mid1).add(Pair.of("index1", "col4")); + manager.midPriorityJobs.put(mid2, new HashSet<>()); + manager.midPriorityJobs.get(mid2).add(Pair.of("index1", "col5")); + manager.lowPriorityJobs.put(low1, new HashSet<>()); + manager.lowPriorityJobs.get(low1).add(Pair.of("index1", "col6")); + manager.lowPriorityJobs.get(low1).add(Pair.of("index1", "col7")); + + + new MockUp() { + @Mock + public AnalysisManager getAnalysisManager() { + return manager; + } + }; + StatisticsAutoCollector collector = new StatisticsAutoCollector(); + Pair>>, JobPriority> job = collector.getJob(); + Assertions.assertEquals(high1, job.first.getKey()); + Assertions.assertEquals(2, job.first.getValue().size()); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col1"))); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col2"))); + Assertions.assertEquals(JobPriority.HIGH, job.second); + + job = collector.getJob(); + Assertions.assertEquals(high2, job.first.getKey()); + Assertions.assertEquals(1, job.first.getValue().size()); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col3"))); + Assertions.assertEquals(JobPriority.HIGH, job.second); + + job = collector.getJob(); + Assertions.assertEquals(mid1, job.first.getKey()); + Assertions.assertEquals(1, job.first.getValue().size()); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col4"))); + Assertions.assertEquals(JobPriority.MID, job.second); + + job = collector.getJob(); + Assertions.assertEquals(mid2, job.first.getKey()); + Assertions.assertEquals(1, job.first.getValue().size()); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col5"))); + Assertions.assertEquals(JobPriority.MID, job.second); + + job = collector.getJob(); + Assertions.assertEquals(low1, job.first.getKey()); + Assertions.assertEquals(2, job.first.getValue().size()); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col6"))); + Assertions.assertTrue(job.first.getValue().contains(Pair.of("index1", "col7"))); + Assertions.assertEquals(JobPriority.LOW, job.second); + + job = collector.getJob(); + Assertions.assertNull(job); } - // For small table, use full @Test - public void testCreateAnalyzeJobForTbl1( - @Injectable OlapTable t1, - @Injectable Database db - ) throws Exception { - new MockUp() { + public void testSupportAutoAnalyze() { + StatisticsAutoCollector collector = new StatisticsAutoCollector(); + Assertions.assertFalse(collector.supportAutoAnalyze(null)); + Column column1 = new Column("placeholder", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table1 = new OlapTable(200, "testTable", schema, null, null, null); + Assertions.assertTrue(collector.supportAutoAnalyze(table1)); - @Mock - public CatalogIf getCatalog() { - return Env.getCurrentInternalCatalog(); - } + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null); + Assertions.assertFalse(collector.supportAutoAnalyze(externalTable)); + new MockUp() { @Mock - public long getId() { - return 0; + public DLAType getDlaType() { + return DLAType.ICEBERG; } }; - new MockUp() { - - int count = 0; + ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + Assertions.assertFalse(collector.supportAutoAnalyze(icebergExternalTable)); + new MockUp() { @Mock - public List getBaseSchema() { - return Lists.newArrayList(new Column("test", PrimitiveType.INT)); - } - - @Mock - public long getDataSize(boolean singleReplica) { - return StatisticsUtil.getHugeTableLowerBoundSizeInBytes() - 1; - } - - @Mock - public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) { - return new OlapAnalysisTask(info); - } - - @Mock - public List getMvColumnIndexIds(String columnName) { - ArrayList objects = new ArrayList<>(); - objects.add(-1L); - return objects; + public DLAType getDlaType() { + return DLAType.HIVE; } }; - - new MockUp() { - @Mock - public TableIf findTable(long catalogId, long dbId, long tblId) { - return t1; - } - }; - - StatisticsAutoCollector sac = new StatisticsAutoCollector(); - List jobInfos = new ArrayList<>(); - sac.createAnalyzeJobForTbl(db, jobInfos, t1); - AnalysisInfo jobInfo = jobInfos.get(0); - List> columnNames = Lists.newArrayList(); - columnNames.add(Pair.of("test", "t1")); - jobInfo = new AnalysisInfoBuilder(jobInfo).setJobColumns(columnNames).build(); - Map analysisTasks = new HashMap<>(); - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false); - Assertions.assertEquals(1, analysisTasks.size()); - for (BaseAnalysisTask task : analysisTasks.values()) { - Assertions.assertNull(task.getTableSample()); - } - } - - // for big table, use sample - @Test - public void testCreateAnalyzeJobForTbl2( - @Injectable OlapTable t1, - @Injectable Database db - ) throws Exception { - new MockUp() { - - @Mock - public CatalogIf getCatalog() { - return Env.getCurrentInternalCatalog(); - } - - @Mock - public long getId() { - return 0; - } - }; - new MockUp() { - - int count = 0; - - @Mock - public List getBaseSchema() { - return Lists.newArrayList(new Column("test", PrimitiveType.INT)); - } - - @Mock - public long getDataSize(boolean singleReplica) { - return StatisticsUtil.getHugeTableLowerBoundSizeInBytes() * 2; - } - - @Mock - public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) { - return new OlapAnalysisTask(info); - } - - @Mock - public List getMvColumnIndexIds(String columnName) { - ArrayList objects = new ArrayList<>(); - objects.add(-1L); - return objects; - } - }; - - new MockUp() { - @Mock - public TableIf findTable(long catalogId, long dbId, long tblId) { - return t1; - } - }; - - StatisticsAutoCollector sac = new StatisticsAutoCollector(); - List jobInfos = new ArrayList<>(); - sac.createAnalyzeJobForTbl(db, jobInfos, t1); - AnalysisInfo jobInfo = jobInfos.get(0); - List> colNames = Lists.newArrayList(); - colNames.add(Pair.of("test", "1")); - jobInfo = new AnalysisInfoBuilder(jobInfo).setJobColumns(colNames).build(); - Map analysisTasks = new HashMap<>(); - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - analysisManager.createTaskForEachColumns(jobInfo, analysisTasks, false); - Assertions.assertEquals(1, analysisTasks.size()); - for (BaseAnalysisTask task : analysisTasks.values()) { - Assertions.assertNotNull(task.getTableSample()); - } - } - - @Test - public void testDisableAuto1() throws Exception { - InternalCatalog catalog1 = EnvFactory.getInstance().createInternalCatalog(); - List catalogs = Lists.newArrayList(); - catalogs.add(catalog1); - - new MockUp() { - @Mock - public List getCatalogsInOrder() { - return catalogs; - } - - @Mock - protected boolean canCollect() { - return false; - } - - }; - - StatisticsAutoCollector sac = new StatisticsAutoCollector(); - new Expectations(catalog1) {{ - catalog1.enableAutoAnalyze(); - times = 0; - }}; - - sac.analyzeAll(); - } - - @Test - public void testDisableAuto2() throws Exception { - InternalCatalog catalog1 = EnvFactory.getInstance().createInternalCatalog(); - List catalogs = Lists.newArrayList(); - catalogs.add(catalog1); - - Database db1 = new Database(); - List> dbs = Lists.newArrayList(); - dbs.add(db1); - - new MockUp() { - int count = 0; - boolean[] canCollectReturn = {true, false}; - @Mock - public List getCatalogsInOrder() { - return catalogs; - } - - @Mock - public List> getDatabasesInOrder(CatalogIf catalog) { - return dbs; - } - - @Mock - protected boolean canCollect() { - return canCollectReturn[count++]; - } - - }; - - StatisticsAutoCollector sac = new StatisticsAutoCollector(); - new Expectations(catalog1, db1) {{ - catalog1.enableAutoAnalyze(); - result = true; - times = 1; - db1.getFullName(); - times = 0; - }}; - - sac.analyzeAll(); + ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + Assertions.assertTrue(collector.supportAutoAnalyze(hiveExternalTable)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java new file mode 100644 index 00000000000000..e3255ab23a0381 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java @@ -0,0 +1,281 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.statistics.util.StatisticsUtil; + +import com.google.common.collect.Lists; +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; + +public class StatisticsJobAppenderTest { + + @Test + public void testAppendQueryColumnToHighAndMidJobMap() throws DdlException { + InternalCatalog testCatalog = new InternalCatalog(); + Database db = new Database(100, "testDb"); + testCatalog.unprotectCreateDb(db); + Column column1 = new Column("placeholder", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table1 = new OlapTable(200, "testTable", schema, null, null, null); + OlapTable table2 = new OlapTable(200, "testTable2", schema, null, null, null); + OlapTable table3 = new OlapTable(200, "testTable3", schema, null, null, null); + new MockUp() { + int i = 0; + Table[] tables = {table1, table2, table1, table3, table2}; + + @Mock + public boolean needAnalyzeColumn(TableIf table, Pair column) { + return true; + } + + @Mock + public TableIf findTable(long catalogId, long dbId, long tblId) { + return tables[i++]; + } + }; + + new MockUp
() { + @Mock + public DatabaseIf getDatabase() { + return db; + } + + @Mock + public Column getColumn(String name) { + return new Column("mockCol", Type.INT); + } + }; + + new MockUp() { + @Mock + public Set> getColumnIndexPairs(Set columns) { + String column = columns.iterator().next(); + return Collections.singleton(Pair.of("mockIndex", column)); + } + }; + + Queue testQueue = new ArrayBlockingQueue<>(100); + Map>> testMap = new HashMap<>(); + QueryColumn high1 = new QueryColumn(10, 20, 30, "high1"); + testQueue.add(high1); + + StatisticsJobAppender appender = new StatisticsJobAppender(); + appender.appendColumnsToJobs(testQueue, testMap); + Assertions.assertEquals(1, testMap.size()); + Assertions.assertEquals(1, testMap.values().size()); + Assertions.assertTrue(testMap.get(new TableName("internal", "testDb", "testTable")).contains(Pair.of("mockIndex", "high1"))); + + QueryColumn high2 = new QueryColumn(10, 20, 30, "high2"); + QueryColumn high3 = new QueryColumn(10, 20, 30, "high3"); + testQueue.add(high2); + testQueue.add(high3); + appender.appendColumnsToJobs(testQueue, testMap); + Assertions.assertEquals(2, testMap.size()); + + Set> table1Column = testMap.get(new TableName("internal", "testDb", "testTable")); + Assertions.assertEquals(2, table1Column.size()); + Assertions.assertTrue(table1Column.contains(Pair.of("mockIndex", "high1"))); + Assertions.assertTrue(table1Column.contains(Pair.of("mockIndex", "high3"))); + + Set> table2Column = testMap.get(new TableName("internal", "testDb", "testTable2")); + Assertions.assertEquals(1, table2Column.size()); + Assertions.assertTrue(table2Column.contains(Pair.of("mockIndex", "high2"))); + + for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE - 2; i++) { + testMap.put(new TableName("a", "b", UUID.randomUUID().toString()), new HashSet<>()); + } + Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testMap.size()); + + QueryColumn high4 = new QueryColumn(10, 20, 30, "high4"); + testQueue.add(high4); + appender.appendColumnsToJobs(testQueue, testMap); + Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testMap.size()); + + QueryColumn high5 = new QueryColumn(10, 20, 30, "high5"); + testQueue.add(high5); + appender.appendColumnsToJobs(testQueue, testMap); + table2Column = testMap.get(new TableName("internal", "testDb", "testTable2")); + Assertions.assertEquals(2, table2Column.size()); + Assertions.assertTrue(table2Column.contains(Pair.of("mockIndex", "high2"))); + Assertions.assertTrue(table2Column.contains(Pair.of("mockIndex", "high5"))); + } + + @Test + public void testAppendQueryColumnToLowJobMap() throws DdlException { + InternalCatalog testCatalog = new InternalCatalog(); + int id = 10; + for (int i = 0; i < 70; i++) { + Database db = new Database(id++, "testDb" + i); + testCatalog.unprotectCreateDb(db); + Column column1 = new Column("placeholder", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); + OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); + db.createTableWithLock(table1, true, false); + db.createTableWithLock(table2, true, false); + } + + new MockUp() { + @Mock + public InternalCatalog getCurrentInternalCatalog() { + return testCatalog; + } + }; + + new MockUp() { + @Mock + public List getBaseSchema() { + return Lists.newArrayList(); + } + + @Mock + public Set> getColumnIndexPairs(Set columns) { + return Collections.singleton(Pair.of("mockIndex", "mockColumn")); + } + }; + + Map>> testMap = new HashMap<>(); + StatisticsJobAppender appender = new StatisticsJobAppender(); + appender.appendToLowJobs(testMap); + Assertions.assertEquals(100, testMap.size()); + testMap.clear(); + appender.appendToLowJobs(testMap); + Assertions.assertEquals(40, testMap.size()); + + for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) { + Database db = new Database(id++, "testDb" + i); + testCatalog.unprotectCreateDb(db); + Column column1 = new Column("placeholder", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); + OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); + db.createTableWithLock(table1, true, false); + db.createTableWithLock(table2, true, false); + } + + testMap.clear(); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testMap.size()); + } + + @Test + public void testSkipWideTable() throws DdlException { + InternalCatalog testCatalog = new InternalCatalog(); + int id = 10; + Database db = new Database(id++, "testDb"); + testCatalog.unprotectCreateDb(db); + Column column1 = new Column("placeholder", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); + db.createTableWithLock(table1, true, false); + new MockUp() { + @Mock + public InternalCatalog getCurrentInternalCatalog() { + return testCatalog; + } + }; + new MockUp() { + @Mock + public List getBaseSchema() { + return Lists.newArrayList(new Column("col1", Type.INT), new Column("col2", Type.INT)); + } + + @Mock + public Set> getColumnIndexPairs(Set columns) { + return Collections.singleton(Pair.of("1", "1")); + } + }; + + new MockUp() { + int count = 0; + int[] thresholds = {1, 10}; + + @Mock + public int getAutoAnalyzeTableWidthThreshold() { + return thresholds[count++]; + } + }; + Map>> testMap = new HashMap<>(); + StatisticsJobAppender appender = new StatisticsJobAppender(); + appender.appendToLowJobs(testMap); + Assertions.assertEquals(0, testMap.size()); + appender.setLastRoundFinishTime(0); + appender.appendToLowJobs(testMap); + Assertions.assertEquals(1, testMap.size()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/TableStatsMetaTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/TableStatsMetaTest.java index 94eab9e00cc501..10e1973aa3318d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/TableStatsMetaTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/TableStatsMetaTest.java @@ -19,27 +19,19 @@ import org.apache.doris.catalog.OlapTable; -import mockit.Mock; -import mockit.MockUp; import mockit.Mocked; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; +import java.util.HashSet; class TableStatsMetaTest { @Test void update(@Mocked OlapTable table) { - new MockUp() { - @Mock - public long getRowCount() { - return 4; - } - }; TableStatsMeta tableStatsMeta = new TableStatsMeta(); - AnalysisInfo jobInfo = new AnalysisInfoBuilder().setJobColumns(new ArrayList<>()) - .setColName("col1").build(); + AnalysisInfo jobInfo = new AnalysisInfoBuilder().setRowCount(4) + .setJobColumns(new HashSet<>()).setColName("col1").build(); tableStatsMeta.update(jobInfo, table); Assertions.assertEquals(4, tableStatsMeta.rowCount); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 724e0363833305..275471a66982b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -17,10 +17,21 @@ package org.apache.doris.statistics.util; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.statistics.AnalysisManager; +import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.ResultRow; +import org.apache.doris.statistics.TableStatsMeta; import com.google.common.collect.Lists; import mockit.Mock; @@ -33,6 +44,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Base64; +import java.util.List; class StatisticsUtilTest { @Test @@ -150,4 +162,141 @@ void testEscape() { // \\''"" Assertions.assertEquals("\\\\''\"", StatisticsUtil.escapeSQL(origin)); } + + @Test + void testNeedAnalyzeColumn() { + Column column = new Column("testColumn", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column); + OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); + // Test table stats meta is null. + new MockUp() { + @Mock + public TableStatsMeta findTableStatsStatus(long tblId) { + return null; + } + }; + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test user injected flag is set. + TableStatsMeta tableMeta = new TableStatsMeta(); + tableMeta.userInjected = true; + new MockUp() { + @Mock + public TableStatsMeta findTableStatsStatus(long tblId) { + return tableMeta; + } + }; + Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test column meta is null. + tableMeta.userInjected = false; + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + new MockUp() { + @Mock + public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { + return new ColStatsMeta(0, null, null, null, 0, 0, 0); + } + }; + + // Test not supported external table type. + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null); + Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(externalTable, Pair.of("index", column.getName()))); + + // Test hms external table not hive type. + new MockUp() { + @Mock + public DLAType getDlaType() { + return DLAType.ICEBERG; + } + }; + ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsExternalTable, Pair.of("index", column.getName()))); + + // Test partition first load. + new MockUp() { + @Mock + public boolean isPartitionColumn(String columnName) { + return true; + } + }; + tableMeta.newPartitionLoaded.set(true); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test empty table to non-empty table. + new MockUp() { + @Mock + public long getRowCount() { + return 100; + } + }; + tableMeta.newPartitionLoaded.set(false); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test non-empty table to empty table. + new MockUp() { + @Mock + public long getRowCount() { + return 0; + } + }; + new MockUp() { + @Mock + public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { + return new ColStatsMeta(0, null, null, null, 0, 100, 0); + } + }; + tableMeta.newPartitionLoaded.set(false); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test table still empty. + new MockUp() { + @Mock + public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { + return new ColStatsMeta(0, null, null, null, 0, 0, 0); + } + }; + tableMeta.newPartitionLoaded.set(false); + Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test row count changed more than threshold. + new MockUp() { + @Mock + public long getRowCount() { + return 1000; + } + }; + new MockUp() { + @Mock + public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { + return new ColStatsMeta(0, null, null, null, 0, 500, 0); + } + }; + tableMeta.newPartitionLoaded.set(false); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test update rows changed more than threshold. + new MockUp() { + @Mock + public long getRowCount() { + return 120; + } + }; + new MockUp() { + @Mock + public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { + return new ColStatsMeta(0, null, null, null, 0, 100, 80); + } + }; + tableMeta.newPartitionLoaded.set(false); + tableMeta.updatedRows.set(200); + Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + // Test update rows changed less than threshold + tableMeta.newPartitionLoaded.set(false); + tableMeta.updatedRows.set(100); + Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(table, Pair.of("index", column.getName()))); + + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index f8e53af077dd85..6ed7c23ec3c2f8 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1468,6 +1468,18 @@ struct TReportCommitTxnResultRequest { 4: optional binary payload } +struct TQueryColumn { + 1: optional string catalogId + 2: optional string dbId + 3: optional string tblId + 4: optional string colName +} + +struct TSyncQueryColumns { + 1: optional list highPriorityColumns; + 2: optional list midPriorityColumns; +} + service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) @@ -1558,4 +1570,5 @@ service FrontendService { TShowProcessListResult showProcessList(1: TShowProcessListRequest request) Status.TStatus reportCommitTxnResult(1: TReportCommitTxnResultRequest request) TShowUserResult showUser(1: TShowUserRequest request) + Status.TStatus syncQueryColumns(1: TSyncQueryColumns request) } diff --git a/regression-test/suites/external_table_p2/hive/test_hive_statistic_auto.groovy b/regression-test/suites/external_table_p2/hive/test_hive_statistic_auto.groovy index 8a7591daeb1b1e..eddf0bd8e7c16c 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_statistic_auto.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_statistic_auto.groovy @@ -32,7 +32,7 @@ suite("test_hive_statistic_auto", "p2,external,hive,external_remote,external_rem logger.info("catalog " + catalog_name + " created") // Test analyze table without init. - sql """analyze database ${catalog_name}.statistics PROPERTIES("use.auto.analyzer"="true")""" + sql """analyze table ${catalog_name}.statistics.statistics PROPERTIES("use.auto.analyzer"="true")""" sql """use ${catalog_name}.statistics""" for (int i = 0; i < 10; i++) { diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 7c6231aa48fd02..bcbec86b1a77b9 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -2774,7 +2774,7 @@ PARTITION `p599` VALUES IN (599) // Test auto analyze with job type SYSTEM sql """drop stats trigger_test""" - sql """analyze database trigger PROPERTIES("use.auto.analyzer"="true")""" + sql """analyze table trigger_test PROPERTIES("use.auto.analyzer"="true")""" int i = 0; for (0; i < 10; i++) { result = sql """show column stats trigger_test""" From f34114d35fe17e859ce12189f8d63d948b7173dd Mon Sep 17 00:00:00 2001 From: feiniaofeiafei <53502832+feiniaofeiafei@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:34:24 +0800 Subject: [PATCH 043/163] [Fix](nereids) fix rule merge_aggregate when has project (#33892) --- .../doris/nereids/jobs/executor/Rewriter.java | 4 +- .../nereids/rules/rewrite/MergeAggregate.java | 23 +++--- .../merge_aggregate/merge_aggregate.out | 51 ++++++++++++ .../merge_aggregate/merge_aggregate.groovy | 80 +++++++++++++++++++ 4 files changed, 148 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 51d2f4f44d0d55..335de322c614ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -304,7 +304,9 @@ public class Rewriter extends AbstractBatchJobExecutor { topic("Eliminate GroupBy", topDown(new EliminateGroupBy(), - new MergeAggregate()) + new MergeAggregate(), + // need to adjust min/max/sum nullable attribute after merge aggregate + new AdjustAggregateNullableForEmptySet()) ), topic("Eager aggregation", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeAggregate.java index 9a0b9f8b5e0353..a2c23dd9b412b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeAggregate.java @@ -34,10 +34,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -87,15 +89,14 @@ private Plan mergeTwoAggregate(LogicalAggregate> outerAgg private Plan mergeAggProjectAgg(LogicalAggregate>> outerAgg) { LogicalProject> project = outerAgg.child(); LogicalAggregate innerAgg = project.child(); - + List outputExpressions = outerAgg.getOutputExpressions(); + List replacedOutputExpressions = PlanUtils.replaceExpressionByProjections( + project.getProjects(), (List) outputExpressions); // rewrite agg function. e.g. max(max) - List aggFunc = outerAgg.getOutputExpressions().stream() + List replacedAggFunc = replacedOutputExpressions.stream() .filter(expr -> (expr instanceof Alias) && (expr.child(0) instanceof AggregateFunction)) .map(e -> rewriteAggregateFunction(e, innerAggExprIdToAggFunc)) .collect(Collectors.toList()); - // rewrite agg function directly refer to the slot below the project - List replacedAggFunc = PlanUtils.replaceExpressionByProjections(project.getProjects(), - (List) aggFunc); // replace groupByKeys directly refer to the slot below the project List replacedGroupBy = PlanUtils.replaceExpressionByProjections(project.getProjects(), outerAgg.getGroupByExpressions()); @@ -138,13 +139,17 @@ private NamedExpression rewriteAggregateFunction(NamedExpression e, } boolean commonCheck(LogicalAggregate outerAgg, LogicalAggregate innerAgg, - boolean sameGroupBy) { + boolean sameGroupBy, Optional projectOptional) { innerAggExprIdToAggFunc = innerAgg.getOutputExpressions().stream() .filter(expr -> (expr instanceof Alias) && (expr.child(0) instanceof AggregateFunction)) .collect(Collectors.toMap(NamedExpression::getExprId, value -> (AggregateFunction) value.child(0), (existValue, newValue) -> existValue)); Set aggregateFunctions = outerAgg.getAggregateFunctions(); - for (AggregateFunction outerFunc : aggregateFunctions) { + List replacedAggFunctions = projectOptional.map(project -> + (List) PlanUtils.replaceExpressionByProjections( + projectOptional.get().getProjects(), new ArrayList<>(aggregateFunctions))) + .orElse(new ArrayList<>(aggregateFunctions)); + for (AggregateFunction outerFunc : replacedAggFunctions) { if (!(ALLOW_MERGE_AGGREGATE_FUNCTIONS.contains(outerFunc.getName()))) { return false; } @@ -188,7 +193,7 @@ private boolean canMergeAggregateWithoutProject(LogicalAggregate>> outerAgg) { @@ -206,6 +211,6 @@ private boolean canMergeAggregateWithProject(LogicalAggregate Date: Fri, 26 Apr 2024 12:37:08 +0800 Subject: [PATCH 044/163] [fix](Nereids) column pruning should prune map in cte consumer (#34079) we save bi-map in cte consumer to get the maping between producer and consumer. the consumer's output is decided by the map in it. so, cte consumer should be output prunable, and should remove useless entry from map when do column pruning --- .../apache/doris/nereids/CascadesContext.java | 26 +--- .../doris/nereids/StatementContext.java | 41 ++--- .../translator/PhysicalPlanTranslator.java | 6 +- .../doris/nereids/jobs/executor/Rewriter.java | 4 +- .../apache/doris/nereids/rules/RuleType.java | 3 +- .../rewrite/CollectCteConsumerOutput.java | 41 +++++ .../rewrite/CollectProjectAboveConsumer.java | 81 ---------- .../nereids/rules/rewrite/ColumnPruning.java | 31 ++-- .../rules/rewrite/RewriteCteChildren.java | 17 ++- .../plans/logical/LogicalCTEConsumer.java | 22 ++- .../nereids_hint_tpcds_p0/shape/query1.out | 3 +- .../nereids_hint_tpcds_p0/shape/query24.out | 3 +- .../data/nereids_p0/hint/multi_leading.out | 6 +- .../shape/query1.out | 3 +- .../shape/query23.out | 12 +- .../shape/query24.out | 3 +- .../shape/query30.out | 3 +- .../shape/query31.out | 48 +++--- .../shape/query39.out | 10 +- .../shape/query47.out | 5 +- .../shape/query57.out | 5 +- .../shape/query59.out | 19 ++- .../shape/query70.out | 1 + .../shape/query81.out | 3 +- .../shape/query95.out | 6 +- .../constraints/query23.out | 12 +- .../noStatsRfPrune/query1.out | 3 +- .../noStatsRfPrune/query23.out | 12 +- .../noStatsRfPrune/query24.out | 3 +- .../noStatsRfPrune/query30.out | 3 +- .../noStatsRfPrune/query31.out | 40 ++--- .../noStatsRfPrune/query39.out | 10 +- .../noStatsRfPrune/query47.out | 5 +- .../noStatsRfPrune/query57.out | 5 +- .../noStatsRfPrune/query59.out | 19 ++- .../noStatsRfPrune/query70.out | 1 + .../noStatsRfPrune/query81.out | 3 +- .../noStatsRfPrune/query95.out | 6 +- .../no_stats_shape/query1.out | 3 +- .../no_stats_shape/query23.out | 12 +- .../no_stats_shape/query24.out | 3 +- .../no_stats_shape/query30.out | 3 +- .../no_stats_shape/query31.out | 40 ++--- .../no_stats_shape/query39.out | 10 +- .../no_stats_shape/query47.out | 5 +- .../no_stats_shape/query57.out | 5 +- .../no_stats_shape/query59.out | 19 ++- .../no_stats_shape/query81.out | 3 +- .../no_stats_shape/query95.out | 6 +- .../rf_prune/query1.out | 3 +- .../rf_prune/query23.out | 12 +- .../rf_prune/query24.out | 3 +- .../rf_prune/query30.out | 3 +- .../rf_prune/query31.out | 48 +++--- .../rf_prune/query39.out | 10 +- .../rf_prune/query47.out | 5 +- .../rf_prune/query57.out | 5 +- .../rf_prune/query59.out | 19 ++- .../rf_prune/query70.out | 1 + .../rf_prune/query81.out | 3 +- .../rf_prune/query95.out | 6 +- .../shape/query1.out | 3 +- .../shape/query23.out | 12 +- .../shape/query24.out | 3 +- .../shape/query30.out | 3 +- .../shape/query31.out | 48 +++--- .../shape/query39.out | 10 +- .../shape/query47.out | 5 +- .../shape/query57.out | 5 +- .../shape/query59.out | 19 ++- .../shape/query81.out | 3 +- .../shape/query95.out | 6 +- .../cte/test_cte_column_pruning.groovy | 143 ++++++++++++++++++ 73 files changed, 513 insertions(+), 498 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectCteConsumerOutput.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java create mode 100644 regression-test/suites/nereids_p0/cte/test_cte_column_pruning.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index dd569ef8f7519a..3b9ba912383e2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -50,7 +50,6 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; @@ -102,7 +101,7 @@ public class CascadesContext implements ScheduleContext { private Optional currentRootRewriteJobContext; // in optimize stage, the plan will storage in the memo private Memo memo; - private StatementContext statementContext; + private final StatementContext statementContext; private final CTEContext cteContext; private final RuleSet ruleSet; @@ -616,16 +615,6 @@ public void putCTEIdToConsumer(LogicalCTEConsumer cteConsumer) { consumers.add(cteConsumer); } - public void putCTEIdToProject(CTEId cteId, NamedExpression p) { - Set projects = this.statementContext.getCteIdToProjects() - .computeIfAbsent(cteId, k -> new HashSet<>()); - projects.add(p); - } - - public Set getProjectForProducer(CTEId cteId) { - return this.statementContext.getCteIdToProjects().get(cteId); - } - public Map> getCteIdToConsumers() { return this.statementContext.getCteIdToConsumers(); } @@ -639,17 +628,6 @@ public Map> getConsumerIdToFilters() { return this.statementContext.getConsumerIdToFilters(); } - public void markConsumerUnderProject(LogicalCTEConsumer cteConsumer) { - Set consumerIds = this.statementContext.getCteIdToConsumerUnderProjects() - .computeIfAbsent(cteConsumer.getCteId(), k -> new HashSet<>()); - consumerIds.add(cteConsumer.getRelationId()); - } - - public boolean couldPruneColumnOnProducer(CTEId cteId) { - Set consumerIds = this.statementContext.getCteIdToConsumerUnderProjects().get(cteId); - return consumerIds.size() == this.statementContext.getCteIdToConsumers().get(cteId).size(); - } - public void addCTEConsumerGroup(CTEId cteId, Group g, Map producerSlotToConsumerSlot) { List, Group>> consumerGroups = this.statementContext.getCteIdToConsumerGroup().computeIfAbsent(cteId, k -> new ArrayList<>()); @@ -746,7 +724,7 @@ public void printPlanProcess() { public static void printPlanProcess(List planProcesses) { for (PlanProcess row : planProcesses) { - LOG.info("RULE: " + row.ruleName + "\nBEFORE:\n" + row.beforeShape + "\nafter:\n" + row.afterShape); + LOG.info("RULE: {}\nBEFORE:\n{}\nafter:\n{}", row.ruleName, row.beforeShape, row.afterShape); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 819ff032cd6878..3274233c16be75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.ObjectId; @@ -54,7 +53,6 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; -import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -103,9 +101,8 @@ public class StatementContext implements Closeable { private final IdGenerator cteIdGenerator = CTEId.createGenerator(); private final Map> cteIdToConsumers = new HashMap<>(); - private final Map> cteIdToProjects = new HashMap<>(); + private final Map> cteIdToOutputIds = new HashMap<>(); private final Map> consumerIdToFilters = new HashMap<>(); - private final Map> cteIdToConsumerUnderProjects = new HashMap<>(); // Used to update consumer's stats private final Map, Group>>> cteIdToConsumerGroup = new HashMap<>(); private final Map rewrittenCteProducer = new HashMap<>(); @@ -134,12 +131,13 @@ public class StatementContext implements Closeable { private BitSet disableRules; // table locks - private Stack plannerResources = new Stack<>(); + private final Stack plannerResources = new Stack<>(); // for create view support in nereids // key is the start and end position of the sql substring that needs to be replaced, // and value is the new string used for replacement. - private TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + private final TreeMap, String> indexInSqlToString + = new TreeMap<>(new Pair.PairComparator<>()); public StatementContext() { this(ConnectContext.get(), null, 0); @@ -216,10 +214,6 @@ public Optional getSqlCacheContext() { return Optional.ofNullable(sqlCacheContext); } - public int getMaxContinuousJoin() { - return joinCount; - } - public Set getAllPathsSlots() { Set allSlotReferences = Sets.newHashSet(); for (Map, SlotReference> slotReferenceMap : subColumnSlotRefMap.values()) { @@ -240,19 +234,16 @@ public Slot getRewrittenSlotRefByOriginalExpr(Expression originalExpr) { * Add a slot ref attached with paths in context to avoid duplicated slot */ public void addPathSlotRef(Slot root, List paths, SlotReference slotRef, Expression originalExpr) { - subColumnSlotRefMap.computeIfAbsent(root, k -> Maps.newTreeMap(new Comparator>() { - @Override - public int compare(List lst1, List lst2) { - Iterator it1 = lst1.iterator(); - Iterator it2 = lst2.iterator(); - while (it1.hasNext() && it2.hasNext()) { - int result = it1.next().compareTo(it2.next()); - if (result != 0) { - return result; - } + subColumnSlotRefMap.computeIfAbsent(root, k -> Maps.newTreeMap((lst1, lst2) -> { + Iterator it1 = lst1.iterator(); + Iterator it2 = lst2.iterator(); + while (it1.hasNext() && it2.hasNext()) { + int result = it1.next().compareTo(it2.next()); + if (result != 0) { + return result; } - return Integer.compare(lst1.size(), lst2.size()); } + return Integer.compare(lst1.size(), lst2.size()); })); subColumnSlotRefMap.get(root).put(paths, slotRef); subColumnOriginalExprMap.put(slotRef, originalExpr); @@ -349,18 +340,14 @@ public Map> getCteIdToConsumers() { return cteIdToConsumers; } - public Map> getCteIdToProjects() { - return cteIdToProjects; + public Map> getCteIdToOutputIds() { + return cteIdToOutputIds; } public Map> getConsumerIdToFilters() { return consumerIdToFilters; } - public Map> getCteIdToConsumerUnderProjects() { - return cteIdToConsumerUnderProjects; - } - public Map, Group>>> getCteIdToConsumerGroup() { return cteIdToConsumerGroup; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 98bc581fe8ccd8..3184937e2e051d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1123,9 +1123,13 @@ public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer, // update expr to slot mapping TupleDescriptor tupleDescriptor = null; for (Slot producerSlot : cteProducer.getOutput()) { - Slot consumerSlot = cteConsumer.getProducerToConsumerSlotMap().get(producerSlot); SlotRef slotRef = context.findSlotRef(producerSlot.getExprId()); tupleDescriptor = slotRef.getDesc().getParent(); + Slot consumerSlot = cteConsumer.getProducerToConsumerSlotMap().get(producerSlot); + // consumerSlot could be null if we prune partial consumers' columns + if (consumerSlot == null) { + continue; + } context.addExprIdSlotRefPair(consumerSlot.getExprId(), slotRef); } CTEScanNode cteScanNode = new CTEScanNode(tupleDescriptor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 335de322c614ed..cf0d8a638d1b24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -44,8 +44,8 @@ import org.apache.doris.nereids.rules.rewrite.CheckMatchExpression; import org.apache.doris.nereids.rules.rewrite.CheckMultiDistinct; import org.apache.doris.nereids.rules.rewrite.CheckPrivileges; +import org.apache.doris.nereids.rules.rewrite.CollectCteConsumerOutput; import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer; -import org.apache.doris.nereids.rules.rewrite.CollectProjectAboveConsumer; import org.apache.doris.nereids.rules.rewrite.ColumnPruning; import org.apache.doris.nereids.rules.rewrite.ConvertInnerOrCrossJoin; import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite; @@ -418,7 +418,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topic("Push project and filter on cte consumer to cte producer", topDown( new CollectFilterAboveConsumer(), - new CollectProjectAboveConsumer() + new CollectCteConsumerOutput() ) ), topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index f1a797f4e2bc75..52e24827c316a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -306,8 +306,7 @@ public enum RuleType { COLLECT_FILTER(RuleTypeClass.REWRITE), COLLECT_JOIN_CONSTRAINT(RuleTypeClass.REWRITE), - COLLECT_PROJECT_ABOVE_CTE_CONSUMER(RuleTypeClass.REWRITE), - COLLECT_PROJECT_ABOVE_FILTER_CTE_CONSUMER(RuleTypeClass.REWRITE), + COLLECT_CTE_CONSUMER_OUTPUT(RuleTypeClass.REWRITE), LEADING_JOIN(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectCteConsumerOutput.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectCteConsumerOutput.java new file mode 100644 index 00000000000000..20b6dc40230b4b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectCteConsumerOutput.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Slot; + +import java.util.HashSet; +import java.util.Set; + +/** + * Collect outputs of CTE Consumer. + */ +public class CollectCteConsumerOutput extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalCTEConsumer().thenApply(ctx -> { + Set producerOutputs = ctx.statementContext + .getCteIdToOutputIds().computeIfAbsent(ctx.root.getCteId(), k -> new HashSet<>()); + producerOutputs.addAll(ctx.root.getProducerToConsumerOutputMap().keySet()); + return null; + }).toRule(RuleType.COLLECT_CTE_CONSUMER_OUTPUT); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java deleted file mode 100644 index 0ecace726dbcbc..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectProjectAboveConsumer.java +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -/** - * Collect Projects Above CTE Consumer. - */ -public class CollectProjectAboveConsumer implements RewriteRuleFactory { - - @Override - public List buildRules() { - return ImmutableList.of(RuleType.COLLECT_PROJECT_ABOVE_CTE_CONSUMER - .build(logicalProject(logicalCTEConsumer()).thenApply(ctx -> { - LogicalProject project = ctx.root; - List namedExpressions = project.getProjects(); - LogicalCTEConsumer cteConsumer = project.child(); - collectProject(ctx.cascadesContext, namedExpressions, cteConsumer); - return ctx.root; - })), - RuleType.COLLECT_PROJECT_ABOVE_FILTER_CTE_CONSUMER - .build(logicalProject(logicalFilter(logicalCTEConsumer())).thenApply(ctx -> { - LogicalProject> project = ctx.root; - LogicalFilter filter = project.child(); - Set filterSlots = filter.getInputSlots(); - List namedExpressions = new ArrayList<>(project.getProjects()); - for (Slot slot : filterSlots) { - if (!project.getOutput().contains(slot)) { - namedExpressions.add(slot); - } - } - collectProject(ctx.cascadesContext, namedExpressions, filter.child()); - return ctx.root; - })) - ); - } - - private static void collectProject(CascadesContext ctx, - List namedExpressions, LogicalCTEConsumer cteConsumer) { - for (Expression expr : namedExpressions) { - expr.foreach(node -> { - if (!(node instanceof Slot)) { - return; - } - Slot slot = cteConsumer.getProducerSlot((Slot) node); - ctx.putCTEIdToProject(cteConsumer.getCteId(), slot); - ctx.markConsumerUnderProject(cteConsumer); - }); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java index e36c0f5172ad70..4cb18e8a380177 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; @@ -200,13 +201,21 @@ public Plan visitLogicalRepeat(LogicalRepeat repeat, PruneContex return pruneAggregate(repeat, context); } - private Plan pruneAggregate(Aggregate agg, PruneContext context) { - // first try to prune group by and aggregate functions - Aggregate prunedOutputAgg = pruneOutput(agg, agg.getOutputs(), agg::pruneOutputs, context); + @Override + public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProducer, PruneContext context) { + return skipPruneThisAndFirstLevelChildren(cteProducer); + } - Aggregate fillUpAggr = fillUpGroupByAndOutput(prunedOutputAgg); + @Override + public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, PruneContext context) { + return super.visitLogicalCTEConsumer(cteConsumer, context); + } - return pruneChildren(fillUpAggr); + private Plan pruneAggregate(Aggregate agg, PruneContext context) { + // first try to prune group by and aggregate functions + Aggregate prunedOutputAgg = pruneOutput(agg, agg.getOutputs(), agg::pruneOutputs, context); + Aggregate fillUpAggregate = fillUpGroupByAndOutput(prunedOutputAgg); + return pruneChildren(fillUpAggregate); } private Plan skipPruneThisAndFirstLevelChildren(Plan plan) { @@ -217,7 +226,7 @@ private Plan skipPruneThisAndFirstLevelChildren(Plan plan) { return pruneChildren(plan, requireAllOutputOfChildren.build()); } - private static Aggregate fillUpGroupByAndOutput(Aggregate prunedOutputAgg) { + private static Aggregate fillUpGroupByAndOutput(Aggregate prunedOutputAgg) { List groupBy = prunedOutputAgg.getGroupByExpressions(); List output = prunedOutputAgg.getOutputExpressions(); @@ -239,12 +248,11 @@ private static Aggregate fillUpGroupByAndOutput(Aggregate prunedOutp ImmutableList.Builder newGroupByExprList = ImmutableList.builderWithExpectedSize(newOutputList.size()); for (NamedExpression e : newOutputList) { - if (!(aggregateFunctions.contains(e) - || (e instanceof Alias && aggregateFunctions.contains(e.child(0))))) { + if (!(e instanceof Alias && aggregateFunctions.contains(e.child(0)))) { newGroupByExprList.add(e); } } - return ((LogicalAggregate) prunedOutputAgg).withGroupByAndOutput( + return ((LogicalAggregate) prunedOutputAgg).withGroupByAndOutput( newGroupByExprList.build(), newOutputList); } @@ -371,11 +379,6 @@ private Plan doPruneChild(Plan plan, Plan child, Set childRequiredSlots) { return prunedChild; } - @Override - public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProducer, PruneContext context) { - return skipPruneThisAndFirstLevelChildren(cteProducer); - } - /** PruneContext */ public static class PruneContext { public Set requiredSlots; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java index 72a4603fadc949..3a2da623b4c4d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteCteChildren.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; @@ -41,7 +42,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.apache.commons.collections.CollectionUtils; import java.util.HashSet; import java.util.List; @@ -109,10 +109,17 @@ public Plan visitLogicalCTEProducer(LogicalCTEProducer cteProduc } else { child = (LogicalPlan) cteProducer.child(); child = tryToConstructFilter(cascadesContext, cteProducer.getCteId(), child); - Set projects = cascadesContext.getProjectForProducer(cteProducer.getCteId()); - if (CollectionUtils.isNotEmpty(projects) - && cascadesContext.couldPruneColumnOnProducer(cteProducer.getCteId())) { - child = new LogicalProject<>(ImmutableList.copyOf(projects), child); + Set producerOutputs = cascadesContext.getStatementContext() + .getCteIdToOutputIds().get(cteProducer.getCteId()); + if (producerOutputs.size() < child.getOutput().size()) { + ImmutableList.Builder projectsBuilder + = ImmutableList.builderWithExpectedSize(producerOutputs.size()); + for (Slot slot : child.getOutput()) { + if (producerOutputs.contains(slot)) { + projectsBuilder.add(slot); + } + } + child = new LogicalProject<>(projectsBuilder.build(), child); child = pushPlanUnderAnchor(child); } CascadesContext rewrittenCtx = CascadesContext.newSubtreeContext( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 71b1c43f791191..5fd088a2bba196 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; @@ -36,6 +37,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; @@ -43,7 +45,7 @@ * LogicalCTEConsumer */ //TODO: find cte producer and propagate its functional dependencies -public class LogicalCTEConsumer extends LogicalRelation implements BlockFuncDepsPropagation { +public class LogicalCTEConsumer extends LogicalRelation implements BlockFuncDepsPropagation, OutputPrunable { private final String name; private final CTEId cteId; @@ -145,6 +147,24 @@ public List computeOutput() { return ImmutableList.copyOf(producerToConsumerOutputMap.values()); } + @Override + public Plan pruneOutputs(List prunedOutputs) { + Map consumerToProducerOutputMap = new LinkedHashMap<>(this.consumerToProducerOutputMap.size()); + Map producerToConsumerOutputMap = new LinkedHashMap<>(this.consumerToProducerOutputMap.size()); + for (Entry consumerToProducerSlot : this.consumerToProducerOutputMap.entrySet()) { + if (prunedOutputs.contains(consumerToProducerSlot.getKey())) { + consumerToProducerOutputMap.put(consumerToProducerSlot.getKey(), consumerToProducerSlot.getValue()); + producerToConsumerOutputMap.put(consumerToProducerSlot.getValue(), consumerToProducerSlot.getKey()); + } + } + return withTwoMaps(consumerToProducerOutputMap, producerToConsumerOutputMap); + } + + @Override + public List getOutputs() { + return (List) this.getOutput(); + } + public CTEId getCteId() { return cteId; } diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query1.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query1.out index e0104b54a423b2..996ccd7623c2a7 100644 --- a/regression-test/data/nereids_hint_tpcds_p0/shape/query1.out +++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query1.out @@ -24,8 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] --------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------PhysicalProject -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalProject ----------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ctr_store_sk] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query24.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query24.out index f37faa9bff4cf8..2e2658db1620a8 100644 --- a/regression-test/data/nereids_hint_tpcds_p0/shape/query24.out +++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query24.out @@ -47,8 +47,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecGather] ------------------hashAgg[LOCAL] --------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------PhysicalProject -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecReplicated] --------------PhysicalProject ----------------hashAgg[GLOBAL] diff --git a/regression-test/data/nereids_p0/hint/multi_leading.out b/regression-test/data/nereids_p0/hint/multi_leading.out index 51ecab29494cac..71db5aec524b10 100644 --- a/regression-test/data/nereids_p0/hint/multi_leading.out +++ b/regression-test/data/nereids_p0/hint/multi_leading.out @@ -613,8 +613,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = cte.c11)) otherCondition=() ------------------------PhysicalOlapScan[t1] ------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !sql5_2 -- PhysicalCteAnchor ( cteId=CTEId#0 ) @@ -639,8 +638,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = cte.c11)) otherCondition=() ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------PhysicalDistribute[DistributionSpecReplicated] --------------------------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query1.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query1.out index b2b5a87ac2bc84..2317f9435be541 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query1.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query1.out @@ -39,6 +39,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out index b937f23400b9a3..0475cae9f9522a 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out @@ -59,8 +59,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[item_sk] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4 ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject --------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() @@ -73,13 +72,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF6 ws_item_sk->[item_sk] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject --------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() @@ -92,6 +89,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out index 83f4e91b4a29f1..ebdb27b0c5d023 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out @@ -55,6 +55,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecGather] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query30.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query30.out index 7dcac891ad1dc1..7272f6c9e2641c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query30.out @@ -43,6 +43,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query31.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query31.out index 11e812ae39b681..f22860f874de84 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query31.out @@ -2,44 +2,42 @@ -- !ds_shape_31 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalProject +------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------------PhysicalProject +----------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 +--------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------PhysicalProject +------------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +--------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] +--PhysicalCteAnchor ( cteId=CTEId#1 ) +----PhysicalCteProducer ( cteId=CTEId#1 ) ------hashAgg[GLOBAL] --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk] +--------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ws_bill_addr_sk] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] ----------------------PhysicalProject -------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 +------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +--------------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) ----------------------------PhysicalOlapScan[date_dim] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] ---PhysicalCteAnchor ( cteId=CTEId#1 ) -----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ws_bill_addr_sk] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 -------------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------------PhysicalProject -----------------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) -------------------------------PhysicalOlapScan[date_dim] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer_address] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query39.out index 2fb33f5848e1cc..9d2e1eb162d5d0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query39.out @@ -29,11 +29,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalQuickSort[LOCAL_SORT] ----------hashJoin[INNER_JOIN] hashCondition=((inv1.i_item_sk = inv2.i_item_sk) and (inv1.w_warehouse_sk = inv2.w_warehouse_sk)) otherCondition=() ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv1.d_moy = 1)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv1.d_moy = 1)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv2.d_moy = 2)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv2.d_moy = 2)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out index 1ff72561a11a74..e57e35ba769342 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out @@ -41,9 +41,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out index 628a6aa98a3da7..79c67bde5cb36b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out @@ -41,9 +41,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query59.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query59.out index ca30f76b1d7829..84bd2d432ac7f4 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query59.out @@ -2,17 +2,16 @@ -- !ds_shape_59 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------PhysicalProject +----------------PhysicalOlapScan[store_sales] apply RFs: RF0 +--------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[date_dim] +------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query70.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query70.out index fdbfa12a4b73f7..5eb9fba4824825 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query70.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query70.out @@ -45,3 +45,4 @@ PhysicalResultSink --------------------------------------------------PhysicalDistribute[DistributionSpecReplicated] ----------------------------------------------------PhysicalProject ------------------------------------------------------PhysicalOlapScan[store] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query81.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query81.out index 20a169a0faa968..61f4343fd1f0a0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query81.out @@ -44,6 +44,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out index b0a0655caff31e..c6f2d22db1554c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out @@ -23,16 +23,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number] --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5 RF6 +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5 RF6 --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 ----------------------PhysicalProject ------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF7 ws_order_number->[ws_order_number,ws_order_number] --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk] ------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out index ddff36aebc71b7..8668943e20f864 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/constraints/query23.out @@ -69,11 +69,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() ----------------------PhysicalDistribute[DistributionSpecHash] @@ -88,9 +86,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out index 65dca9e89f3eb1..8528bf7b49b992 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out @@ -37,6 +37,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] --------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------PhysicalProject -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out index 431330e9039d15..22f483bb5a7b93 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out @@ -65,11 +65,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 ------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) @@ -84,11 +82,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 ------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query24.out index b9a89aae5e9bbf..9699ebf9255d22 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query24.out @@ -52,6 +52,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecGather] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query30.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query30.out index 985c714b0ab82e..61524dd01387bc 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query30.out @@ -39,8 +39,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject ------------------filter((customer_address.ca_state = 'IN')) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query31.out index 3cacf0ae184f1e..0852d3abe2ed20 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query31.out @@ -2,30 +2,30 @@ -- !ds_shape_31 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +--------------PhysicalProject ----------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() ------------------PhysicalProject --------------------PhysicalOlapScan[store_sales] apply RFs: RF1 ------------------PhysicalDistribute[DistributionSpecReplicated] --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) -----------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalProject +------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ws_sold_date_sk] +------hashAgg[GLOBAL] +--------PhysicalDistribute[DistributionSpecHash] +----------hashAgg[LOCAL] +------------PhysicalProject +--------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ws_sold_date_sk] +----------------PhysicalProject ------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() --------------------PhysicalDistribute[DistributionSpecHash] ----------------------PhysicalProject @@ -33,10 +33,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer_address] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) -------------------------PhysicalOlapScan[date_dim] +----------------PhysicalDistribute[DistributionSpecReplicated] +------------------PhysicalProject +--------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) +----------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out index 9ffc6dc5e07fc4..421fa8a749997d 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out @@ -28,11 +28,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalQuickSort[LOCAL_SORT] ----------hashJoin[INNER_JOIN] hashCondition=((inv1.i_item_sk = inv2.i_item_sk) and (inv1.w_warehouse_sk = inv2.w_warehouse_sk)) otherCondition=() ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv1.d_moy = 1)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv1.d_moy = 1)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv2.d_moy = 2)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv2.d_moy = 2)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out index fb20900a0b67c3..430c3c4067384f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out @@ -46,7 +46,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out index 18d9a45e7d6755..ed1d2952975370 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out @@ -46,7 +46,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query59.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query59.out index e871b1a1a44480..3347fd02ab46e0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query59.out @@ -2,17 +2,16 @@ -- !ds_shape_59 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() +--------------PhysicalProject +----------------PhysicalOlapScan[store_sales] +--------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[date_dim] +------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query70.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query70.out index abdcd1b0149063..d3e7d441cec28c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query70.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query70.out @@ -45,3 +45,4 @@ PhysicalResultSink ------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[store] + diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query81.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query81.out index ac9cf29ee123d4..e9fea1c43c2cff 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query81.out @@ -40,8 +40,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------PhysicalProject ------------------filter((customer_address.ca_state = 'CA')) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out index b042e3531742ae..3cc3f5843b2b08 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out @@ -21,8 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF6 ws_order_number->[ws_order_number] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF5 web_site_sk->[ws_web_site_sk] --------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[ws_ship_addr_sk] @@ -31,8 +30,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() ------------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------------PhysicalProject -----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2 diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out index e1300825fc29a8..422dff53364487 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out @@ -37,6 +37,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] --------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------PhysicalProject -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out index 049a85a8860ef2..662fc3aa2dd7a8 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out @@ -65,11 +65,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 ------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) @@ -84,11 +82,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 ------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------PhysicalProject -----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +--------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query24.out index 9f6191dac84b55..15252933e01d4c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query24.out @@ -52,6 +52,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecGather] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query30.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query30.out index bfa6bc4fe650b8..ccd62e114b44e7 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query30.out @@ -39,8 +39,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject ------------------filter((customer_address.ca_state = 'IN')) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query31.out index 5a0036ae4d5b8d..65a1a9afe0c750 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query31.out @@ -2,30 +2,30 @@ -- !ds_shape_31 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +--------------PhysicalProject ----------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[ss_addr_sk] ------------------PhysicalProject --------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ------------------PhysicalDistribute[DistributionSpecReplicated] --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) -----------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalProject +------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ws_sold_date_sk] +------hashAgg[GLOBAL] +--------PhysicalDistribute[DistributionSpecHash] +----------hashAgg[LOCAL] +------------PhysicalProject +--------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ws_sold_date_sk] +----------------PhysicalProject ------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_bill_addr_sk] --------------------PhysicalDistribute[DistributionSpecHash] ----------------------PhysicalProject @@ -33,10 +33,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer_address] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) -------------------------PhysicalOlapScan[date_dim] +----------------PhysicalDistribute[DistributionSpecReplicated] +------------------PhysicalProject +--------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) +----------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out index 40f877acac568b..d1d9fb39429644 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out @@ -28,11 +28,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalQuickSort[LOCAL_SORT] ----------hashJoin[INNER_JOIN] hashCondition=((inv1.i_item_sk = inv2.i_item_sk) and (inv1.w_warehouse_sk = inv2.w_warehouse_sk)) otherCondition=() ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv1.d_moy = 1)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv1.d_moy = 1)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv2.d_moy = 2)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv2.d_moy = 2)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out index 59d526865c49a0..788b686c26f4e1 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out @@ -46,7 +46,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out index 52cd80d56cc1de..e2c13ea729c532 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out @@ -46,7 +46,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query59.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query59.out index 6027c75690447d..fd888cc3382785 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query59.out @@ -2,17 +2,16 @@ -- !ds_shape_59 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------PhysicalProject +----------------PhysicalOlapScan[store_sales] apply RFs: RF0 +--------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[date_dim] +------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query81.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query81.out index 6463028a8fec4c..fb68f6ce1a3ddb 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query81.out @@ -40,8 +40,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------PhysicalProject ------------------filter((customer_address.ca_state = 'CA')) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out index 9e96715c5e4990..4fd762ec994fdd 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out @@ -21,8 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF12 ws_order_number->[ws_order_number];RF13 ws_order_number->[ws_order_number] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF10 web_site_sk->[ws_web_site_sk];RF11 web_site_sk->[ws_web_site_sk] --------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF8 ca_address_sk->[ws_ship_addr_sk];RF9 ca_address_sk->[ws_ship_addr_sk] @@ -31,8 +30,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF2 wr_order_number->[ws_order_number];RF3 wr_order_number->[ws_order_number] ------------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------------PhysicalProject -----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 RF3 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 RF3 ------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF4 RF5 diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query1.out index eca7d46a8f3ef1..8996d789efa954 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query1.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query1.out @@ -39,6 +39,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out index 8132fd343d5fa8..4d4463732f09c3 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out @@ -59,8 +59,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[item_sk] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4 ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject --------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() @@ -73,13 +72,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF6 ws_item_sk->[item_sk] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject --------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() @@ -92,6 +89,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out index 67183c68f45617..ddf2dfd6c057b2 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out @@ -55,6 +55,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecGather] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query30.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query30.out index 9f72a79a16ece1..40293a67cb991d 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query30.out @@ -43,6 +43,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query31.out index 835238e3d707ce..21bd99fb9f31d4 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query31.out @@ -2,44 +2,42 @@ -- !ds_shape_31 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalProject +------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------------PhysicalProject +----------------------PhysicalOlapScan[store_sales] apply RFs: RF0 +--------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------PhysicalProject +------------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] +--PhysicalCteAnchor ( cteId=CTEId#1 ) +----PhysicalCteProducer ( cteId=CTEId#1 ) ------hashAgg[GLOBAL] --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() +--------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] ----------------------PhysicalProject -------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 +------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) ----------------------------PhysicalOlapScan[date_dim] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] ---PhysicalCteAnchor ( cteId=CTEId#1 ) -----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 -------------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------------PhysicalProject -----------------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) -------------------------------PhysicalOlapScan[date_dim] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer_address] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out index 7a0a69965bdccd..11ec8af267cb77 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out @@ -28,11 +28,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalQuickSort[LOCAL_SORT] ----------hashJoin[INNER_JOIN] hashCondition=((inv1.i_item_sk = inv2.i_item_sk) and (inv1.w_warehouse_sk = inv2.w_warehouse_sk)) otherCondition=() ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv1.d_moy = 1)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv1.d_moy = 1)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv2.d_moy = 2)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv2.d_moy = 2)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out index 174fd05d61dc04..03cb37e0f810f3 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query47.out @@ -41,9 +41,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out index d09ed9ca40fe42..555d7716af6512 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out @@ -42,9 +42,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query59.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query59.out index 644b59d4db3312..b9e7a9b40de441 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query59.out @@ -2,17 +2,16 @@ -- !ds_shape_59 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() +--------------PhysicalProject +----------------PhysicalOlapScan[store_sales] +--------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[date_dim] +------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query70.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query70.out index f3e524aabcfe08..6fc8a52f8398e0 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query70.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query70.out @@ -45,3 +45,4 @@ PhysicalResultSink --------------------------------------------------PhysicalDistribute[DistributionSpecReplicated] ----------------------------------------------------PhysicalProject ------------------------------------------------------PhysicalOlapScan[store] + diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query81.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query81.out index 9b9a03af8fe25e..22ab8efaf190e3 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query81.out @@ -44,6 +44,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out index a835868fd8c78e..2a0ae9b4138686 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out @@ -23,16 +23,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 ----------------------PhysicalProject ------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF7 ws_order_number->[ws_order_number,ws_order_number] --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk] ------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out index eca7d46a8f3ef1..8996d789efa954 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query1.out @@ -39,6 +39,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out index 520c3910152f74..edd849ea421d13 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out @@ -59,8 +59,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[item_sk] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF4 ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject --------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() @@ -73,13 +72,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) ------------------PhysicalProject --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() build RFs:RF6 ws_item_sk->[item_sk] ----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject --------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() @@ -92,6 +89,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------PhysicalProject ---------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) +------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out index cf64374e507aa6..c0d202025b646f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out @@ -55,6 +55,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalDistribute[DistributionSpecGather] ----------------------hashAgg[LOCAL] ------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalProject -----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out index 0160329ec263ee..2880145be2057e 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query30.out @@ -43,6 +43,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out index f12c5e5cb28f78..f759ca84798c82 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query31.out @@ -2,44 +2,42 @@ -- !ds_shape_31 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalProject +------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------------PhysicalProject +----------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 +--------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------PhysicalProject +------------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------------PhysicalOlapScan[date_dim] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] +--PhysicalCteAnchor ( cteId=CTEId#1 ) +----PhysicalCteProducer ( cteId=CTEId#1 ) ------hashAgg[GLOBAL] --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk] +--------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ws_bill_addr_sk] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] ----------------------PhysicalProject -------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 +------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) ----------------------------PhysicalOlapScan[date_dim] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] ---PhysicalCteAnchor ( cteId=CTEId#1 ) -----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ws_bill_addr_sk] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 -------------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------------PhysicalProject -----------------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) -------------------------------PhysicalOlapScan[date_dim] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[customer_address] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out index 4ccedd314026d1..88b2869175c9a1 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out @@ -28,11 +28,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalQuickSort[LOCAL_SORT] ----------hashJoin[INNER_JOIN] hashCondition=((inv1.i_item_sk = inv2.i_item_sk) and (inv1.w_warehouse_sk = inv2.w_warehouse_sk)) otherCondition=() ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv1.d_moy = 1)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv1.d_moy = 1)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((inv2.d_moy = 2)) -------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------filter((inv2.d_moy = 2)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out index 214cdaaee62239..e8f28d6ea4e8cf 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query47.out @@ -41,9 +41,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out index ea7531482a7ee3..f479209035e353 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out @@ -42,9 +42,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) -------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out index 08d5a9d3f1f018..2db7d57863df6b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query59.out @@ -2,17 +2,16 @@ -- !ds_shape_59 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +--------------PhysicalProject +----------------PhysicalOlapScan[store_sales] apply RFs: RF0 +--------------PhysicalDistribute[DistributionSpecReplicated] ----------------PhysicalProject -------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[date_dim] +------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out index aa637bc6469b2d..99cf6c48bb75e7 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query81.out @@ -44,6 +44,5 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] ----------------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out index 4763e6aa34cf49..9a971d30aec2de 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out @@ -23,16 +23,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF10 wr_order_number->[ws_order_number];RF11 wr_order_number->[ws_order_number] --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF10 RF11 RF12 RF13 +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF10 RF11 RF12 RF13 --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_returns] apply RFs: RF12 RF13 ----------------------PhysicalProject ------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF14 ws_order_number->[ws_order_number,ws_order_number];RF15 ws_order_number->[ws_order_number,ws_order_number] --------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject -------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF6 web_site_sk->[ws_web_site_sk];RF7 web_site_sk->[ws_web_site_sk] ------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_ship_date_sk];RF5 d_date_sk->[ws_ship_date_sk] diff --git a/regression-test/suites/nereids_p0/cte/test_cte_column_pruning.groovy b/regression-test/suites/nereids_p0/cte/test_cte_column_pruning.groovy new file mode 100644 index 00000000000000..7a465c212518f2 --- /dev/null +++ b/regression-test/suites/nereids_p0/cte/test_cte_column_pruning.groovy @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_cte_column_pruning") { + sql "SET enable_nereids_planner=true" + sql "SET enable_pipeline_engine=true" + sql "SET enable_fallback_to_original_planner=false" + + sql """drop table if exists t1""" + sql """drop table if exists t2""" + sql """drop table if exists t3""" + sql """drop table if exists t4""" + + sql """ + create table if not exists t1 ( + c2 int , + c1 int , + c3 int , + c4 int , + pk int + ) + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ + create table if not exists t2 ( + c1 int , + c2 int , + c3 int , + c4 int , + pk int + ) + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ + create table if not exists t3 ( + c2 int , + c1 int , + c3 int , + c4 int , + pk int + ) + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ + create table if not exists t4 ( + c1 int , + c2 int , + c3 int , + c4 int , + pk int + ) + distributed by hash(pk) buckets 10 + properties("replication_num" = "1"); + """ + + sql """ + insert into t1(pk,c1,c2,c3,c4) values (0,7,2,3328056,7),(1,3,5,3,3045349),(2,2130015,0,7,-7116176),(3,4411710,1203314,1,2336164),(4,4,-8001461,0,8),(5,9,3,6,2),(6,-8088092,null,-7256698,-2025142),(7,8,2,5,1),(8,4,4953685,3,null),(9,-6662413,-3845449,4,2),(10,5315281,0,5,null),(11,9,3,7,7),(12,4341905,null,null,8),(13,3,6,5,1),(14,5,9,6541164,3),(15,1,-582319,1,9),(16,5533636,4,39841,0),(17,1,1,null,7),(18,742881,-1420303,6,1),(19,281430,6753011,3,2),(20,7,1,4,-31350),(21,-5663089,9,2278262,9),(22,6,0,2706409,3),(23,-3841713,9,3,9),(24,1,6,3,4059303),(25,0,1,-5700982,3),(26,5,0,6,1),(27,7,2,2,4),(28,6,-2140815,-8190613,6),(29,-8214516,4,3,6),(30,4393731,null,7,2),(31,-2524331,8,2,9),(32,5,1,7,3),(33,2,968001,-1718546,0),(34,9,2,null,-7682164),(35,5,-3302521,8,2),(36,2,1325975,null,2826927),(37,-6607898,null,4,8),(38,7,3,5284408,-265983),(39,1,null,2,-559197),(40,9,7,2,6),(41,-7193680,null,3,8),(42,-4800310,8,9,5),(43,0,8,0,-2429477),(44,-1007106,-7583038,9,2627388),(45,7,-6572230,4,-1789489),(46,8,4,null,7837867),(47,7,8,7,null),(48,8,-2618403,2723851,3),(49,1,3,1,0),(50,null,3241893,0,8),(51,1934849,-1353430,1,9),(52,5148268,6,null,1),(53,null,3922713,4,47559),(54,2038005,-7625242,null,-5606136),(55,4,449100,2108275,5147506),(56,5,5,4316929,null),(57,5049864,null,4,9),(58,null,7,2,9),(59,5,2,5,7),(60,9,9,5,-2774033),(61,4,0,6,1),(62,5,-7700238,6,3),(63,658183,-7933445,1,4),(64,8,8,-7019658,-7873828),(65,1,1,null,0),(66,1,2,9,7320481),(67,3,2099077,9,3),(68,-7120763,276954,0,4),(69,9,5,5170840,null),(70,null,6,220899,-5774478),(71,null,null,3,6),(72,7,2,8101877,null),(73,1,null,5,-5141920),(74,8,-7143195,0,6),(75,6,5,3388863,4),(76,6,6,-8015259,1),(77,5207959,-4325820,791546,7),(78,2,4411975,2,null),(79,9,2379417,8,3),(80,3,null,-6968517,-336360),(81,null,0,5,1),(82,3,0,6,-4536269),(83,2,7,0,7),(84,1,7,1,5),(85,3,3,7509217,2920951),(86,6,null,8,3),(87,9,8,8,5941004),(88,8023576,1036293,9,2),(89,5,3,1,5),(90,5,5,6,2170127),(91,null,1,7,null),(92,-5659717,4,null,6),(93,848413,9,-2742042,4980140),(94,1,9,467168,9),(95,6,6,4783371,-5096980),(96,3,2,4,3),(97,3,2,2,1),(98,8,0,-6734149,2),(99,4985816,3,null,8); + """ + + sql """ + insert into t2(pk,c1,c2,c3,c4) values (0,5,4,189864,-7663457),(1,7,null,6,1),(2,null,8,-3362640,9),(3,3,2,5,-2197130),(4,2,3,7160615,1),(5,null,-57834,420441,3),(6,0,null,2,2),(7,1,-3681539,3,4),(8,548866,3,0,5),(9,8,-2824887,0,3246956),(10,5,3,7,2),(11,8,8,6,8),(12,0,2,7,9),(13,8,6,null,null),(14,-4103729,4,5,8),(15,-3659292,2,7,5),(16,8,7,1,null),(17,2526018,4,8069607,5),(18,6,6,5,2802235),(19,9,0,6379201,null),(20,3,null,4,3),(21,0,8,-5506402,2),(22,6,4,3,1),(23,4,5225086,3,1),(24,-211796,2,0,null),(25,5,2,-4100572,7),(26,2345127,2,null,1),(27,8,2,4893754,2),(28,null,-5580446,4,0),(29,3,1,2,6); + """ + + sql """ + insert into t3(pk,c1,c2,c3,c4) values (0,3,2,6,-3164679),(1,-6216443,3437690,-288827,6),(2,4,-5352286,-1005469,4118240),(3,9,6795167,5,1616205),(4,8,-4659990,-4816829,6),(5,0,9,4,8),(6,-4454766,2,2510766,3),(7,7860071,-3434966,8,3),(8,null,0,2,1),(9,8031908,2,-6673194,-5981416),(10,5,6716310,8,2529959),(11,null,-3622116,1,-7891010),(12,null,3527222,7993802,null),(13,null,1,2,1),(14,2,8,7,7),(15,0,9,5,null),(16,7452083,null,-4620796,0),(17,9,9,null,6),(18,3,1,-1578776,5),(19,9,2532045,-3577349,null); + """ + + sql """ + insert into t4(pk,c1,c2,c3,c4) values (0,-4263513,null,null,6),(1,1,3,4,null),(2,2460936,6,5,6299003),(3,null,7,7107446,-2366754),(4,6247611,4785035,3,-8014875),(5,0,2,5249218,3),(6,null,253825,4,3),(7,null,2,9,-350785),(8,6,null,null,4),(9,1,3,1,3422691),(10,0,-6596165,1808018,3),(11,2,752342,null,1),(12,-5220927,2676278,9,7),(13,6025864,2,1,4),(14,7,4,4,9),(15,5,9,9,849881),(16,-4253076,null,-4404479,-6365351),(17,null,6,4240023,3),(18,7,1276495,7,6),(19,null,-4459040,178194,-6974337),(20,6,2498738,9,6),(21,8,-1047876,8,-3519551),(22,4477868,6,3,-7237985),(23,9,1,null,7),(24,null,2,-6996324,4),(25,2,2,-7965145,2),(26,5339549,6,null,4),(27,0,4,4,4),(28,null,6563965,-5816143,2),(29,4,7245227,3239886,1),(30,9,9,-8134757,0),(31,-1787881,7769609,8306001,null),(32,-1817246,1,3,-8163782),(33,7,4018844,0,4),(34,null,5,3,4),(35,8,-1698017,0,3024748),(36,2,7,5330073,3654557),(37,null,null,1,7),(38,6,9,0,2),(39,-3988946,-1465296,3,3),(40,4939439,null,null,3),(41,6,-7235968,1,0),(42,5141520,-7389145,8,1),(43,5,89342,1,0),(44,1,641063,9,4718353),(45,5,4,4,6),(46,2,6,4,4),(47,3,2,2,-7137584),(48,6735548,0,1,7),(49,6,4,7,-4864341); + """ + + sql """ + sync + """ + + sql """ + WITH tbl1 AS ( + SELECT + tbl2.c1 AS c1, + tbl3.c2 AS c2, + tbl5.c4 AS c3, + tbl3.c1 AS c4 + FROM + t1 AS tbl1 + JOIN t2 AS tbl2 ON tbl1.c2 = tbl1.c4 + RIGHT JOIN t1 AS tbl3 ON tbl3.c3 = tbl1.c3 + INNER JOIN t3 AS tbl4 ON tbl3.c2 = tbl4.c3 + INNER JOIN t4 AS tbl5 ON tbl5.c4 = tbl4.c2 + AND tbl3.c3 = tbl3.c4 + WHERE + ( + tbl2.c4 = (0 + 9) + AND ((4 + 1) IS NULL) + ) + ORDER BY + 1, + 2, + 3, + 4 DESC + LIMIT + 6666 OFFSET 500 + ) + SELECT + tbl3.c4 AS c1, + tbl2.c4 AS c2, + tbl3.c3 AS c3, + tbl2.c2 AS c4 + FROM + tbl1 AS tbl2 + JOIN tbl1 AS tbl3 ON tbl3.c2 = tbl2.c2 + WHERE + ( + tbl2.c3 != tbl2.c3 + AND ((2 + 0) IS NOT NULL) + ) + ORDER BY + 2, + 4, + 1, + 3 ASC + LIMIT + 6666 OFFSET 2; + """ +} From d6a8e24a82b314d4831168fc6211cf3ebd9fb2b0 Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Fri, 26 Apr 2024 12:52:17 +0800 Subject: [PATCH 045/163] [Fix](cloud) Fix concurrency bugs on creating auto partition #34135 --- .../transaction/CloudGlobalTransactionMgr.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 35b3cd285298d3..aa1218ab9b3f3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -385,12 +385,17 @@ private Set getBaseTabletsFromTables(List
tableList, List tabletIds = tabletCommitInfos.stream().map(TabletCommitInfo::getTabletId).collect(Collectors.toSet()); baseTabletIds.retainAll(tabletIds); From a28499c53fa80c065ef8d9b689126f79f986dad0 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:06:48 +0800 Subject: [PATCH 046/163] Implement HLL with 128 buckets to support statistics cache. (#34124) --- .../java/org/apache/doris/common/io/Hll.java | 15 +- .../org/apache/doris/common/io/HllTest.java | 4 +- .../apache/doris/statistics/util/Hll128.java | 214 ++++++++++++++++++ .../doris/statistics/util/Hll128Test.java | 204 +++++++++++++++++ 4 files changed, 432 insertions(+), 5 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/statistics/util/Hll128.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/statistics/util/Hll128Test.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/io/Hll.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/Hll.java index b00912598d1220..f70df61749855d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/io/Hll.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/Hll.java @@ -46,7 +46,7 @@ public class Hll { public static final int HLL_COLUMN_PRECISION = 14; public static final int HLL_ZERO_COUNT_BITS = (64 - HLL_COLUMN_PRECISION); - public static final int HLL_EXPLICLIT_INT64_NUM = 160; + public static final int HLL_EXPLICIT_INT64_NUM = 160; public static final int HLL_SPARSE_THRESHOLD = 4096; public static final int HLL_REGISTERS_COUNT = 16 * 1024; @@ -122,7 +122,7 @@ public void update(long hashValue) { type = HLL_DATA_EXPLICIT; break; case HLL_DATA_EXPLICIT: - if (hashSet.size() < HLL_EXPLICLIT_INT64_NUM) { + if (hashSet.size() < HLL_EXPLICIT_INT64_NUM) { hashSet.add(hashValue); break; } @@ -157,7 +157,7 @@ public void merge(Hll other) { switch (other.type) { // CHECKSTYLE IGNORE THIS LINE: missing switch default case HLL_DATA_EXPLICIT: this.hashSet.addAll(other.hashSet); - if (this.hashSet.size() > HLL_EXPLICLIT_INT64_NUM) { + if (this.hashSet.size() > HLL_EXPLICIT_INT64_NUM) { convertExplicitToRegister(); this.type = HLL_DATA_FULL; } @@ -393,4 +393,13 @@ public int getType() { return type; } + // For convert to statistics used Hll128 + public byte[] getRegisters() { + return registers; + } + + // For convert to statistics used Hll128 + public Set getHashSet() { + return hashSet; + } } diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/io/HllTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/io/HllTest.java index fabc7c1f8da70f..94333f255a657d 100644 --- a/fe/fe-common/src/test/java/org/apache/doris/common/io/HllTest.java +++ b/fe/fe-common/src/test/java/org/apache/doris/common/io/HllTest.java @@ -55,11 +55,11 @@ public void hllBasicTest() throws IOException { // test explicit Hll explicitHll = new Hll(); - for (int i = 0; i < Hll.HLL_EXPLICLIT_INT64_NUM; i++) { + for (int i = 0; i < Hll.HLL_EXPLICIT_INT64_NUM; i++) { explicitHll.updateWithHash(i); } Assert.assertTrue(explicitHll.getType() == Hll.HLL_DATA_EXPLICIT); - Assert.assertTrue(explicitHll.estimateCardinality() == Hll.HLL_EXPLICLIT_INT64_NUM); + Assert.assertTrue(explicitHll.estimateCardinality() == Hll.HLL_EXPLICIT_INT64_NUM); ByteArrayOutputStream explicitOutputStream = new ByteArrayOutputStream(); DataOutput explicitOutput = new DataOutputStream(explicitOutputStream); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/Hll128.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/Hll128.java new file mode 100644 index 00000000000000..094cff4b7d9def --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/Hll128.java @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.util; + +import org.apache.doris.common.io.Hll; + +import java.math.BigInteger; +import java.util.HashSet; +import java.util.Set; + +/** + * This is an HLL implementation with 128 Buckets. + * Mainly used for statistics partition ndv cache. + * We can convert the org.apache.doris.common.io.Hll object with 16K buckets to Hll128 to reduce memory consumption. + */ +public class Hll128 { + + public static final byte HLL_DATA_EMPTY = 0; + public static final byte HLL_DATA_EXPLICIT = 1; + public static final byte HLL_DATA_FULL = 3; + + public static final int HLL_COLUMN_PRECISION = 7; + public static final int HLL_ZERO_COUNT_BITS = (64 - HLL_COLUMN_PRECISION); + public static final int HLL_EXPLICLIT_INT64_NUM = 160; + public static final int HLL_REGISTERS_COUNT = 128; + + private int type; + private Set hashSet; + private byte[] registers; + + public Hll128() { + type = Hll.HLL_DATA_EMPTY; + this.hashSet = new HashSet<>(); + } + + private void convertExplicitToRegister() { + assert this.type == HLL_DATA_EXPLICIT; + registers = new byte[HLL_REGISTERS_COUNT]; + for (Long value : hashSet) { + updateRegisters(value); + } + hashSet.clear(); + } + + private void updateRegisters(long hashValue) { + int idx; + // hash value less than zero means we get a unsigned long + // so need to transfer to BigInter to mod + if (hashValue < 0) { + BigInteger unint64HashValue = new BigInteger(Long.toUnsignedString(hashValue)); + unint64HashValue = unint64HashValue.mod(new BigInteger(Long.toUnsignedString(HLL_REGISTERS_COUNT))); + idx = unint64HashValue.intValue(); + } else { + idx = (int) (hashValue % HLL_REGISTERS_COUNT); + } + + hashValue >>>= HLL_COLUMN_PRECISION; + hashValue |= (1L << HLL_ZERO_COUNT_BITS); + byte firstOneBit = (byte) (Hll.getLongTailZeroNum(hashValue) + 1); + registers[idx] = registers[idx] > firstOneBit ? registers[idx] : firstOneBit; + } + + private void mergeRegisters(byte[] other) { + for (int i = 0; i < HLL_REGISTERS_COUNT; i++) { + this.registers[i] = this.registers[i] > other[i] ? this.registers[i] : other[i]; + } + } + + public void update(long hashValue) { + switch (this.type) { // CHECKSTYLE IGNORE THIS LINE: missing switch default + case HLL_DATA_EMPTY: + hashSet.add(hashValue); + type = HLL_DATA_EXPLICIT; + break; + case HLL_DATA_EXPLICIT: + if (hashSet.size() < HLL_EXPLICLIT_INT64_NUM) { + hashSet.add(hashValue); + break; + } + convertExplicitToRegister(); + type = HLL_DATA_FULL; + case HLL_DATA_FULL: // CHECKSTYLE IGNORE THIS LINE: fall through + updateRegisters(hashValue); + break; + } + } + + public void merge(Hll128 other) { + if (other.type == HLL_DATA_EMPTY) { + return; + } + switch (this.type) { // CHECKSTYLE IGNORE THIS LINE: missing switch default + case HLL_DATA_EMPTY: + this.type = other.type; + switch (other.type) { // CHECKSTYLE IGNORE THIS LINE: missing switch default + case HLL_DATA_EXPLICIT: + this.hashSet.addAll(other.hashSet); + break; + case HLL_DATA_FULL: + this.registers = new byte[HLL_REGISTERS_COUNT]; + System.arraycopy(other.registers, 0, this.registers, 0, HLL_REGISTERS_COUNT); + break; + } + break; + case HLL_DATA_EXPLICIT: + switch (other.type) { // CHECKSTYLE IGNORE THIS LINE: missing switch default + case HLL_DATA_EXPLICIT: + this.hashSet.addAll(other.hashSet); + if (this.hashSet.size() > HLL_EXPLICLIT_INT64_NUM) { + convertExplicitToRegister(); + this.type = HLL_DATA_FULL; + } + break; + case HLL_DATA_FULL: + convertExplicitToRegister(); + mergeRegisters(other.registers); + this.type = HLL_DATA_FULL; + break; + } + break; + case HLL_DATA_FULL: + switch (other.type) { // CHECKSTYLE IGNORE THIS LINE: missing switch default + case HLL_DATA_EXPLICIT: + for (long value : other.hashSet) { + update(value); + } + break; + case HLL_DATA_FULL: + mergeRegisters(other.registers); + break; + } + break; + } + } + + // use strictfp to force java follow IEEE 754 to deal float point strictly + public strictfp long estimateCardinality() { + if (type == HLL_DATA_EMPTY) { + return 0; + } + if (type == HLL_DATA_EXPLICIT) { + return hashSet.size(); + } + + int numStreams = HLL_REGISTERS_COUNT; + float alpha = 0.7213f / (1 + 1.079f / numStreams); + float harmonicMean = 0; + int numZeroRegisters = 0; + + for (int i = 0; i < HLL_REGISTERS_COUNT; i++) { + harmonicMean += Math.pow(2.0f, -registers[i]); + + if (registers[i] == 0) { + numZeroRegisters++; + } + } + + harmonicMean = 1.0f / harmonicMean; + double estimate = alpha * numStreams * numStreams * harmonicMean; + + if (estimate <= numStreams * 2.5 && numZeroRegisters != 0) { + estimate = numStreams * Math.log(((float) numStreams) / ((float) numZeroRegisters)); + } + + return (long) (estimate + 0.5); + } + + public int getType() { + return type; + } + + public static Hll128 fromHll(Hll hll) { + Hll128 hll128 = new Hll128(); + if (hll == null || hll.getType() == Hll.HLL_DATA_EMPTY) { + return hll128; + } + if (hll.getType() == Hll.HLL_DATA_EXPLICIT) { + hll128.type = HLL_DATA_EXPLICIT; + hll128.hashSet.addAll(hll.getHashSet()); + return hll128; + } + + byte[] registers = hll.getRegisters(); + byte[] registers128 = new byte[HLL_REGISTERS_COUNT]; + int groupSize = Hll.HLL_REGISTERS_COUNT / HLL_REGISTERS_COUNT; + for (int i = 0; i < HLL_REGISTERS_COUNT; i++) { + for (int j = 0; j < groupSize; j++) { + registers128[i] = registers128[i] < registers[i * groupSize + j] + ? registers[i * groupSize + j] + : registers128[i]; + } + } + hll128.registers = registers128; + hll128.type = HLL_DATA_FULL; + return hll128; + } + +} + diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/Hll128Test.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/Hll128Test.java new file mode 100644 index 00000000000000..d692b90ce3171b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/Hll128Test.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.util; + +import org.apache.doris.common.io.Hll; + +import org.apache.commons.codec.binary.StringUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class Hll128Test { + + @Test + public void basicTest() { + // test empty + Hll128 emptyHll = new Hll128(); + Assert.assertEquals(Hll128.HLL_DATA_EMPTY, emptyHll.getType()); + Assert.assertEquals(0, emptyHll.estimateCardinality()); + + // test explicit + Hll128 explicitHll = new Hll128(); + for (int i = 0; i < Hll.HLL_EXPLICIT_INT64_NUM; i++) { + explicitHll.update(i); + } + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicitHll.getType()); + Assert.assertEquals(Hll.HLL_EXPLICIT_INT64_NUM, explicitHll.estimateCardinality()); + + // test full + Hll128 fullHll = new Hll128(); + for (int i = 1; i <= Short.MAX_VALUE; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + fullHll.update(Hll.hash64(v, v.length, Hll.SEED)); + } + Assert.assertEquals(Hll.HLL_DATA_FULL, fullHll.getType()); + Assert.assertEquals(33141, fullHll.estimateCardinality()); + Assert.assertTrue(fullHll.estimateCardinality() > Short.MAX_VALUE * (1 - 0.1) + && fullHll.estimateCardinality() < Short.MAX_VALUE * (1 + 0.1)); + + } + + @Test + public void testFromHll() throws IOException { + // test empty + Hll emptyHll = new Hll(); + Hll128 hll128 = Hll128.fromHll(emptyHll); + Assert.assertEquals(Hll128.HLL_DATA_EMPTY, hll128.getType()); + Assert.assertEquals(0, hll128.estimateCardinality()); + + // test explicit + Hll explicitHll = new Hll(); + for (int i = 0; i < Hll.HLL_EXPLICIT_INT64_NUM; i++) { + explicitHll.updateWithHash(i); + } + hll128 = Hll128.fromHll(explicitHll); + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, hll128.getType()); + Assert.assertEquals(Hll.HLL_EXPLICIT_INT64_NUM, hll128.estimateCardinality()); + + // test full + Hll fullHll = new Hll(); + for (int i = 0; i < 10000; i++) { + fullHll.updateWithHash(i); + } + hll128 = Hll128.fromHll(fullHll); + Assert.assertEquals(Hll128.HLL_DATA_FULL, hll128.getType()); + Assert.assertTrue(hll128.estimateCardinality() > 9000 && hll128.estimateCardinality() < 11000); + } + + @Test + public void testMerge() throws IOException { + // test empty merge empty + Hll128 empty1 = new Hll128(); + Hll128 empty2 = new Hll128(); + empty1.merge(empty2); + Assert.assertEquals(Hll128.HLL_DATA_EMPTY, empty1.getType()); + Assert.assertEquals(0, empty1.estimateCardinality()); + + // test empty merge explicit + Hll128 empty = new Hll128(); + Hll128 explicit = new Hll128(); + for (int i = 1; i < Hll.HLL_EXPLICIT_INT64_NUM; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit.update(Hll.hash64(v, v.length, Hll.SEED)); + } + empty.merge(explicit); + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, empty.getType()); + Assert.assertEquals(Hll.HLL_EXPLICIT_INT64_NUM - 1, empty.estimateCardinality()); + + // test empty merge full + empty = new Hll128(); + Hll128 full = new Hll128(); + for (int i = 1; i < 10000; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + full.update(Hll.hash64(v, v.length, Hll.SEED)); + } + empty.merge(full); + Assert.assertEquals(Hll128.HLL_DATA_FULL, empty.getType()); + Assert.assertTrue(empty.estimateCardinality() > 9000 && empty.estimateCardinality() < 11000); + + // test explicit merge empty + empty = new Hll128(); + explicit = new Hll128(); + for (int i = 1; i < Hll.HLL_EXPLICIT_INT64_NUM; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit.update(Hll.hash64(v, v.length, Hll.SEED)); + } + explicit.merge(empty); + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicit.getType()); + Assert.assertEquals(Hll.HLL_EXPLICIT_INT64_NUM - 1, explicit.estimateCardinality()); + + // test explicit merge explicit + Hll128 explicit1 = new Hll128(); + Hll128 explicit2 = new Hll128(); + for (int i = 0; i < 10; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit1.update(Hll.hash64(v, v.length, Hll.SEED)); + } + for (int i = 0; i < 30; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit2.update(Hll.hash64(v, v.length, Hll.SEED)); + } + explicit1.merge(explicit2); + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicit1.getType()); + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicit2.getType()); + Assert.assertEquals(30, explicit1.estimateCardinality()); + + explicit2 = new Hll128(); + for (int i = 10001; i < 10000 + Hll.HLL_EXPLICIT_INT64_NUM; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit2.update(Hll.hash64(v, v.length, Hll.SEED)); + } + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicit1.getType()); + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicit2.getType()); + explicit1.merge(explicit2); + Assert.assertEquals(Hll128.HLL_DATA_FULL, explicit1.getType()); + Assert.assertTrue(explicit1.estimateCardinality() > 170 && explicit1.estimateCardinality() < 210); + + // Test explicit merge full + explicit = new Hll128(); + for (int i = 0; i < 10; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit.update(Hll.hash64(v, v.length, Hll.SEED)); + } + full = new Hll128(); + for (int i = 1; i < 10000; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + full.update(Hll.hash64(v, v.length, Hll.SEED)); + } + Assert.assertEquals(Hll128.HLL_DATA_FULL, full.getType()); + explicit.merge(full); + Assert.assertEquals(Hll128.HLL_DATA_FULL, explicit.getType()); + Assert.assertTrue(explicit.estimateCardinality() > 9000 && explicit.estimateCardinality() < 11000); + + // Test full merge explicit + explicit = new Hll128(); + for (int i = 0; i < 10; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + explicit.update(Hll.hash64(v, v.length, Hll.SEED)); + } + full = new Hll128(); + for (int i = 1; i < 10000; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + full.update(Hll.hash64(v, v.length, Hll.SEED)); + } + Assert.assertEquals(Hll128.HLL_DATA_EXPLICIT, explicit.getType()); + full.merge(explicit); + Assert.assertEquals(Hll128.HLL_DATA_FULL, full.getType()); + Assert.assertTrue(full.estimateCardinality() > 9000 && full.estimateCardinality() < 11000); + + // Test full merge full + Hll128 full1 = new Hll128(); + Hll128 full2 = new Hll128(); + for (int i = 1; i < 10000; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + full1.update(Hll.hash64(v, v.length, Hll.SEED)); + } + for (int i = 5000; i < 15000; i++) { + byte[] v = StringUtils.getBytesUtf8(String.valueOf(i)); + full2.update(Hll.hash64(v, v.length, Hll.SEED)); + } + Assert.assertEquals(Hll128.HLL_DATA_FULL, full1.getType()); + Assert.assertEquals(Hll128.HLL_DATA_FULL, full2.getType()); + full1.merge(full2); + Assert.assertEquals(Hll128.HLL_DATA_FULL, full1.getType()); + Assert.assertTrue(full1.estimateCardinality() > 13500 && full1.estimateCardinality() < 16500); + } + +} From fcc7e079dc9103de8e99c77bd3cd86d88c9c252e Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 26 Apr 2024 13:43:08 +0800 Subject: [PATCH 047/163] [Test](hive-writer) Adjust test_hive_write_partitions regression test to resolve special characters issue with git on windows. (#34026) --- .../scripts/create_preinstalled_table.hql | 13 +-- .../varchar_col=varchar_value1/000000_0 | Bin .../varchar_col=varchar_value1/000000_0 | Bin .../varchar_col=varchar_value1/000000_0 | Bin .../hive/write/test_hive_write_partitions.out | 105 +++++++++--------- .../write/test_hive_write_partitions.groovy | 2 +- 6 files changed, 58 insertions(+), 62 deletions(-) rename docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/{binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-21/timestamp_col=2024-03-21 12%3A00%3A00/char_col=char_value1 => date_col=2024-03-21/char_col=char_value1}/varchar_col=varchar_value1/000000_0 (100%) rename docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/{binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-22/timestamp_col=2024-03-22 12%3A00%3A00/char_col=char_value1 => date_col=2024-03-22/char_col=char_value1}/varchar_col=varchar_value1/000000_0 (100%) rename docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=123456.789012/string_col=string_value/{binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-20/timestamp_col=2024-03-20 12%3A00%3A00/char_col=char_value1 => date_col=2024-03-20/char_col=char_value1}/varchar_col=varchar_value1/000000_0 (100%) diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql index dbeeab972f6b0d..0ab4a204b84ce0 100644 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql @@ -2406,10 +2406,8 @@ CREATE TABLE `all_partition_types2_parquet_snappy_src`( PARTITIONED BY ( `decimal_col` decimal(18,6), `string_col` string, - `binary_col` binary, `date_col` date, - `timestamp_col` timestamp, - `char_col` char(50), + `char_col` char(11), `varchar_col` varchar(50)) stored as parquet LOCATION @@ -2423,10 +2421,8 @@ CREATE TABLE `all_partition_types2_parquet_snappy`( PARTITIONED BY ( `decimal_col` decimal(18,6), `string_col` string, - `binary_col` binary, `date_col` date, - `timestamp_col` timestamp, - `char_col` char(50), + `char_col` char(11), `varchar_col` varchar(50)) stored as parquet TBLPROPERTIES('parquet.compression'='SNAPPY'); @@ -2437,10 +2433,9 @@ CREATE TABLE `all_partition_types2_orc_zlib`( PARTITIONED BY ( `decimal_col` decimal(18,6), `string_col` string, - `binary_col` binary, `date_col` date, - `timestamp_col` timestamp, - `char_col` char(50), + `char_col` char(11), `varchar_col` varchar(50)) stored as orc TBLPROPERTIES("orc.compress"="ZLIB"); + diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-21/timestamp_col=2024-03-21 12%3A00%3A00/char_col=char_value1 /varchar_col=varchar_value1/000000_0 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/date_col=2024-03-21/char_col=char_value1/varchar_col=varchar_value1/000000_0 similarity index 100% rename from docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-21/timestamp_col=2024-03-21 12%3A00%3A00/char_col=char_value1 /varchar_col=varchar_value1/000000_0 rename to docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/date_col=2024-03-21/char_col=char_value1/varchar_col=varchar_value1/000000_0 diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-22/timestamp_col=2024-03-22 12%3A00%3A00/char_col=char_value1 /varchar_col=varchar_value1/000000_0 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/date_col=2024-03-22/char_col=char_value1/varchar_col=varchar_value1/000000_0 similarity index 100% rename from docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-22/timestamp_col=2024-03-22 12%3A00%3A00/char_col=char_value1 /varchar_col=varchar_value1/000000_0 rename to docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=-123456.789012/string_col=string_value/date_col=2024-03-22/char_col=char_value1/varchar_col=varchar_value1/000000_0 diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=123456.789012/string_col=string_value/binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-20/timestamp_col=2024-03-20 12%3A00%3A00/char_col=char_value1 /varchar_col=varchar_value1/000000_0 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=123456.789012/string_col=string_value/date_col=2024-03-20/char_col=char_value1/varchar_col=varchar_value1/000000_0 similarity index 100% rename from docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=123456.789012/string_col=string_value/binary_col=62 69 6e 61 72 79 5f 76 61 6c 75 65/date_col=2024-03-20/timestamp_col=2024-03-20 12%3A00%3A00/char_col=char_value1 /varchar_col=varchar_value1/000000_0 rename to docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/all_partition_types2_parquet_snappy_src/decimal_col=123456.789012/string_col=string_value/date_col=2024-03-20/char_col=char_value1/varchar_col=varchar_value1/000000_0 diff --git a/regression-test/data/external_table_p0/hive/write/test_hive_write_partitions.out b/regression-test/data/external_table_p0/hive/write/test_hive_write_partitions.out index c0e695544e2b34..1a81e07a2f3141 100644 --- a/regression-test/data/external_table_p0/hive/write/test_hive_write_partitions.out +++ b/regression-test/data/external_table_p0/hive/write/test_hive_write_partitions.out @@ -41,27 +41,27 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 7 true 127 32767 2147483647 9223372036854775807 123.45 123456.789 -- !q01 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 -- !q02 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 -- !q03 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q04 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q05 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -7 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +7 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q01 -- 1 \N -128 \N -2147483648 \N -123.45 \N @@ -130,27 +130,27 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 7 true 127 32767 2147483647 9223372036854775807 123.45 123456.789 -- !q01 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 -- !q02 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 -- !q03 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q04 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q05 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -7 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +7 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q01 -- 1 \N -128 \N -2147483648 \N -123.45 \N @@ -219,27 +219,27 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 7 true 127 32767 2147483647 9223372036854775807 123.45 123456.789 -- !q01 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 -- !q02 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 -- !q03 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q04 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q05 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -7 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +7 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q01 -- 1 \N -128 \N -2147483648 \N -123.45 \N @@ -308,27 +308,27 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 7 true 127 32767 2147483647 9223372036854775807 123.45 123456.789 -- !q01 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 -- !q02 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 -- !q03 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q04 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 -3 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 +3 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q05 -- -1 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-22 2024-03-22T12:00 char_value1 varchar_value1 -2 -123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-21 2024-03-21T12:00 char_value1 varchar_value1 -7 123456.789012 string_value 62 69 6e 61 72 79 5f 76 61 6c 75 65 2024-03-20 2024-03-20T12:00 char_value1 varchar_value1 +1 -123456.789012 string_value 2024-03-22 char_value1 varchar_value1 +2 -123456.789012 string_value 2024-03-21 char_value1 varchar_value1 +7 123456.789012 string_value 2024-03-20 char_value1 varchar_value1 -- !q01 -- 1 \N -128 \N -2147483648 \N -123.45 \N @@ -354,3 +354,4 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 3 \N 127 \N 2147483647 \N 123.45 \N 3 \N 127 \N 2147483647 \N 123.45 \N 7 true 127 32767 2147483647 9223372036854775807 123.45 123456.789 + diff --git a/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy b/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy index 695b608e1521a5..0435a68b09a42a 100644 --- a/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy +++ b/regression-test/suites/external_table_p0/hive/write/test_hive_write_partitions.groovy @@ -134,7 +134,7 @@ suite("test_hive_write_partitions", "p0,external,hive,external_docker,external_d sql """ INSERT OVERWRITE TABLE all_partition_types2_${format_compression}_${catalog_name}_q03 - SELECT CAST(7 as INT) as id, decimal_col, string_col, binary_col, date_col, timestamp_col, char_col, varchar_col FROM all_partition_types2_parquet_snappy_src where id = 3; + SELECT CAST(7 as INT) as id, decimal_col, string_col, date_col, char_col, varchar_col FROM all_partition_types2_parquet_snappy_src where id = 3; """ order_qt_q05 """ select * from all_partition_types2_${format_compression}_${catalog_name}_q03; """ From 9bffc246959af69af494b90a6fdf581bea4e5df5 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 26 Apr 2024 13:47:43 +0800 Subject: [PATCH 048/163] [Improvement](pipeline) Use hash shuffle for 1-phase Agg/Analytic ope rator #34122 --- be/src/pipeline/exec/aggregation_sink_operator.cpp | 11 +++++++---- be/src/pipeline/exec/aggregation_sink_operator.h | 2 +- .../exec/partitioned_aggregation_sink_operator.cpp | 6 ++++-- .../exec/partitioned_aggregation_sink_operator.h | 2 +- .../pipeline_x/pipeline_x_fragment_context.cpp | 10 ++++++++-- .../pipeline/pipeline_x/pipeline_x_fragment_context.h | 2 ++ 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/be/src/pipeline/exec/aggregation_sink_operator.cpp b/be/src/pipeline/exec/aggregation_sink_operator.cpp index fd88b0d15218b0..6c9d27e2a2b063 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.cpp +++ b/be/src/pipeline/exec/aggregation_sink_operator.cpp @@ -616,7 +616,7 @@ void AggSinkLocalState::_init_hash_method(const vectorized::VExprContextSPtrs& p } AggSinkOperatorX::AggSinkOperatorX(ObjectPool* pool, int operator_id, const TPlanNode& tnode, - const DescriptorTbl& descs) + const DescriptorTbl& descs, bool require_bucket_distribution) : DataSinkOperatorX(operator_id, tnode.node_id), _intermediate_tuple_id(tnode.agg_node.intermediate_tuple_id), _intermediate_tuple_desc(nullptr), @@ -629,9 +629,12 @@ AggSinkOperatorX::AggSinkOperatorX(ObjectPool* pool, int operator_id, const TPla _limit(tnode.limit), _have_conjuncts((tnode.__isset.vconjunct && !tnode.vconjunct.nodes.empty()) || (tnode.__isset.conjuncts && !tnode.conjuncts.empty())), - _partition_exprs(tnode.__isset.distribute_expr_lists ? tnode.distribute_expr_lists[0] - : std::vector {}), - _is_colocate(tnode.agg_node.__isset.is_colocate && tnode.agg_node.is_colocate), + _partition_exprs(require_bucket_distribution ? (tnode.__isset.distribute_expr_lists + ? tnode.distribute_expr_lists[0] + : std::vector {}) + : tnode.agg_node.grouping_exprs), + _is_colocate(tnode.agg_node.__isset.is_colocate && tnode.agg_node.is_colocate && + require_bucket_distribution), _agg_fn_output_row_descriptor(descs, tnode.row_tuples, tnode.nullable_tuples) {} Status AggSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* state) { diff --git a/be/src/pipeline/exec/aggregation_sink_operator.h b/be/src/pipeline/exec/aggregation_sink_operator.h index b3ffa19d6db791..0c34acfd7dfe84 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.h +++ b/be/src/pipeline/exec/aggregation_sink_operator.h @@ -143,7 +143,7 @@ class AggSinkLocalState : public PipelineXSinkLocalState { class AggSinkOperatorX final : public DataSinkOperatorX { public: AggSinkOperatorX(ObjectPool* pool, int operator_id, const TPlanNode& tnode, - const DescriptorTbl& descs); + const DescriptorTbl& descs, bool require_bucket_distribution); ~AggSinkOperatorX() override = default; Status init(const TDataSink& tsink) override { return Status::InternalError("{} should not init with TPlanNode", diff --git a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp index 78079a0ddf8c98..7eb09555aa8ab4 100644 --- a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp +++ b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.cpp @@ -122,9 +122,11 @@ void PartitionedAggSinkLocalState::update_profile(RuntimeProfile* child_profile) PartitionedAggSinkOperatorX::PartitionedAggSinkOperatorX(ObjectPool* pool, int operator_id, const TPlanNode& tnode, - const DescriptorTbl& descs) + const DescriptorTbl& descs, + bool require_bucket_distribution) : DataSinkOperatorX(operator_id, tnode.node_id) { - _agg_sink_operator = std::make_unique(pool, operator_id, tnode, descs); + _agg_sink_operator = std::make_unique(pool, operator_id, tnode, descs, + require_bucket_distribution); } Status PartitionedAggSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* state) { diff --git a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h index 1755cd866f270f..1233f66b56294c 100644 --- a/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h +++ b/be/src/pipeline/exec/partitioned_aggregation_sink_operator.h @@ -294,7 +294,7 @@ class PartitionedAggSinkLocalState class PartitionedAggSinkOperatorX : public DataSinkOperatorX { public: PartitionedAggSinkOperatorX(ObjectPool* pool, int operator_id, const TPlanNode& tnode, - const DescriptorTbl& descs); + const DescriptorTbl& descs, bool require_bucket_distribution); ~PartitionedAggSinkOperatorX() override = default; Status init(const TDataSink& tsink) override { return Status::InternalError("{} should not init with TPlanNode", diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp index bf2c255a1274f0..fc0234c62904b6 100644 --- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp +++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp @@ -1034,14 +1034,16 @@ Status PipelineXFragmentContext::_create_operator(ObjectPool* pool, const TPlanN DataSinkOperatorXPtr sink; if (_runtime_state->enable_agg_spill() && !tnode.agg_node.grouping_exprs.empty()) { sink.reset(new PartitionedAggSinkOperatorX(pool, next_sink_operator_id(), tnode, - descs)); + descs, _require_bucket_distribution)); } else { - sink.reset(new AggSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); + sink.reset(new AggSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, + _require_bucket_distribution)); } sink->set_dests_id({op->operator_id()}); RETURN_IF_ERROR(cur_pipe->set_sink(sink)); RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); } + _require_bucket_distribution = true; break; } case TPlanNodeType::HASH_JOIN_NODE: { @@ -1106,6 +1108,7 @@ Status PipelineXFragmentContext::_create_operator(ObjectPool* pool, const TPlanN _pipeline_parent_map.push(op->node_id(), cur_pipe); _pipeline_parent_map.push(op->node_id(), build_side_pipe); } + _require_bucket_distribution = true; break; } case TPlanNodeType::CROSS_JOIN_NODE: { @@ -1211,6 +1214,7 @@ Status PipelineXFragmentContext::_create_operator(ObjectPool* pool, const TPlanN sink->set_dests_id({op->operator_id()}); RETURN_IF_ERROR(cur_pipe->set_sink(sink)); RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); + _require_bucket_distribution = true; break; } case TPlanNodeType::INTERSECT_NODE: { @@ -1268,6 +1272,8 @@ Status PipelineXFragmentContext::_create_operator(ObjectPool* pool, const TPlanN print_plan_node_type(tnode.node_type)); } + _require_bucket_distribution = true; + return Status::OK(); } // NOLINTEND(readability-function-cognitive-complexity) diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h index 31febc0d8aaf4d..c87f8f4f784051 100644 --- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h +++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h @@ -239,6 +239,8 @@ class PipelineXFragmentContext : public PipelineFragmentContext { // Total instance num running on all BEs int _total_instances = -1; + + bool _require_bucket_distribution = false; }; } // namespace pipeline From 95a1be4adf11df9f69518c467e35e989ff6ee746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E5=B0=8F=E5=88=9A?= Date: Fri, 26 Apr 2024 13:51:30 +0800 Subject: [PATCH 049/163] [opt](parquet)Skip page with offset index (#33082) Make skip_page() in ColumnChunkReader more efficient. No more reading page headers if there are pagelocations in chunk. --- .../parquet/vparquet_column_chunk_reader.cpp | 82 ++--- .../parquet/vparquet_column_chunk_reader.h | 11 +- .../format/parquet/vparquet_column_reader.cpp | 7 +- .../format/parquet/vparquet_column_reader.h | 22 +- .../format/parquet/vparquet_group_reader.cpp | 12 +- .../format/parquet/vparquet_group_reader.h | 1 - .../format/parquet/vparquet_page_reader.cpp | 15 +- .../format/parquet/vparquet_page_reader.h | 123 +++++++- .../exec/format/parquet/vparquet_reader.cpp | 11 +- .../vec/exec/format/parquet/vparquet_reader.h | 3 +- .../vec/exec/parquet/parquet_thrift_test.cpp | 3 +- .../hive/test_hive_parquet_skip_page.out | 289 ++++++++++++++++++ .../hive/test_hive_parquet_skip_page.groovy | 131 ++++++++ 13 files changed, 641 insertions(+), 69 deletions(-) create mode 100644 regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp index 6feb9bc1025b33..af30e63d1e35d9 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp @@ -47,12 +47,14 @@ namespace doris::vectorized { ColumnChunkReader::ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, + const tparquet::OffsetIndex* offset_index, cctz::time_zone* ctz, io::IOContext* io_ctx) : _field_schema(field_schema), _max_rep_level(field_schema->repetition_level), _max_def_level(field_schema->definition_level), _stream_reader(reader), _metadata(column_chunk->meta_data), + _offset_index(offset_index), // _ctz(ctz), _io_ctx(io_ctx) {} @@ -61,7 +63,9 @@ Status ColumnChunkReader::init() { ? _metadata.dictionary_page_offset : _metadata.data_page_offset; size_t chunk_size = _metadata.total_compressed_size; - _page_reader = std::make_unique(_stream_reader, _io_ctx, start_offset, chunk_size); + // create page reader + _page_reader = create_page_reader(_stream_reader, _io_ctx, start_offset, chunk_size, + _metadata.num_values, _offset_index); // get the block compression codec RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); if (_metadata.__isset.dictionary_page_offset) { @@ -88,24 +92,27 @@ Status ColumnChunkReader::next_page() { if (UNLIKELY(_remaining_num_values != 0)) { return Status::Corruption("Should skip current page"); } + RETURN_IF_ERROR(_page_reader->next_page_header()); - if (_page_reader->get_page_header()->type == tparquet::PageType::DICTIONARY_PAGE) { - // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, - // so we should parse the directory page in next_page() - RETURN_IF_ERROR(_decode_dict_page()); - // parse the real first data page - return next_page(); - } else if (_page_reader->get_page_header()->type == tparquet::PageType::DATA_PAGE_V2) { - _remaining_num_values = _page_reader->get_page_header()->data_page_header_v2.num_values; - _chunk_parsed_values += _remaining_num_values; - _state = HEADER_PARSED; - return Status::OK(); - } else { - _remaining_num_values = _page_reader->get_page_header()->data_page_header.num_values; - _chunk_parsed_values += _remaining_num_values; - _state = HEADER_PARSED; - return Status::OK(); + + if (!_dict_checked) { + _dict_checked = true; + const tparquet::PageHeader* header; + RETURN_IF_ERROR(_page_reader->get_page_header(header)); + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + return next_page(); + } } + + RETURN_IF_ERROR(_page_reader->get_num_values(_remaining_num_values)); + _chunk_parsed_values += _remaining_num_values; + _state = HEADER_PARSED; + + return Status::OK(); } void ColumnChunkReader::_get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, @@ -119,17 +126,19 @@ void ColumnChunkReader::_get_uncompressed_levels(const tparquet::DataPageHeaderV } Status ColumnChunkReader::load_page_data() { + // TODO: remove checking HEADER_PARSED or change name if (UNLIKELY(_state != HEADER_PARSED)) { return Status::Corruption("Should parse page header"); } - const auto& header = *_page_reader->get_page_header(); - int32_t uncompressed_size = header.uncompressed_page_size; + const tparquet::PageHeader* header; + RETURN_IF_ERROR(_page_reader->get_page_header(header)); + int32_t uncompressed_size = header->uncompressed_page_size; if (_block_compress_codec != nullptr) { Slice compressed_data; RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); - if (header.__isset.data_page_header_v2) { - const tparquet::DataPageHeaderV2& header_v2 = header.data_page_header_v2; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; // uncompressed_size = rl + dl + uncompressed_data_size // compressed_size = rl + dl + compressed_data_size uncompressed_size -= header_v2.repetition_levels_byte_length + @@ -137,8 +146,8 @@ Status ColumnChunkReader::load_page_data() { _get_uncompressed_levels(header_v2, compressed_data); } bool is_v2_compressed = - header.__isset.data_page_header_v2 && header.data_page_header_v2.is_compressed; - if (header.__isset.data_page_header || is_v2_compressed) { + header->__isset.data_page_header_v2 && header->data_page_header_v2.is_compressed; + if (header->__isset.data_page_header || is_v2_compressed) { // check decompressed buffer size _reserve_decompress_buf(uncompressed_size); _page_data = Slice(_decompress_buf.get(), uncompressed_size); @@ -151,36 +160,36 @@ Status ColumnChunkReader::load_page_data() { } } else { RETURN_IF_ERROR(_page_reader->get_page_data(_page_data)); - if (header.__isset.data_page_header_v2) { - _get_uncompressed_levels(header.data_page_header_v2, _page_data); + if (header->__isset.data_page_header_v2) { + _get_uncompressed_levels(header->data_page_header_v2, _page_data); } } // Initialize repetition level and definition level. Skip when level = 0, which means required field. if (_max_rep_level > 0) { SCOPED_RAW_TIMER(&_statistics.decode_level_time); - if (header.__isset.data_page_header_v2) { + if (header->__isset.data_page_header_v2) { RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, _remaining_num_values)); } else { RETURN_IF_ERROR(_rep_level_decoder.init( - &_page_data, header.data_page_header.repetition_level_encoding, _max_rep_level, + &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, _remaining_num_values)); } } if (_max_def_level > 0) { SCOPED_RAW_TIMER(&_statistics.decode_level_time); - if (header.__isset.data_page_header_v2) { + if (header->__isset.data_page_header_v2) { RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, _remaining_num_values)); } else { RETURN_IF_ERROR(_def_level_decoder.init( - &_page_data, header.data_page_header.definition_level_encoding, _max_def_level, + &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, _remaining_num_values)); } } - auto encoding = header.__isset.data_page_header_v2 ? header.data_page_header_v2.encoding - : header.data_page_header.encoding; + auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding + : header->data_page_header.encoding; // change the deprecated encoding to RLE_DICTIONARY if (encoding == tparquet::Encoding::PLAIN_DICTIONARY) { encoding = tparquet::Encoding::RLE_DICTIONARY; @@ -207,14 +216,15 @@ Status ColumnChunkReader::load_page_data() { } Status ColumnChunkReader::_decode_dict_page() { - const tparquet::PageHeader& header = *_page_reader->get_page_header(); - DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header.type); + const tparquet::PageHeader* header; + RETURN_IF_ERROR(_page_reader->get_page_header(header)); + DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); SCOPED_RAW_TIMER(&_statistics.decode_dict_time); // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files. // refer: https://github.com/apache/parquet-format/blob/master/Encodings.md - tparquet::Encoding::type dict_encoding = header.dictionary_page_header.encoding; + tparquet::Encoding::type dict_encoding = header->dictionary_page_header.encoding; if (dict_encoding != tparquet::Encoding::PLAIN_DICTIONARY && dict_encoding != tparquet::Encoding::PLAIN) { return Status::InternalError("Unsupported dictionary encoding {}", @@ -222,7 +232,7 @@ Status ColumnChunkReader::_decode_dict_page() { } // Prepare dictionary data - int32_t uncompressed_size = header.uncompressed_page_size; + int32_t uncompressed_size = header->uncompressed_page_size; std::unique_ptr dict_data(new uint8_t[uncompressed_size]); if (_block_compress_codec != nullptr) { Slice compressed_data; @@ -246,7 +256,7 @@ Status ColumnChunkReader::_decode_dict_page() { // page_decoder->init(_field_schema, _ctz); // Set the dictionary data RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, - header.dictionary_page_header.num_values)); + header->dictionary_page_header.num_values)); _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] = std::move(page_decoder); _has_dict = true; diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h index 0ca6859ac83876..79ee3cd646306c 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h @@ -71,7 +71,7 @@ using ColumnString = ColumnStr; * // Or, we can call the chunk_reader.skip_page() to skip current page. * chunk_reader.load_page_data(); * // Decode values into column or slice. - * // Or, we can call chunk_reader.slip_values(num_values) to skip some values. + * // Or, we can call chunk_reader.skip_values(num_values) to skip some values. * chunk_reader.decode_values(slice, num_values); * } */ @@ -84,10 +84,13 @@ class ColumnChunkReader { int64_t decode_value_time = 0; int64_t decode_dict_time = 0; int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; }; ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, - FieldSchema* field_schema, cctz::time_zone* ctz, io::IOContext* io_ctx); + FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + cctz::time_zone* ctz, io::IOContext* io_ctx); ~ColumnChunkReader() = default; // Initialize chunk reader, will generate the decoder and codec. @@ -170,6 +173,8 @@ class ColumnChunkReader { Statistics& statistics() { _statistics.decode_header_time = _page_reader->statistics().decode_header_time; + _statistics.skip_page_header_num = _page_reader->statistics().skip_page_header_num; + _statistics.parse_page_header_num = _page_reader->statistics().parse_page_header_num; return _statistics; } @@ -204,6 +209,7 @@ class ColumnChunkReader { io::BufferedStreamReader* _stream_reader = nullptr; tparquet::ColumnMetaData _metadata; + const tparquet::OffsetIndex* _offset_index; // cctz::time_zone* _ctz; io::IOContext* _io_ctx = nullptr; @@ -219,6 +225,7 @@ class ColumnChunkReader { size_t _decompress_buf_size = 0; Slice _v2_rep_levels; Slice _v2_def_levels; + bool _dict_checked = false; bool _has_dict = false; Decoder* _page_decoder = nullptr; // Map: encoding -> Decoder diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp index 2a3782ab44944c..85d03daebc5609 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp @@ -108,7 +108,7 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, const std::vector& row_ranges, cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, - size_t max_buf_size) { + size_t max_buf_size, const tparquet::OffsetIndex* offset_index) { if (field->type.type == TYPE_ARRAY) { std::unique_ptr element_reader; RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, @@ -144,7 +144,8 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, reader.reset(struct_reader.release()); } else { const tparquet::ColumnChunk& chunk = row_group.columns[field->physical_column_index]; - auto scalar_reader = ScalarColumnReader::create_unique(row_ranges, chunk, ctz, io_ctx); + auto scalar_reader = + ScalarColumnReader::create_unique(row_ranges, chunk, offset_index, ctz, io_ctx); RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size)); reader.reset(scalar_reader.release()); } @@ -190,7 +191,7 @@ Status ScalarColumnReader::init(io::FileReaderSPtr file, FieldSchema* field, siz _stream_reader = std::make_unique(file, chunk_start, chunk_len, prefetch_buffer_size); _chunk_reader = std::make_unique(_stream_reader.get(), &_chunk_meta, field, - _ctz, _io_ctx); + _offset_index, _ctz, _io_ctx); RETURN_IF_ERROR(_chunk_reader->init()); return Status::OK(); } diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h index d12eac2f383d58..f0eadb8bcd61c5 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h @@ -65,7 +65,9 @@ class ParquetColumnReader { decode_value_time(0), decode_dict_time(0), decode_level_time(0), - decode_null_map_time(0) {} + decode_null_map_time(0), + skip_page_header_num(0), + parse_page_header_num(0) {} Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReader::Statistics& cs, int64_t null_map_time) @@ -79,7 +81,9 @@ class ParquetColumnReader { decode_value_time(cs.decode_value_time), decode_dict_time(cs.decode_dict_time), decode_level_time(cs.decode_level_time), - decode_null_map_time(null_map_time) {} + decode_null_map_time(null_map_time), + skip_page_header_num(cs.skip_page_header_num), + parse_page_header_num(cs.parse_page_header_num) {} int64_t read_time; int64_t read_calls; @@ -92,6 +96,8 @@ class ParquetColumnReader { int64_t decode_dict_time; int64_t decode_level_time; int64_t decode_null_map_time; + int64_t skip_page_header_num; + int64_t parse_page_header_num; void merge(Statistics& statistics) { read_time += statistics.read_time; @@ -105,6 +111,8 @@ class ParquetColumnReader { decode_dict_time += statistics.decode_dict_time; decode_level_time += statistics.decode_level_time; decode_null_map_time += statistics.decode_null_map_time; + skip_page_header_num += statistics.skip_page_header_num; + parse_page_header_num += statistics.parse_page_header_num; } }; @@ -134,7 +142,7 @@ class ParquetColumnReader { const tparquet::RowGroup& row_group, const std::vector& row_ranges, cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, - size_t max_buf_size); + size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr); void set_nested_column() { _nested_column = true; } virtual const std::vector& get_rep_level() const = 0; virtual const std::vector& get_def_level() const = 0; @@ -160,9 +168,12 @@ class ScalarColumnReader : public ParquetColumnReader { ENABLE_FACTORY_CREATOR(ScalarColumnReader) public: ScalarColumnReader(const std::vector& row_ranges, - const tparquet::ColumnChunk& chunk_meta, cctz::time_zone* ctz, + const tparquet::ColumnChunk& chunk_meta, + const tparquet::OffsetIndex* offset_index, cctz::time_zone* ctz, io::IOContext* io_ctx) - : ParquetColumnReader(row_ranges, ctz, io_ctx), _chunk_meta(chunk_meta) {} + : ParquetColumnReader(row_ranges, ctz, io_ctx), + _chunk_meta(chunk_meta), + _offset_index(offset_index) {} ~ScalarColumnReader() override { close(); } Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size); Status read_column_data(ColumnPtr& doris_column, DataTypePtr& type, @@ -182,6 +193,7 @@ class ScalarColumnReader : public ParquetColumnReader { private: tparquet::ColumnChunk _chunk_meta; + const tparquet::OffsetIndex* _offset_index; std::unique_ptr _stream_reader; std::unique_ptr _chunk_reader; std::vector _rep_levels; diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 3f8000c3173e1c..335207070dd367 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -39,7 +39,6 @@ #include "runtime/thread_context.h" #include "runtime/types.h" #include "schema_desc.h" -#include "util/simd/bits.h" #include "vec/columns/column_const.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_string.h" @@ -124,12 +123,17 @@ Status RowGroupReader::init( const size_t MAX_GROUP_BUF_SIZE = config::parquet_rowgroup_max_buffer_mb << 20; const size_t MAX_COLUMN_BUF_SIZE = config::parquet_column_max_buffer_mb << 20; size_t max_buf_size = std::min(MAX_COLUMN_BUF_SIZE, MAX_GROUP_BUF_SIZE / _read_columns.size()); - for (auto& read_col : _read_columns) { - auto field = const_cast(schema.get_column(read_col)); + for (const auto& read_col : _read_columns) { + auto* field = const_cast(schema.get_column(read_col)); + auto physical_index = field->physical_column_index; std::unique_ptr reader; + // TODO : support rested column types + const tparquet::OffsetIndex* offset_index = + col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index] + : nullptr; RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader, - max_buf_size)); + max_buf_size, offset_index)); if (reader == nullptr) { VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed"; return Status::Corruption("Init row group reader failed"); diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h index 128a7450554327..d38f5a74adf3d7 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h @@ -29,7 +29,6 @@ #include "io/fs/file_reader_writer_fwd.h" #include "vec/columns/column.h" -#include "vec/common/allocator.h" #include "vec/exec/format/parquet/parquet_common.h" #include "vec/exprs/vexpr_fwd.h" #include "vparquet_column_reader.h" diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp index 3b4e18c27da7aa..a321e77c69289c 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp @@ -40,11 +40,23 @@ namespace doris::vectorized { static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; +std::unique_ptr create_page_reader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, int64_t num_values, + const tparquet::OffsetIndex* offset_index) { + if (offset_index) { + return std::make_unique(reader, io_ctx, offset, length, + num_values, offset_index); + } else { + return std::make_unique(reader, io_ctx, offset, length); + } +} + PageReader::PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length) : _reader(reader), _io_ctx(io_ctx), _start_offset(offset), _end_offset(offset + length) {} -Status PageReader::next_page_header() { +Status PageReader::_parse_page_header() { if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { return Status::IOError("Out-of-bounds Access"); } @@ -82,6 +94,7 @@ Status PageReader::next_page_header() { header_size <<= 2; } + _statistics.parse_page_header_num++; _offset += real_header_size; _next_header_offset = _offset + _cur_page_header.compressed_page_size; _state = HEADER_PARSED; diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.h b/be/src/vec/exec/format/parquet/vparquet_page_reader.h index bdd0a8d0f5ff24..5765df4fc1f389 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.h @@ -20,6 +20,8 @@ #include #include +#include + #include "common/status.h" namespace doris { @@ -39,11 +41,13 @@ class PageReader { public: struct Statistics { int64_t decode_header_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; }; PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length); - ~PageReader() = default; + virtual ~PageReader() = default; // Deprecated // Parquet file may not be standardized, @@ -52,13 +56,31 @@ class PageReader { // [[deprecated]] bool has_next_page() const { return _offset < _end_offset; } - Status next_page_header(); + virtual Status next_page_header() { return _parse_page_header(); } - Status skip_page(); + virtual Status get_page_header(const tparquet::PageHeader*& page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + page_header = &_cur_page_header; + return Status::OK(); + } - const tparquet::PageHeader* get_page_header() const { return &_cur_page_header; } + virtual Status get_num_values(uint32_t& num_values) { + if (_state != HEADER_PARSED) { + return Status::InternalError("Page header not parsed"); + } + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + num_values = _cur_page_header.data_page_header_v2.num_values; + } else { + num_values = _cur_page_header.data_page_header.num_values; + } + return Status::OK(); + } - Status get_page_data(Slice& slice); + virtual Status skip_page(); + + virtual Status get_page_data(Slice& slice); Statistics& statistics() { return _statistics; } @@ -68,20 +90,99 @@ class PageReader { _state = INITIALIZED; } -private: +protected: enum PageReaderState { INITIALIZED, HEADER_PARSED }; - - io::BufferedStreamReader* _reader = nullptr; - io::IOContext* _io_ctx = nullptr; + PageReaderState _state = INITIALIZED; tparquet::PageHeader _cur_page_header; Statistics _statistics; - PageReaderState _state = INITIALIZED; + Status _parse_page_header(); + +private: + io::BufferedStreamReader* _reader = nullptr; + io::IOContext* _io_ctx = nullptr; uint64_t _offset = 0; uint64_t _next_header_offset = 0; - uint64_t _start_offset = 0; uint64_t _end_offset = 0; }; +class PageReaderWithOffsetIndex : public PageReader { +public: + PageReaderWithOffsetIndex(io::BufferedStreamReader* reader, io::IOContext* io_ctx, + uint64_t offset, uint64_t length, int64_t num_values, + const tparquet::OffsetIndex* offset_index) + : PageReader(reader, io_ctx, offset, length), + _num_values(num_values), + _offset_index(offset_index) {} + + Status next_page_header() override { + // lazy to parse page header in get_page_header + return Status::OK(); + } + + Status get_page_header(const tparquet::PageHeader*& page_header) override { + if (_state != HEADER_PARSED) { + RETURN_IF_ERROR(_parse_page_header()); + } + page_header = &_cur_page_header; + return Status::OK(); + } + + Status get_num_values(uint32_t& num_values) override { + if (UNLIKELY(_page_index >= _offset_index->page_locations.size())) { + return Status::IOError("End of page"); + } + + if (_page_index < _offset_index->page_locations.size() - 1) { + num_values = _offset_index->page_locations[_page_index + 1].first_row_index - + _offset_index->page_locations[_page_index].first_row_index; + } else { + num_values = _num_values - _offset_index->page_locations[_page_index].first_row_index; + } + return Status::OK(); + } + + Status skip_page() override { + if (UNLIKELY(_page_index >= _offset_index->page_locations.size())) { + return Status::IOError("End of page"); + } + + if (_state != HEADER_PARSED) { + _statistics.skip_page_header_num++; + } + + seek_to_page(_offset_index->page_locations[_page_index].offset + + _offset_index->page_locations[_page_index].compressed_page_size); + _page_index++; + return Status::OK(); + } + + Status get_page_data(Slice& slice) override { + if (_page_index >= _offset_index->page_locations.size()) { + return Status::IOError("End of page"); + } + if (_state != HEADER_PARSED) { + RETURN_IF_ERROR(_parse_page_header()); + } + + // dirctionary page is not in page location + if (LIKELY(_cur_page_header.type != tparquet::PageType::DICTIONARY_PAGE)) { + _page_index++; + } + + return PageReader::get_page_data(slice); + } + +private: + size_t _page_index = 0; + int64_t _num_values = 0; + const tparquet::OffsetIndex* _offset_index; +}; + +std::unique_ptr create_page_reader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, int64_t num_values = 0, + const tparquet::OffsetIndex* offset_index = nullptr); + } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp index 632bc9bcda335d..ded745f5a5c95a 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp @@ -23,18 +23,14 @@ #include #include -#include #include #include "common/status.h" #include "exec/schema_scanner.h" -#include "gen_cpp/descriptors.pb.h" -#include "gtest/gtest_pred_impl.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader.h" #include "io/fs/file_reader_writer_fwd.h" -#include "olap/olap_common.h" #include "parquet_pred_cmp.h" #include "parquet_thrift_util.h" #include "runtime/define_primitive_type.h" @@ -170,6 +166,10 @@ void ParquetReader::_init_profile() { ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeLevelTime", parquet_profile, 1); _parquet_profile.decode_null_map_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeNullMapTime", parquet_profile, 1); + _parquet_profile.skip_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "SkipPageHeaderNum", TUnit::UNIT, parquet_profile, 1); + _parquet_profile.parse_page_header_num = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "ParsePageHeaderNum", TUnit::UNIT, parquet_profile, 1); } } @@ -921,6 +921,9 @@ void ParquetReader::_collect_profile() { COUNTER_UPDATE(_parquet_profile.page_index_filter_time, _statistics.page_index_filter_time); COUNTER_UPDATE(_parquet_profile.row_group_filter_time, _statistics.row_group_filter_time); + COUNTER_UPDATE(_parquet_profile.skip_page_header_num, _column_statistics.skip_page_header_num); + COUNTER_UPDATE(_parquet_profile.parse_page_header_num, + _column_statistics.parse_page_header_num); COUNTER_UPDATE(_parquet_profile.file_read_time, _column_statistics.read_time); COUNTER_UPDATE(_parquet_profile.file_read_calls, _column_statistics.read_calls); COUNTER_UPDATE(_parquet_profile.file_meta_read_calls, _column_statistics.meta_read_calls); diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h index d5e2ffe8ee0f54..416004f90a0ac5 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_reader.h @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -183,6 +182,8 @@ class ParquetReader : public GenericReader { RuntimeProfile::Counter* decode_dict_time = nullptr; RuntimeProfile::Counter* decode_level_time = nullptr; RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* skip_page_header_num = nullptr; + RuntimeProfile::Counter* parse_page_header_num = nullptr; }; Status _open_file(); diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index 26da99607b6cba..741bb242bfdae1 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -205,7 +205,8 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column io::BufferedFileStreamReader stream_reader(file_reader, start_offset, chunk_size, 1024); - ColumnChunkReader chunk_reader(&stream_reader, column_chunk, field_schema, &ctz, nullptr); + ColumnChunkReader chunk_reader(&stream_reader, column_chunk, field_schema, nullptr, &ctz, + nullptr); // initialize chunk reader static_cast(chunk_reader.init()); // seek to next page header diff --git a/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out b/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out new file mode 100644 index 00000000000000..6c869dbc789466 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out @@ -0,0 +1,289 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +1 2132 4633 4 28.00 28955.64 0.09 0.06 N O 1996-04-21 1996-03-30 1996-05-16 NONE AIR lites. fluffily even de +1 15635 638 6 32.00 49620.16 0.07 0.02 N O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON MAIL arefully slyly ex +1 24027 1534 5 24.00 22824.48 0.10 0.04 N O 1996-03-30 1996-03-14 1996-04-01 NONE FOB pending foxes. slyly re +1 63700 3701 3 8.00 13309.60 0.10 0.02 N O 1996-01-29 1996-03-05 1996-01-31 TAKE BACK RETURN REG AIR riously. regular, express dep +1 67310 7311 2 36.00 45983.16 0.09 0.06 N O 1996-04-12 1996-02-28 1996-04-20 TAKE BACK RETURN MAIL ly final dependencies: slyly bold +1 155190 7706 1 17.00 21168.23 0.04 0.02 N O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON TRUCK egular courts above the +2 106170 1191 1 38.00 44694.46 0.00 0.05 N O 1997-01-28 1997-01-14 1997-02-02 TAKE BACK RETURN RAIL ven requests. deposits breach a +3 4297 1798 1 45.00 54058.05 0.06 0.00 R F 1994-02-02 1994-01-04 1994-02-23 NONE AIR ongside of the furiously brave acco +3 19036 6540 2 49.00 46796.47 0.10 0.00 R F 1993-11-09 1993-12-20 1993-11-24 TAKE BACK RETURN RAIL unusual accounts. eve +3 29380 1883 4 2.00 2618.76 0.01 0.06 A F 1993-12-04 1994-01-07 1994-01-01 NONE TRUCK y. fluffily pending d + +-- !q02 -- +5999008 16312 6313 4 2.00 2456.62 0.08 0.05 R F 1994-04-28 1994-06-01 1994-05-12 COLLECT COD FOB longside of the slo +5999008 32738 2739 3 39.00 65158.47 0.09 0.08 R F 1994-07-11 1994-06-15 1994-08-04 TAKE BACK RETURN TRUCK equests nag along +5999008 64711 2230 1 29.00 48595.59 0.00 0.02 R F 1994-05-16 1994-07-04 1994-05-18 NONE FOB final requests across +5999008 192755 5275 2 32.00 59128.00 0.07 0.08 R F 1994-05-15 1994-05-22 1994-06-07 COLLECT COD RAIL ts sleep slyly about the slyly ironic acco +5999009 12147 7150 1 21.00 22241.94 0.00 0.05 N O 1997-11-01 1997-12-11 1997-11-05 NONE AIR deposits after the blithely ex +5999010 106595 6596 1 31.00 49649.29 0.02 0.05 N O 1997-11-29 1997-10-24 1997-12-11 DELIVER IN PERSON MAIL ilent instructions? slyly r +5999010 141441 1442 2 42.00 62262.48 0.05 0.08 N O 1997-09-21 1997-10-13 1997-09-25 TAKE BACK RETURN SHIP c, even ideas. ruth +5999010 193075 8114 3 40.00 46722.80 0.04 0.07 N O 1997-11-19 1997-09-25 1997-11-25 DELIVER IN PERSON AIR accounts sleep blithely even, +5999010 198678 1198 4 12.00 21320.04 0.00 0.04 N O 1997-09-19 1997-10-15 1997-10-05 DELIVER IN PERSON REG AIR ironic foxes. slyly special id +5999011 98609 6137 1 44.00 70734.40 0.03 0.01 N O 1998-04-05 1998-05-16 1998-05-05 DELIVER IN PERSON REG AIR ructions along the blit + +-- !q03 -- +2000001 16877 6878 1 36.00 64579.32 0.01 0.05 A F 1995-01-23 1995-01-31 1995-02-20 TAKE BACK RETURN FOB regular deposits. even +2000001 50928 3434 2 36.00 67641.12 0.02 0.03 A F 1995-02-03 1995-02-07 1995-02-13 NONE AIR ickly slyl +2000001 117877 7878 4 20.00 37897.40 0.02 0.07 R F 1995-03-29 1995-03-16 1995-04-01 COLLECT COD RAIL . realms boost unusual theodoli +2000001 135534 8048 3 38.00 59642.14 0.00 0.05 R F 1994-12-31 1995-03-06 1995-01-26 COLLECT COD MAIL l theodolites affix quickly alongside of +2000001 149269 1784 5 15.00 19773.90 0.05 0.05 R F 1995-03-08 1995-02-10 1995-03-23 DELIVER IN PERSON AIR e bold, silent foxes solve dog +2000002 41816 1817 6 8.00 14062.48 0.05 0.06 N O 1996-02-14 1995-12-25 1996-03-12 DELIVER IN PERSON FOB y quickly pending foxes. quickly ironic acc +2000002 62662 2663 2 48.00 77983.68 0.01 0.05 N O 1995-11-20 1996-01-11 1995-12-05 TAKE BACK RETURN SHIP requests sleep blithely. slyly +2000002 77402 7403 4 47.00 64831.80 0.07 0.03 N O 1996-02-24 1996-02-04 1996-03-09 NONE SHIP ong the carefully silent instructions. even +2000002 80719 8244 5 42.00 71387.82 0.10 0.03 N O 1995-11-24 1996-01-01 1995-11-25 COLLECT COD SHIP ing to the carefully final deposits. care +2000002 156357 6358 3 28.00 39573.80 0.09 0.05 N O 1995-12-26 1996-01-15 1996-01-25 COLLECT COD MAIL ely regular instr + +-- !q04 -- +1 Customer#000000001 IVhzIApeRb ot,c,E 15 25-989-741-2988 711.56 BUILDING to the even, regular platelets. regular, ironic epitaphs nag e +2 Customer#000000002 XSTf4,NCwDVaWNe6tEgvwfmRchLXak 13 23-768-687-3665 121.65 AUTOMOBILE l accounts. blithely ironic theodolites integrate boldly: caref +3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12 AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov +4 Customer#000000004 XxVSJsLAGtn 4 14-128-190-5944 2866.83 MACHINERY requests. final, regular ideas sleep final accou +5 Customer#000000005 KvpyuHCplrB84WgAiGV6sYpZq7Tj 3 13-750-942-6364 794.47 HOUSEHOLD n accounts will have to unwind. foxes cajole accor +6 Customer#000000006 sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn 20 30-114-968-4951 7638.57 AUTOMOBILE tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious +7 Customer#000000007 TcGe5gaZNgVePxU5kRrvXBfkasDTea 18 28-190-982-9759 9561.95 AUTOMOBILE ainst the ironic, express theodolites. express, even pinto beans among the exp +8 Customer#000000008 I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 17 27-147-574-9335 6819.74 BUILDING among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide +9 Customer#000000009 xKiAFTjUsCuxfeleNqefumTrjS 8 18-338-906-3675 8324.07 FURNITURE r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl +10 Customer#000000010 6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2 5 15-741-346-9870 2753.54 HOUSEHOLD es regular deposits haggle. fur + +-- !q05 -- +140001 Customer#000140001 CkN1egC06Sc51bbDyQ8VnFn Bz6N1p 15 25-647-696-2830 2747.48 AUTOMOBILE heodolites. slyly bold theodolites d +140002 Customer#000140002 8UWLS,im0k94ivCVx 23 33-146-814-9234 7914.10 HOUSEHOLD eep behind the quickly bold foxes. furiously ironic ideas shall have to sleep. regular packages +140003 Customer#000140003 2duVgk HhGGlOeP4S,brWKjKG62bGkupful 23 33-326-909-7916 9389.77 FURNITURE pending, even packages are. slyly regular accounts wake ironically final packages. bold +140004 Customer#000140004 S,V7RhLnmqPK0TDghbcdxotzTyKeUC 13 23-594-312-2596 5931.66 HOUSEHOLD t blithely blithely regular packages. never silent dependencies cajo +140005 Customer#000140005 yQemRDs9i8MmTJla7xha xqeZjMznW 20 30-169-231-7354 9489.56 BUILDING accounts. even ideas sleep carefu +140006 Customer#000140006 5,eRqyFjpobN2Wtvt2oXuLcJcNE8oTyRh 15 25-681-278-7283 67.66 MACHINERY at the accounts are bold escapades. furiously final foxes use carefully unusual orb +140007 Customer#000140007 AX75sSePE5PlDjD5qS6W1dx08Levf09 24 34-818-770-8059 2093.37 MACHINERY ily according to the furiously final packages? quickly spe +140008 Customer#000140008 2zpry AYh9otf4c5vESISPvKLWPKe9i 14 24-552-949-6395 3264.69 AUTOMOBILE nstructions are against the requests. fin +140009 Customer#000140009 dNwNUcCv,,0YE6WFYfOgM,6A2 4 14-940-856-8557 -359.36 HOUSEHOLD beans. blithely silent dependencies haggle slyly. carefully quick accounts across the depos +140010 Customer#000140010 vZxOW,NtvppKR9mpTl6RDl9sWJJbosYDoLineEm 7 17-151-800-8260 8216.11 BUILDING nding foxes across the quickly regular forges nod accounts. slyly express ex + +-- !q06 -- +100001 Customer#000100001 gQ1s5C45A3PxWmZ1oFFSxt8u EcZ, 24 34-705-443-4055 1726.66 HOUSEHOLD ts. ironic instructions sleep. final deposits +100002 Customer#000100002 qOmTcZ7kHzJLSoaLenr9,Gu 17 27-453-414-8560 -39.14 BUILDING wake carefully. blithely regular epitaphs are among the quickly regular deposits. +100003 Customer#000100003 5AYbJxvjo7ErQB,cGIpKZRAE9,w2l9 5 15-783-309-8970 72.71 BUILDING ckly blithely special accounts. theodolites are carefully. pending requests ha +100004 Customer#000100004 cpIOYQpMlm 18 28-316-370-8752 9990.05 BUILDING y above the slyly regular pains. unusual requests against the always special packages bre +100005 Customer#000100005 Wud8n74NcIpwiKSjPS zZ 16 26-935-603-9031 7789.25 BUILDING ing dugouts boost slyly above the pending, final accounts? regular deposits wake slyly alongside of the blithely i +100006 Customer#000100006 AkjXh4y,QNaF7,0xzbP,sG 7 17-964-673-7626 974.05 MACHINERY grate across the slyly even packages; final, special idea +100007 Customer#000100007 d94JW9Hc2ZtGriOBNKyIjOeP,VZZqIX7S 17 27-244-129-5307 777.86 HOUSEHOLD foxes are against the ironic theodolites. evenly pending ideas according to the qu +100008 Customer#000100008 Hv2A,YqfNnGRIKaY 18 28-828-394-8424 3374.90 BUILDING ccounts. even deposits wake quickly pinto beans. bold instructions integrate? never bold theodolites are s +100009 Customer#000100009 OioQ3EjJZRvxCNh6Q8E3QZH 6 16-928-807-2622 3932.63 MACHINERY aggle blithely quickly final accounts. carefully final deposits above the fluffily unus +100010 Customer#000100010 Tbiz2WMJX 0 10-147-978-7806 5693.02 BUILDING y regular ideas. quickly unusual gifts n + +-- !q07 -- +1 36901 O 173665.47 1996-01-02 5-LOW Clerk#000000951 0 nstructions sleep furiously among +2 78002 O 46929.18 1996-12-01 1-URGENT Clerk#000000880 0 foxes. pending accounts at the pending, silent asymptot +3 123314 F 193846.25 1993-10-14 5-LOW Clerk#000000955 0 sly final accounts boost. carefully regular ideas cajole carefully. depos +4 136777 O 32151.78 1995-10-11 5-LOW Clerk#000000124 0 sits. slyly regular warthogs cajole. regular, regular theodolites acro +5 44485 F 144659.20 1994-07-30 5-LOW Clerk#000000925 0 quickly. bold deposits sleep slyly. packages use slyly +6 55624 F 58749.59 1992-02-21 4-NOT SPECIFIED Clerk#000000058 0 ggle. special, final requests are against the furiously specia +7 39136 O 252004.18 1996-01-10 2-HIGH Clerk#000000470 0 ly special requests +32 130057 O 208660.75 1995-07-16 2-HIGH Clerk#000000616 0 ise blithely bold, regular requests. quickly unusual dep +33 66958 F 163243.98 1993-10-27 3-MEDIUM Clerk#000000409 0 uriously. furiously final request +34 61001 O 58949.67 1998-07-21 3-MEDIUM Clerk#000000223 0 ly final packages. fluffily final deposits wake blithely ideas. spe + +-- !q08 -- +5990016 100807 F 102428.29 1994-01-31 1-URGENT Clerk#000000554 0 . fluffily unusual requests cajole furiously. fluffily pending accounts ca +5990017 12382 F 176602.99 1992-07-01 5-LOW Clerk#000000205 0 ual pinto beans. final instructions haggle quickly alongside of the furio +5990018 51145 F 78440.49 1992-05-28 1-URGENT Clerk#000000996 0 quests play daringly. regula +5990019 85478 O 250306.69 1998-06-29 5-LOW Clerk#000000900 0 ainst the sly pinto beans. unu +5990020 62137 O 229287.04 1996-08-15 1-URGENT Clerk#000000801 0 fluffily special pinto beans. regular, regular pinto beans slee +5990021 24235 O 265459.10 1996-12-16 3-MEDIUM Clerk#000000113 0 gside of the ironic, unusual escapades. evenly silent tithes are +5990022 35143 O 141070.92 1996-07-01 4-NOT SPECIFIED Clerk#000000546 0 ests haggle across the blithely bo +5990023 65318 F 171515.91 1993-07-04 1-URGENT Clerk#000000178 0 r the express accounts haggle blithely ironic accounts-- regu +5990048 88213 O 70608.62 1997-10-23 2-HIGH Clerk#000000303 0 slyly enticing foxes doze regularly even requests. +5990049 115694 F 183390.98 1992-05-21 1-URGENT Clerk#000000450 0 ckly final theodolites ca + +-- !q09 -- +2000001 44200 F 257495.03 1994-12-18 5-LOW Clerk#000000314 0 ometimes theodolites. quickly even accounts among the blithely bold +2000002 55241 O 263734.77 1995-11-13 1-URGENT Clerk#000000749 0 uses along the brave excuses sleep for the packages. packages affix? slyl +2000003 84553 F 78066.42 1992-10-10 5-LOW Clerk#000000314 0 e slyly regular asymptotes. fluf +2000004 125197 F 246917.53 1993-01-06 1-URGENT Clerk#000000675 0 ironic ideas. platelets are regularly after the +2000005 117907 O 229611.23 1996-10-16 2-HIGH Clerk#000000458 0 he furiously regular excuses haggle slyly along the slyly pending a +2000006 1538 O 32011.55 1995-12-09 1-URGENT Clerk#000000279 0 ual, regular deposits sleep carefully carefully final dependencies. dep +2000007 42958 F 48446.75 1993-03-28 5-LOW Clerk#000000956 0 uickly final ideas. final, final requests are courts. slyly unu +2000032 34156 F 56186.58 1994-09-05 4-NOT SPECIFIED Clerk#000000612 0 fully regular instructions doze +2000033 141263 O 130829.92 1997-06-05 3-MEDIUM Clerk#000000118 0 inst the final dependencies. even, final pat +2000034 149275 O 64568.70 1997-09-23 2-HIGH Clerk#000000335 0 regular asymptotes. carefu + +-- !q10 -- +1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901.00 ly. slyly ironi +2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902.00 lar accounts amo +3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903.00 egular deposits hag +4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904.00 p furiously r +5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905.00 wake carefully +6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906.00 sual a +7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907.00 lyly. ex +8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908.00 eposi +9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909.00 ironic foxe +10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910.01 ithely final deposit + +-- !q08 -- +190001 powder coral chiffon burnished bisque Manufacturer#2 Brand#22 MEDIUM ANODIZED NICKEL 26 WRAP BOX 1091.00 ly busy deposi +190002 peru coral rosy azure green Manufacturer#4 Brand#41 LARGE POLISHED TIN 21 SM PKG 1092.00 express, daring sh +190003 white salmon lemon cornsilk ghost Manufacturer#4 Brand#41 PROMO ANODIZED TIN 41 LG BAG 1093.00 ckages according to th +190004 ivory almond honeydew metallic dodger Manufacturer#4 Brand#44 PROMO PLATED NICKEL 23 MED DRUM 1094.00 blithely regular t +190005 slate indian forest chartreuse rosy Manufacturer#1 Brand#11 SMALL BRUSHED BRASS 3 SM CASE 1095.00 ly blithe, regula +190006 navajo lavender smoke puff olive Manufacturer#5 Brand#55 SMALL BRUSHED BRASS 35 LG CASE 1096.00 ilent ideas boo +190007 khaki lime goldenrod pink grey Manufacturer#1 Brand#11 STANDARD PLATED BRASS 30 SM PKG 1097.00 fully final gift +190008 cream dark peru thistle gainsboro Manufacturer#3 Brand#31 ECONOMY ANODIZED STEEL 46 WRAP CASE 1098.00 pinto beans. fur +190009 orchid goldenrod metallic frosted powder Manufacturer#3 Brand#33 STANDARD ANODIZED COPPER 25 LG BAG 1099.00 es cajole f +190010 misty mint white seashell papaya Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 38 JUMBO BOX 1100.01 pecia + +-- !q12 -- +100001 seashell cyan plum purple honeydew Manufacturer#3 Brand#35 STANDARD BRUSHED TIN 37 JUMBO CASE 1001.00 ronic dependencies d +100002 steel moccasin forest cornflower brown Manufacturer#3 Brand#34 STANDARD ANODIZED NICKEL 11 WRAP CAN 1002.00 quickly pending +100003 beige powder violet orchid yellow Manufacturer#2 Brand#21 MEDIUM PLATED BRASS 41 SM BOX 1003.00 carefully even pac +100004 snow blanched khaki indian azure Manufacturer#4 Brand#42 SMALL POLISHED TIN 29 SM CASE 1004.00 sly. blithely +100005 grey midnight orange peach pale Manufacturer#2 Brand#21 SMALL POLISHED STEEL 7 MED BAG 1005.00 ajole? blithe +100006 violet sandy olive yellow orange Manufacturer#4 Brand#45 STANDARD BURNISHED COPPER 23 WRAP CASE 1006.00 he slyly regular pack +100007 snow magenta pale lemon metallic Manufacturer#1 Brand#12 PROMO BURNISHED COPPER 4 MED PKG 1007.00 ronic accounts in +100008 spring powder sienna purple lime Manufacturer#4 Brand#45 ECONOMY BRUSHED BRASS 19 SM PKG 1008.00 ts. furious +100009 goldenrod sandy beige hot orange Manufacturer#3 Brand#32 SMALL BURNISHED STEEL 41 WRAP BOX 1009.00 dinos about the quick +100010 lime lavender slate cream brown Manufacturer#4 Brand#43 PROMO ANODIZED COPPER 19 JUMBO PACK 1010.01 gle slyly above the b + +-- !q01 -- +1 2132 4633 4 28.00 28955.64 0.09 0.06 N O 1996-04-21 1996-03-30 1996-05-16 NONE AIR lites. fluffily even de +1 15635 638 6 32.00 49620.16 0.07 0.02 N O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON MAIL arefully slyly ex +1 24027 1534 5 24.00 22824.48 0.10 0.04 N O 1996-03-30 1996-03-14 1996-04-01 NONE FOB pending foxes. slyly re +1 63700 3701 3 8.00 13309.60 0.10 0.02 N O 1996-01-29 1996-03-05 1996-01-31 TAKE BACK RETURN REG AIR riously. regular, express dep +1 67310 7311 2 36.00 45983.16 0.09 0.06 N O 1996-04-12 1996-02-28 1996-04-20 TAKE BACK RETURN MAIL ly final dependencies: slyly bold +1 155190 7706 1 17.00 21168.23 0.04 0.02 N O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON TRUCK egular courts above the +2 106170 1191 1 38.00 44694.46 0.00 0.05 N O 1997-01-28 1997-01-14 1997-02-02 TAKE BACK RETURN RAIL ven requests. deposits breach a +3 4297 1798 1 45.00 54058.05 0.06 0.00 R F 1994-02-02 1994-01-04 1994-02-23 NONE AIR ongside of the furiously brave acco +3 19036 6540 2 49.00 46796.47 0.10 0.00 R F 1993-11-09 1993-12-20 1993-11-24 TAKE BACK RETURN RAIL unusual accounts. eve +3 29380 1883 4 2.00 2618.76 0.01 0.06 A F 1993-12-04 1994-01-07 1994-01-01 NONE TRUCK y. fluffily pending d + +-- !q02 -- +5999008 16312 6313 4 2.00 2456.62 0.08 0.05 R F 1994-04-28 1994-06-01 1994-05-12 COLLECT COD FOB longside of the slo +5999008 32738 2739 3 39.00 65158.47 0.09 0.08 R F 1994-07-11 1994-06-15 1994-08-04 TAKE BACK RETURN TRUCK equests nag along +5999008 64711 2230 1 29.00 48595.59 0.00 0.02 R F 1994-05-16 1994-07-04 1994-05-18 NONE FOB final requests across +5999008 192755 5275 2 32.00 59128.00 0.07 0.08 R F 1994-05-15 1994-05-22 1994-06-07 COLLECT COD RAIL ts sleep slyly about the slyly ironic acco +5999009 12147 7150 1 21.00 22241.94 0.00 0.05 N O 1997-11-01 1997-12-11 1997-11-05 NONE AIR deposits after the blithely ex +5999010 106595 6596 1 31.00 49649.29 0.02 0.05 N O 1997-11-29 1997-10-24 1997-12-11 DELIVER IN PERSON MAIL ilent instructions? slyly r +5999010 141441 1442 2 42.00 62262.48 0.05 0.08 N O 1997-09-21 1997-10-13 1997-09-25 TAKE BACK RETURN SHIP c, even ideas. ruth +5999010 193075 8114 3 40.00 46722.80 0.04 0.07 N O 1997-11-19 1997-09-25 1997-11-25 DELIVER IN PERSON AIR accounts sleep blithely even, +5999010 198678 1198 4 12.00 21320.04 0.00 0.04 N O 1997-09-19 1997-10-15 1997-10-05 DELIVER IN PERSON REG AIR ironic foxes. slyly special id +5999011 98609 6137 1 44.00 70734.40 0.03 0.01 N O 1998-04-05 1998-05-16 1998-05-05 DELIVER IN PERSON REG AIR ructions along the blit + +-- !q03 -- +2000001 16877 6878 1 36.00 64579.32 0.01 0.05 A F 1995-01-23 1995-01-31 1995-02-20 TAKE BACK RETURN FOB regular deposits. even +2000001 50928 3434 2 36.00 67641.12 0.02 0.03 A F 1995-02-03 1995-02-07 1995-02-13 NONE AIR ickly slyl +2000001 117877 7878 4 20.00 37897.40 0.02 0.07 R F 1995-03-29 1995-03-16 1995-04-01 COLLECT COD RAIL . realms boost unusual theodoli +2000001 135534 8048 3 38.00 59642.14 0.00 0.05 R F 1994-12-31 1995-03-06 1995-01-26 COLLECT COD MAIL l theodolites affix quickly alongside of +2000001 149269 1784 5 15.00 19773.90 0.05 0.05 R F 1995-03-08 1995-02-10 1995-03-23 DELIVER IN PERSON AIR e bold, silent foxes solve dog +2000002 41816 1817 6 8.00 14062.48 0.05 0.06 N O 1996-02-14 1995-12-25 1996-03-12 DELIVER IN PERSON FOB y quickly pending foxes. quickly ironic acc +2000002 62662 2663 2 48.00 77983.68 0.01 0.05 N O 1995-11-20 1996-01-11 1995-12-05 TAKE BACK RETURN SHIP requests sleep blithely. slyly +2000002 77402 7403 4 47.00 64831.80 0.07 0.03 N O 1996-02-24 1996-02-04 1996-03-09 NONE SHIP ong the carefully silent instructions. even +2000002 80719 8244 5 42.00 71387.82 0.10 0.03 N O 1995-11-24 1996-01-01 1995-11-25 COLLECT COD SHIP ing to the carefully final deposits. care +2000002 156357 6358 3 28.00 39573.80 0.09 0.05 N O 1995-12-26 1996-01-15 1996-01-25 COLLECT COD MAIL ely regular instr + +-- !q04 -- +1 Customer#000000001 IVhzIApeRb ot,c,E 15 25-989-741-2988 711.56 BUILDING to the even, regular platelets. regular, ironic epitaphs nag e +2 Customer#000000002 XSTf4,NCwDVaWNe6tEgvwfmRchLXak 13 23-768-687-3665 121.65 AUTOMOBILE l accounts. blithely ironic theodolites integrate boldly: caref +3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12 AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov +4 Customer#000000004 XxVSJsLAGtn 4 14-128-190-5944 2866.83 MACHINERY requests. final, regular ideas sleep final accou +5 Customer#000000005 KvpyuHCplrB84WgAiGV6sYpZq7Tj 3 13-750-942-6364 794.47 HOUSEHOLD n accounts will have to unwind. foxes cajole accor +6 Customer#000000006 sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn 20 30-114-968-4951 7638.57 AUTOMOBILE tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious +7 Customer#000000007 TcGe5gaZNgVePxU5kRrvXBfkasDTea 18 28-190-982-9759 9561.95 AUTOMOBILE ainst the ironic, express theodolites. express, even pinto beans among the exp +8 Customer#000000008 I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 17 27-147-574-9335 6819.74 BUILDING among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide +9 Customer#000000009 xKiAFTjUsCuxfeleNqefumTrjS 8 18-338-906-3675 8324.07 FURNITURE r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl +10 Customer#000000010 6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2 5 15-741-346-9870 2753.54 HOUSEHOLD es regular deposits haggle. fur + +-- !q05 -- +140001 Customer#000140001 CkN1egC06Sc51bbDyQ8VnFn Bz6N1p 15 25-647-696-2830 2747.48 AUTOMOBILE heodolites. slyly bold theodolites d +140002 Customer#000140002 8UWLS,im0k94ivCVx 23 33-146-814-9234 7914.10 HOUSEHOLD eep behind the quickly bold foxes. furiously ironic ideas shall have to sleep. regular packages +140003 Customer#000140003 2duVgk HhGGlOeP4S,brWKjKG62bGkupful 23 33-326-909-7916 9389.77 FURNITURE pending, even packages are. slyly regular accounts wake ironically final packages. bold +140004 Customer#000140004 S,V7RhLnmqPK0TDghbcdxotzTyKeUC 13 23-594-312-2596 5931.66 HOUSEHOLD t blithely blithely regular packages. never silent dependencies cajo +140005 Customer#000140005 yQemRDs9i8MmTJla7xha xqeZjMznW 20 30-169-231-7354 9489.56 BUILDING accounts. even ideas sleep carefu +140006 Customer#000140006 5,eRqyFjpobN2Wtvt2oXuLcJcNE8oTyRh 15 25-681-278-7283 67.66 MACHINERY at the accounts are bold escapades. furiously final foxes use carefully unusual orb +140007 Customer#000140007 AX75sSePE5PlDjD5qS6W1dx08Levf09 24 34-818-770-8059 2093.37 MACHINERY ily according to the furiously final packages? quickly spe +140008 Customer#000140008 2zpry AYh9otf4c5vESISPvKLWPKe9i 14 24-552-949-6395 3264.69 AUTOMOBILE nstructions are against the requests. fin +140009 Customer#000140009 dNwNUcCv,,0YE6WFYfOgM,6A2 4 14-940-856-8557 -359.36 HOUSEHOLD beans. blithely silent dependencies haggle slyly. carefully quick accounts across the depos +140010 Customer#000140010 vZxOW,NtvppKR9mpTl6RDl9sWJJbosYDoLineEm 7 17-151-800-8260 8216.11 BUILDING nding foxes across the quickly regular forges nod accounts. slyly express ex + +-- !q06 -- +100001 Customer#000100001 gQ1s5C45A3PxWmZ1oFFSxt8u EcZ, 24 34-705-443-4055 1726.66 HOUSEHOLD ts. ironic instructions sleep. final deposits +100002 Customer#000100002 qOmTcZ7kHzJLSoaLenr9,Gu 17 27-453-414-8560 -39.14 BUILDING wake carefully. blithely regular epitaphs are among the quickly regular deposits. +100003 Customer#000100003 5AYbJxvjo7ErQB,cGIpKZRAE9,w2l9 5 15-783-309-8970 72.71 BUILDING ckly blithely special accounts. theodolites are carefully. pending requests ha +100004 Customer#000100004 cpIOYQpMlm 18 28-316-370-8752 9990.05 BUILDING y above the slyly regular pains. unusual requests against the always special packages bre +100005 Customer#000100005 Wud8n74NcIpwiKSjPS zZ 16 26-935-603-9031 7789.25 BUILDING ing dugouts boost slyly above the pending, final accounts? regular deposits wake slyly alongside of the blithely i +100006 Customer#000100006 AkjXh4y,QNaF7,0xzbP,sG 7 17-964-673-7626 974.05 MACHINERY grate across the slyly even packages; final, special idea +100007 Customer#000100007 d94JW9Hc2ZtGriOBNKyIjOeP,VZZqIX7S 17 27-244-129-5307 777.86 HOUSEHOLD foxes are against the ironic theodolites. evenly pending ideas according to the qu +100008 Customer#000100008 Hv2A,YqfNnGRIKaY 18 28-828-394-8424 3374.90 BUILDING ccounts. even deposits wake quickly pinto beans. bold instructions integrate? never bold theodolites are s +100009 Customer#000100009 OioQ3EjJZRvxCNh6Q8E3QZH 6 16-928-807-2622 3932.63 MACHINERY aggle blithely quickly final accounts. carefully final deposits above the fluffily unus +100010 Customer#000100010 Tbiz2WMJX 0 10-147-978-7806 5693.02 BUILDING y regular ideas. quickly unusual gifts n + +-- !q07 -- +1 36901 O 173665.47 1996-01-02 5-LOW Clerk#000000951 0 nstructions sleep furiously among +2 78002 O 46929.18 1996-12-01 1-URGENT Clerk#000000880 0 foxes. pending accounts at the pending, silent asymptot +3 123314 F 193846.25 1993-10-14 5-LOW Clerk#000000955 0 sly final accounts boost. carefully regular ideas cajole carefully. depos +4 136777 O 32151.78 1995-10-11 5-LOW Clerk#000000124 0 sits. slyly regular warthogs cajole. regular, regular theodolites acro +5 44485 F 144659.20 1994-07-30 5-LOW Clerk#000000925 0 quickly. bold deposits sleep slyly. packages use slyly +6 55624 F 58749.59 1992-02-21 4-NOT SPECIFIED Clerk#000000058 0 ggle. special, final requests are against the furiously specia +7 39136 O 252004.18 1996-01-10 2-HIGH Clerk#000000470 0 ly special requests +32 130057 O 208660.75 1995-07-16 2-HIGH Clerk#000000616 0 ise blithely bold, regular requests. quickly unusual dep +33 66958 F 163243.98 1993-10-27 3-MEDIUM Clerk#000000409 0 uriously. furiously final request +34 61001 O 58949.67 1998-07-21 3-MEDIUM Clerk#000000223 0 ly final packages. fluffily final deposits wake blithely ideas. spe + +-- !q08 -- +5990016 100807 F 102428.29 1994-01-31 1-URGENT Clerk#000000554 0 . fluffily unusual requests cajole furiously. fluffily pending accounts ca +5990017 12382 F 176602.99 1992-07-01 5-LOW Clerk#000000205 0 ual pinto beans. final instructions haggle quickly alongside of the furio +5990018 51145 F 78440.49 1992-05-28 1-URGENT Clerk#000000996 0 quests play daringly. regula +5990019 85478 O 250306.69 1998-06-29 5-LOW Clerk#000000900 0 ainst the sly pinto beans. unu +5990020 62137 O 229287.04 1996-08-15 1-URGENT Clerk#000000801 0 fluffily special pinto beans. regular, regular pinto beans slee +5990021 24235 O 265459.10 1996-12-16 3-MEDIUM Clerk#000000113 0 gside of the ironic, unusual escapades. evenly silent tithes are +5990022 35143 O 141070.92 1996-07-01 4-NOT SPECIFIED Clerk#000000546 0 ests haggle across the blithely bo +5990023 65318 F 171515.91 1993-07-04 1-URGENT Clerk#000000178 0 r the express accounts haggle blithely ironic accounts-- regu +5990048 88213 O 70608.62 1997-10-23 2-HIGH Clerk#000000303 0 slyly enticing foxes doze regularly even requests. +5990049 115694 F 183390.98 1992-05-21 1-URGENT Clerk#000000450 0 ckly final theodolites ca + +-- !q09 -- +2000001 44200 F 257495.03 1994-12-18 5-LOW Clerk#000000314 0 ometimes theodolites. quickly even accounts among the blithely bold +2000002 55241 O 263734.77 1995-11-13 1-URGENT Clerk#000000749 0 uses along the brave excuses sleep for the packages. packages affix? slyl +2000003 84553 F 78066.42 1992-10-10 5-LOW Clerk#000000314 0 e slyly regular asymptotes. fluf +2000004 125197 F 246917.53 1993-01-06 1-URGENT Clerk#000000675 0 ironic ideas. platelets are regularly after the +2000005 117907 O 229611.23 1996-10-16 2-HIGH Clerk#000000458 0 he furiously regular excuses haggle slyly along the slyly pending a +2000006 1538 O 32011.55 1995-12-09 1-URGENT Clerk#000000279 0 ual, regular deposits sleep carefully carefully final dependencies. dep +2000007 42958 F 48446.75 1993-03-28 5-LOW Clerk#000000956 0 uickly final ideas. final, final requests are courts. slyly unu +2000032 34156 F 56186.58 1994-09-05 4-NOT SPECIFIED Clerk#000000612 0 fully regular instructions doze +2000033 141263 O 130829.92 1997-06-05 3-MEDIUM Clerk#000000118 0 inst the final dependencies. even, final pat +2000034 149275 O 64568.70 1997-09-23 2-HIGH Clerk#000000335 0 regular asymptotes. carefu + +-- !q10 -- +1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901.00 ly. slyly ironi +2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902.00 lar accounts amo +3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903.00 egular deposits hag +4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904.00 p furiously r +5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905.00 wake carefully +6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906.00 sual a +7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907.00 lyly. ex +8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908.00 eposi +9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909.00 ironic foxe +10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910.01 ithely final deposit + +-- !q08 -- +190001 powder coral chiffon burnished bisque Manufacturer#2 Brand#22 MEDIUM ANODIZED NICKEL 26 WRAP BOX 1091.00 ly busy deposi +190002 peru coral rosy azure green Manufacturer#4 Brand#41 LARGE POLISHED TIN 21 SM PKG 1092.00 express, daring sh +190003 white salmon lemon cornsilk ghost Manufacturer#4 Brand#41 PROMO ANODIZED TIN 41 LG BAG 1093.00 ckages according to th +190004 ivory almond honeydew metallic dodger Manufacturer#4 Brand#44 PROMO PLATED NICKEL 23 MED DRUM 1094.00 blithely regular t +190005 slate indian forest chartreuse rosy Manufacturer#1 Brand#11 SMALL BRUSHED BRASS 3 SM CASE 1095.00 ly blithe, regula +190006 navajo lavender smoke puff olive Manufacturer#5 Brand#55 SMALL BRUSHED BRASS 35 LG CASE 1096.00 ilent ideas boo +190007 khaki lime goldenrod pink grey Manufacturer#1 Brand#11 STANDARD PLATED BRASS 30 SM PKG 1097.00 fully final gift +190008 cream dark peru thistle gainsboro Manufacturer#3 Brand#31 ECONOMY ANODIZED STEEL 46 WRAP CASE 1098.00 pinto beans. fur +190009 orchid goldenrod metallic frosted powder Manufacturer#3 Brand#33 STANDARD ANODIZED COPPER 25 LG BAG 1099.00 es cajole f +190010 misty mint white seashell papaya Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 38 JUMBO BOX 1100.01 pecia + +-- !q12 -- +100001 seashell cyan plum purple honeydew Manufacturer#3 Brand#35 STANDARD BRUSHED TIN 37 JUMBO CASE 1001.00 ronic dependencies d +100002 steel moccasin forest cornflower brown Manufacturer#3 Brand#34 STANDARD ANODIZED NICKEL 11 WRAP CAN 1002.00 quickly pending +100003 beige powder violet orchid yellow Manufacturer#2 Brand#21 MEDIUM PLATED BRASS 41 SM BOX 1003.00 carefully even pac +100004 snow blanched khaki indian azure Manufacturer#4 Brand#42 SMALL POLISHED TIN 29 SM CASE 1004.00 sly. blithely +100005 grey midnight orange peach pale Manufacturer#2 Brand#21 SMALL POLISHED STEEL 7 MED BAG 1005.00 ajole? blithe +100006 violet sandy olive yellow orange Manufacturer#4 Brand#45 STANDARD BURNISHED COPPER 23 WRAP CASE 1006.00 he slyly regular pack +100007 snow magenta pale lemon metallic Manufacturer#1 Brand#12 PROMO BURNISHED COPPER 4 MED PKG 1007.00 ronic accounts in +100008 spring powder sienna purple lime Manufacturer#4 Brand#45 ECONOMY BRUSHED BRASS 19 SM PKG 1008.00 ts. furious +100009 goldenrod sandy beige hot orange Manufacturer#3 Brand#32 SMALL BURNISHED STEEL 41 WRAP BOX 1009.00 dinos about the quick +100010 lime lavender slate cream brown Manufacturer#4 Brand#43 PROMO ANODIZED COPPER 19 JUMBO PACK 1010.01 gle slyly above the b + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy b/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy new file mode 100644 index 00000000000000..67e594b21f180c --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_parquet_skip_page", "p0,external,hive,external_docker,external_docker_hive") { + def q01 = { + qt_q01 """ + select * from lineitem where l_orderkey < 1000 order by l_orderkey,l_partkey limit 10; + """ + } + + def q02 = { + qt_q02 """ + select * from lineitem where l_orderkey > 5999000 order by l_orderkey,l_partkey limit 10; + """ + } + + def q03 = { + qt_q03 """ + select * from lineitem where l_orderkey > 2000000 and l_orderkey < 2001000 order by l_orderkey,l_partkey limit 10; + """ + } + + def q04 = { + qt_q04 """ + select * from customer where c_custkey < 10000 order by c_custkey limit 10; + """ + } + + def q05 = { + qt_q05 """ + select * from customer where c_custkey > 140000 order by c_custkey limit 10; + """ + } + + def q06 = { + qt_q06 """ + select * from customer where c_custkey > 100000 and c_custkey < 110000 order by c_custkey limit 10; + """ + } + + def q07 = { + qt_q07 """ + select * from orders where o_orderkey < 10000 order by o_orderkey limit 10; + """ + } + + def q08 = { + qt_q08 """ + select * from orders where o_orderkey > 5990000 order by o_orderkey limit 10; + """ + } + + def q09 = { + qt_q09 """ + select * from orders where o_orderkey > 2000000 and o_orderkey < 2010000 order by o_orderkey limit 10; + """ + } + + def q10 = { + qt_q10 """ + select * from part where p_partkey < 10000 order by p_partkey limit 10; + """ + } + + def q11 = { + qt_q08 """ + select * from part where p_partkey > 190000 order by p_partkey limit 10; + """ + } + + def q12 = { + qt_q12 """ + select * from part where p_partkey > 100000 and p_partkey < 110000 order by p_partkey limit 10; + """ + } + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + for (String hivePrefix : ["hive2", "hive3"]) { + try { + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_parquet" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + sql """switch ${catalog_name}""" + sql """use `tpch1_parquet`""" + + sql """set enable_profile=true;""" + + q01() + q02() + q03() + q04() + q05() + q06() + q07() + q08() + q09() + q10() + q11() + q12() + + sql """drop catalog if exists ${catalog_name}""" + } finally { + } + } +} From 580181fd1f4070dbfc29a48fbb6712d1cbc243ae Mon Sep 17 00:00:00 2001 From: Lei Zhang <27994433+SWJTU-ZhangLei@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:12:11 +0800 Subject: [PATCH 050/163] [fix](merge-cloud) Fix brpc mbvar prometheus format issue by patching brpc (#34098) * need to update thirdparty * https://github.com/apache/brpc/pull/2235 --- .../brpc-1.8.0-mbvar-format-issue.patch | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 thirdparty/patches/brpc-1.8.0-mbvar-format-issue.patch diff --git a/thirdparty/patches/brpc-1.8.0-mbvar-format-issue.patch b/thirdparty/patches/brpc-1.8.0-mbvar-format-issue.patch new file mode 100644 index 00000000000000..8e8bb1349e9e77 --- /dev/null +++ b/thirdparty/patches/brpc-1.8.0-mbvar-format-issue.patch @@ -0,0 +1,107 @@ +From 6de560b84fb4cc37461bc6698ea2effd64678465 Mon Sep 17 00:00:00 2001 +From: dylan <451809218@qq.com> +Date: Tue, 7 Nov 2023 14:05:37 +0800 +Subject: [PATCH] Fixup mbvar convert prometheus metrics format issue (#2082) + (#2235) + +--- + .../builtin/prometheus_metrics_service.cpp | 13 +++++- + src/brpc/builtin/prometheus_metrics_service.h | 1 + + ...pc_prometheus_metrics_service_unittest.cpp | 42 +++++++++++++++++++ + 3 files changed, 54 insertions(+), 2 deletions(-) + create mode 100644 test/brpc_prometheus_metrics_service_unittest.cpp + +diff --git a/src/brpc/builtin/prometheus_metrics_service.cpp b/src/brpc/builtin/prometheus_metrics_service.cpp +index 7bf8bbf3..88f675bb 100644 +--- a/src/brpc/builtin/prometheus_metrics_service.cpp ++++ b/src/brpc/builtin/prometheus_metrics_service.cpp +@@ -82,6 +82,12 @@ private: + std::map _m; + }; + ++butil::StringPiece GetMetricsName(const std::string& name) { ++ auto pos = name.find_first_of('{'); ++ int size = (pos == std::string::npos) ? name.size() : pos; ++ return butil::StringPiece(name.data(), size); ++} ++ + bool PrometheusMetricsDumper::dump(const std::string& name, + const butil::StringPiece& desc) { + if (!desc.empty() && desc[0] == '"') { +@@ -93,8 +99,11 @@ bool PrometheusMetricsDumper::dump(const std::string& name, + // Leave it to DumpLatencyRecorderSuffix to output Summary. + return true; + } +- *_os << "# HELP " << name << '\n' +- << "# TYPE " << name << " gauge" << '\n' ++ ++ auto metrics_name = GetMetricsName(name); ++ ++ *_os << "# HELP " << metrics_name << '\n' ++ << "# TYPE " << metrics_name << " gauge" << '\n' + << name << " " << desc << '\n'; + return true; + } +diff --git a/src/brpc/builtin/prometheus_metrics_service.h b/src/brpc/builtin/prometheus_metrics_service.h +index c844e1e7..541b395c 100644 +--- a/src/brpc/builtin/prometheus_metrics_service.h ++++ b/src/brpc/builtin/prometheus_metrics_service.h +@@ -31,6 +31,7 @@ public: + ::google::protobuf::Closure* done) override; + }; + ++butil::StringPiece GetMetricsName(const std::string& name); + int DumpPrometheusMetricsToIOBuf(butil::IOBuf* output); + + } // namepace brpc +diff --git a/test/brpc_prometheus_metrics_service_unittest.cpp b/test/brpc_prometheus_metrics_service_unittest.cpp +new file mode 100644 +index 00000000..b5b0bc10 +--- /dev/null ++++ b/test/brpc_prometheus_metrics_service_unittest.cpp +@@ -0,0 +1,42 @@ ++// Licensed to the Apache Software Foundation (ASF) under one ++// or more contributor license agreements. See the NOTICE file ++// distributed with this work for additional information ++// regarding copyright ownership. The ASF licenses this file ++// to you under the Apache License, Version 2.0 (the ++// "License"); you may not use this file except in compliance ++// with the License. You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, ++// software distributed under the License is distributed on an ++// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY ++// KIND, either express or implied. See the License for the ++// specific language governing permissions and limitations ++// under the License. ++ ++// Date: 2023/05/06 15:10:00 ++ ++#include ++ ++#include "butil/strings/string_piece.h" ++#include "butil/iobuf.h" ++#include "brpc/builtin/prometheus_metrics_service.h" ++ ++namespace { ++ ++class PrometheusMetricsDumperTest : public testing::Test { ++protected: ++ void SetUp() {} ++ void TearDown() {} ++}; ++ ++TEST_F(PrometheusMetricsDumperTest, GetMetricsName) { ++ EXPECT_EQ("", brpc::GetMetricsName("")); ++ ++ EXPECT_EQ("commit_count", brpc::GetMetricsName("commit_count")); ++ ++ EXPECT_EQ("commit_count", brpc::GetMetricsName("commit_count{region=\"1000\"}")); ++} ++ ++} +-- +2.39.3 + From ce55fc4c444cf35d25c02a66e62bfd22d91ffab0 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:31:49 +0800 Subject: [PATCH 051/163] [fix](Nereids) support not in predicate for delete command (#34153) --- .../doris/nereids/trees/plans/commands/DeleteFromCommand.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java index 7fc4657a17fec3..6d339a21d943cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java @@ -326,6 +326,8 @@ private void checkPredicate(Expression predicate) { checkIsNull((IsNull) child); } else if (child instanceof ComparisonPredicate) { checkComparisonPredicate((ComparisonPredicate) child); + } else if (child instanceof InPredicate) { + checkInPredicate((InPredicate) child); } else { throw new AnalysisException("Where clause only supports compound predicate," + " binary predicate, is_null predicate or in predicate. But we meet " From 7a26e07f3bba0a6f011cccdf6bdf72c87a928b38 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Fri, 26 Apr 2024 14:33:03 +0800 Subject: [PATCH 052/163] [fix](pipeline_x) Crc32HashPartitioner should use ShuffleChannelIds (#34147) --- be/src/pipeline/exec/exchange_sink_operator.cpp | 8 ++++---- .../pipeline/exec/partitioned_hash_join_probe_operator.h | 4 ++-- .../pipeline/exec/partitioned_hash_join_sink_operator.h | 4 ++-- .../local_exchange/local_exchange_sink_operator.h | 4 ++-- be/src/vec/runtime/partitioner.cpp | 1 - 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp index 79a6ee0e74881f..84381c6a8af746 100644 --- a/be/src/pipeline/exec/exchange_sink_operator.cpp +++ b/be/src/pipeline/exec/exchange_sink_operator.cpp @@ -221,8 +221,8 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) { } if (_part_type == TPartitionType::HASH_PARTITIONED) { _partition_count = channels.size(); - _partitioner.reset( - new vectorized::Crc32HashPartitioner(channels.size())); + _partitioner.reset(new vectorized::Crc32HashPartitioner( + channels.size())); RETURN_IF_ERROR(_partitioner->init(p._texprs)); RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc)); _profile->add_info_string("Partitioner", @@ -269,8 +269,8 @@ Status ExchangeSinkLocalState::open(RuntimeState* state) { } else if (_part_type == TPartitionType::TABLE_SINK_HASH_PARTITIONED) { _partition_count = channels.size() * config::table_sink_partition_write_max_partition_nums_per_writer; - _partitioner.reset( - new vectorized::Crc32HashPartitioner(_partition_count)); + _partitioner.reset(new vectorized::Crc32HashPartitioner( + _partition_count)); _partition_function.reset(new HashPartitionFunction(_partitioner.get())); scale_writer_partitioning_exchanger.reset(new vectorized::ScaleWriterPartitioningExchanger< diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h index 5bdc5278ffcf5e..3702c2e1a6bd7a 100644 --- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h +++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.h @@ -24,16 +24,16 @@ #include "pipeline/exec/hashjoin_build_sink.h" #include "pipeline/exec/hashjoin_probe_operator.h" #include "pipeline/exec/join_build_sink_operator.h" -#include "pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h" // LocalExchangeChannelIds #include "pipeline/pipeline_x/operator.h" #include "vec/runtime/partitioner.h" +#include "vec/sink/vdata_stream_sender.h" // ShuffleChannelIds namespace doris { class RuntimeState; namespace pipeline { -using PartitionerType = vectorized::Crc32HashPartitioner; +using PartitionerType = vectorized::Crc32HashPartitioner; class PartitionedHashJoinProbeOperatorX; diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h index 3f29e3093b6df0..68c6b970163f24 100644 --- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h +++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h @@ -24,9 +24,9 @@ #include "pipeline/exec/hashjoin_build_sink.h" #include "pipeline/exec/hashjoin_probe_operator.h" #include "pipeline/exec/join_build_sink_operator.h" -#include "pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h" // LocalExchangeChannelIds #include "pipeline/pipeline_x/operator.h" #include "vec/runtime/partitioner.h" +#include "vec/sink/vdata_stream_sender.h" // ShuffleChannelIds namespace doris { class ExecNode; @@ -34,7 +34,7 @@ class RuntimeState; namespace pipeline { -using PartitionerType = vectorized::Crc32HashPartitioner; +using PartitionerType = vectorized::Crc32HashPartitioner; class PartitionedHashJoinSinkOperatorX; diff --git a/be/src/pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h b/be/src/pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h index b3ecf29736fda6..db6662a221ad8e 100644 --- a/be/src/pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h +++ b/be/src/pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h @@ -114,8 +114,8 @@ class LocalExchangeSinkOperatorX final : public DataSinkOperatorX(_num_partitions)); + _partitioner.reset(new vectorized::Crc32HashPartitioner( + _num_partitions)); RETURN_IF_ERROR(_partitioner->init(_texprs)); } else if (_type == ExchangeType::BUCKET_HASH_SHUFFLE) { _partitioner.reset(new vectorized::Crc32HashPartitioner( diff --git a/be/src/vec/runtime/partitioner.cpp b/be/src/vec/runtime/partitioner.cpp index db40610723cdb6..fadf6d73b95f76 100644 --- a/be/src/vec/runtime/partitioner.cpp +++ b/be/src/vec/runtime/partitioner.cpp @@ -103,6 +103,5 @@ template class Partitioner; template class XXHashPartitioner; template class Partitioner; template class Crc32HashPartitioner; -template class Crc32HashPartitioner; } // namespace doris::vectorized From 464620431d69e1959287469119e50752c16261eb Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Fri, 26 Apr 2024 16:33:40 +0800 Subject: [PATCH 053/163] Improve analyze timeout. (#33836) --- .../java/org/apache/doris/statistics/AnalysisManager.java | 4 ++-- .../org/apache/doris/statistics/ExternalAnalysisTask.java | 3 +++ .../java/org/apache/doris/statistics/JdbcAnalysisTask.java | 3 +++ .../java/org/apache/doris/statistics/OlapAnalysisTask.java | 3 +++ .../hive/test_hive_statistic_timeout.groovy | 7 ++++--- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 03314fe7748a13..b697c6f8327c64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -867,7 +867,7 @@ public void execute(ThreadPoolExecutor executor) { executor.submit(() -> { try { if (cancelled) { - errorMessages.add("Query timeout or user cancelled." + errorMessages.add("Query Timeout or user Cancelled." + "Could set analyze_timeout to a bigger value."); return; } @@ -890,7 +890,7 @@ public void execute(ThreadPoolExecutor executor) { } if (!colNames.isEmpty()) { if (cancelled) { - throw new RuntimeException("Cancelled"); + throw new RuntimeException("User Cancelled or Timeout."); } throw new RuntimeException("Failed to analyze following columns:[" + String.join(",", colNames) + "] Reasons: " + String.join(",", errorMessages)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java index 7d3c9af254800d..d59133017ab128 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java @@ -56,6 +56,9 @@ public ExternalAnalysisTask(AnalysisInfo info) { } public void doExecute() throws Exception { + if (killed) { + return; + } if (isTableLevelTask) { getTableStats(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java index a318bd4594e5f6..4562d4336e8aee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java @@ -60,6 +60,9 @@ public JdbcAnalysisTask(AnalysisInfo info) { } public void doExecute() throws Exception { + if (killed) { + return; + } if (isTableLevelTask) { getTableStats(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index ce7982d4f1ad58..4b651d322ba76a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -64,6 +64,9 @@ public OlapAnalysisTask(AnalysisInfo info) { } public void doExecute() throws Exception { + if (killed) { + return; + } // For empty table, write empty result directly, no need to run SQL to collect stats. if (info.rowCount == 0 && tableSample != null) { StatsId statsId = new StatsId(concatColumnStatsId(), info.catalogId, info.dbId, diff --git a/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy b/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy index a3329f87712802..a52a230478851b 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_statistic_timeout.groovy @@ -34,9 +34,10 @@ suite("test_hive_statistic_timeout", "p2,external,hive,external_remote,external_ sql """use ${catalog_name}.tpch_1000_parquet""" sql """set global analyze_timeout=1""" try { - sql """analyze table part (p_partkey, p_container, p_type, p_retailprice) with sync with full;""" - } catch (Exception e) { - assertTrue(e.getMessage().contains("Cancelled")); + test { + sql """analyze table part (p_partkey, p_container, p_type, p_retailprice) with sync with full;""" + exception "Timeout" + } } finally { sql """set global analyze_timeout=43200""" } From 4e125b0d567adccd9dde1fc7bd0d805a93c85e89 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 26 Apr 2024 17:18:42 +0800 Subject: [PATCH 054/163] [refactor](cleanup) Clean up pipeline engine (PART I) (#33945) --- .../exec/aggregation_sink_operator.cpp | 4 - .../pipeline/exec/aggregation_sink_operator.h | 23 +- .../exec/aggregation_source_operator.cpp | 3 - .../exec/aggregation_source_operator.h | 20 - .../pipeline/exec/analytic_sink_operator.cpp | 2 - be/src/pipeline/exec/analytic_sink_operator.h | 2 - .../exec/analytic_source_operator.cpp | 2 - .../pipeline/exec/analytic_source_operator.h | 18 - .../exec/assert_num_rows_operator.cpp | 4 - .../pipeline/exec/assert_num_rows_operator.h | 22 +- be/src/pipeline/exec/const_value_operator.h | 52 - be/src/pipeline/exec/datagen_operator.cpp | 14 - be/src/pipeline/exec/datagen_operator.h | 19 - ...ct_streaming_aggregation_sink_operator.cpp | 100 - ...inct_streaming_aggregation_sink_operator.h | 79 - ..._streaming_aggregation_source_operator.cpp | 91 - ...ct_streaming_aggregation_source_operator.h | 67 - be/src/pipeline/exec/empty_set_operator.cpp | 2 - be/src/pipeline/exec/empty_set_operator.h | 24 +- .../pipeline/exec/empty_source_operator.cpp | 27 - be/src/pipeline/exec/empty_source_operator.h | 89 - be/src/pipeline/exec/es_scan_operator.h | 2 - .../pipeline/exec/exchange_sink_operator.cpp | 53 - be/src/pipeline/exec/exchange_sink_operator.h | 30 - .../exec/exchange_source_operator.cpp | 10 - .../pipeline/exec/exchange_source_operator.h | 17 - be/src/pipeline/exec/file_scan_operator.h | 2 - .../exec/group_commit_block_sink_operator.cpp | 4 - .../exec/group_commit_block_sink_operator.h | 25 +- be/src/pipeline/exec/hashjoin_build_sink.cpp | 2 - be/src/pipeline/exec/hashjoin_build_sink.h | 9 +- .../pipeline/exec/hashjoin_probe_operator.cpp | 8 +- .../pipeline/exec/hashjoin_probe_operator.h | 17 - .../exec/hive_table_sink_operator.cpp | 4 - .../pipeline/exec/hive_table_sink_operator.h | 24 +- be/src/pipeline/exec/jdbc_scan_operator.h | 1 - .../pipeline/exec/join_build_sink_operator.h | 8 +- be/src/pipeline/exec/join_probe_operator.h | 8 +- be/src/pipeline/exec/meta_scan_operator.h | 2 - .../exec/multi_cast_data_stream_sink.cpp | 4 - .../exec/multi_cast_data_stream_sink.h | 19 - .../exec/multi_cast_data_stream_source.cpp | 98 - .../exec/multi_cast_data_stream_source.h | 1 - be/src/pipeline/exec/mysql_scan_operator.cpp | 37 - be/src/pipeline/exec/mysql_scan_operator.h | 43 - .../exec/nested_loop_join_build_operator.cpp | 3 +- .../exec/nested_loop_join_build_operator.h | 24 +- .../exec/nested_loop_join_probe_operator.cpp | 13 - .../exec/nested_loop_join_probe_operator.h | 21 - be/src/pipeline/exec/olap_scan_operator.h | 2 - .../exec/olap_table_sink_operator.cpp | 8 - .../pipeline/exec/olap_table_sink_operator.h | 24 +- .../exec/olap_table_sink_v2_operator.cpp | 4 - .../exec/olap_table_sink_v2_operator.h | 7 +- be/src/pipeline/exec/operator.h | 11 - .../exec/partition_sort_sink_operator.cpp | 4 - .../exec/partition_sort_sink_operator.h | 28 +- .../exec/partition_sort_source_operator.cpp | 5 - .../exec/partition_sort_source_operator.h | 20 - .../partitioned_aggregation_source_operator.h | 1 - .../partitioned_hash_join_sink_operator.h | 1 - be/src/pipeline/exec/repeat_operator.cpp | 13 - be/src/pipeline/exec/repeat_operator.h | 18 - .../exec/result_file_sink_operator.cpp | 16 - .../pipeline/exec/result_file_sink_operator.h | 23 +- be/src/pipeline/exec/result_sink_operator.cpp | 19 - be/src/pipeline/exec/result_sink_operator.h | 16 - be/src/pipeline/exec/scan_operator.cpp | 33 - be/src/pipeline/exec/scan_operator.h | 21 - be/src/pipeline/exec/schema_scan_operator.cpp | 13 - be/src/pipeline/exec/schema_scan_operator.h | 19 - be/src/pipeline/exec/select_operator.cpp | 28 - be/src/pipeline/exec/select_operator.h | 21 +- .../pipeline/exec/set_probe_sink_operator.cpp | 37 - .../pipeline/exec/set_probe_sink_operator.h | 33 - be/src/pipeline/exec/set_sink_operator.cpp | 25 - be/src/pipeline/exec/set_sink_operator.h | 28 - be/src/pipeline/exec/set_source_operator.cpp | 25 - be/src/pipeline/exec/set_source_operator.h | 25 - be/src/pipeline/exec/sort_sink_operator.cpp | 2 - be/src/pipeline/exec/sort_sink_operator.h | 25 +- be/src/pipeline/exec/sort_source_operator.cpp | 2 - be/src/pipeline/exec/sort_source_operator.h | 17 - .../streaming_aggregation_sink_operator.cpp | 93 - .../streaming_aggregation_sink_operator.h | 76 - .../streaming_aggregation_source_operator.cpp | 76 - .../streaming_aggregation_source_operator.h | 64 - .../pipeline/exec/table_function_operator.cpp | 12 - .../pipeline/exec/table_function_operator.h | 17 - be/src/pipeline/exec/table_sink_operator.h | 49 - be/src/pipeline/exec/union_sink_operator.cpp | 65 - be/src/pipeline/exec/union_sink_operator.h | 34 - .../pipeline/exec/union_source_operator.cpp | 73 - be/src/pipeline/exec/union_source_operator.h | 35 - be/src/pipeline/pipeline.h | 2 +- be/src/pipeline/pipeline_fragment_context.cpp | 1688 ++++++++++++----- be/src/pipeline/pipeline_fragment_context.h | 220 ++- be/src/pipeline/pipeline_task.cpp | 34 +- be/src/pipeline/pipeline_task.h | 2 - be/src/pipeline/pipeline_x/operator.cpp | 2 - .../pipeline_x_fragment_context.cpp | 1522 --------------- .../pipeline_x/pipeline_x_fragment_context.h | 247 --- .../pipeline/pipeline_x/pipeline_x_task.cpp | 2 +- be/src/pipeline/pipeline_x/pipeline_x_task.h | 2 - be/src/pipeline/task_scheduler.cpp | 218 +-- be/src/pipeline/task_scheduler.h | 33 +- be/src/runtime/exec_env.h | 10 - be/src/runtime/exec_env_init.cpp | 11 +- be/src/runtime/fragment_mgr.cpp | 8 +- be/src/runtime/fragment_mgr.h | 3 +- be/src/runtime/query_context.cpp | 11 +- be/src/runtime/runtime_state.cpp | 2 +- be/src/runtime/runtime_state.h | 6 +- .../runtime/workload_group/workload_group.cpp | 5 +- 114 files changed, 1408 insertions(+), 4941 deletions(-) delete mode 100644 be/src/pipeline/exec/const_value_operator.h delete mode 100644 be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.cpp delete mode 100644 be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h delete mode 100644 be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.cpp delete mode 100644 be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h delete mode 100644 be/src/pipeline/exec/empty_source_operator.cpp delete mode 100644 be/src/pipeline/exec/empty_source_operator.h delete mode 100644 be/src/pipeline/exec/mysql_scan_operator.cpp delete mode 100644 be/src/pipeline/exec/mysql_scan_operator.h delete mode 100644 be/src/pipeline/exec/select_operator.cpp delete mode 100644 be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp delete mode 100644 be/src/pipeline/exec/streaming_aggregation_sink_operator.h delete mode 100644 be/src/pipeline/exec/streaming_aggregation_source_operator.cpp delete mode 100644 be/src/pipeline/exec/streaming_aggregation_source_operator.h delete mode 100644 be/src/pipeline/exec/table_sink_operator.h delete mode 100644 be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp delete mode 100644 be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h diff --git a/be/src/pipeline/exec/aggregation_sink_operator.cpp b/be/src/pipeline/exec/aggregation_sink_operator.cpp index 6c9d27e2a2b063..d947dd55c36194 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.cpp +++ b/be/src/pipeline/exec/aggregation_sink_operator.cpp @@ -19,16 +19,12 @@ #include -#include "pipeline/exec/distinct_streaming_aggregation_sink_operator.h" #include "pipeline/exec/operator.h" -#include "pipeline/exec/streaming_aggregation_sink_operator.h" #include "runtime/primitive_type.h" #include "vec/common/hash_table/hash.h" namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(AggSinkOperator, StreamingOperator) - /// The minimum reduction factor (input rows divided by output rows) to grow hash tables /// in a streaming preaggregation, given that the hash tables are currently the given /// size or above. The sizes roughly correspond to hash table sizes where the bucket diff --git a/be/src/pipeline/exec/aggregation_sink_operator.h b/be/src/pipeline/exec/aggregation_sink_operator.h index 0c34acfd7dfe84..ee31db3691759d 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.h +++ b/be/src/pipeline/exec/aggregation_sink_operator.h @@ -23,26 +23,8 @@ #include "pipeline/pipeline_x/operator.h" #include "runtime/block_spill_manager.h" #include "runtime/exec_env.h" -#include "vec/exec/vaggregation_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { - -class AggSinkOperatorBuilder final : public OperatorBuilder { -public: - AggSinkOperatorBuilder(int32_t, ExecNode*); - - OperatorPtr build_operator() override; - bool is_sink() const override { return true; } -}; - -class AggSinkOperator final : public StreamingOperator { -public: - AggSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* node); - bool can_write() override { return true; } -}; +namespace doris::pipeline { class AggSinkOperatorX; @@ -217,5 +199,4 @@ class AggSinkOperatorX final : public DataSinkOperatorX { RowDescriptor _agg_fn_output_row_descriptor; }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/aggregation_source_operator.cpp b/be/src/pipeline/exec/aggregation_source_operator.cpp index cff6f9fec42fe9..f53d96d71aa2ae 100644 --- a/be/src/pipeline/exec/aggregation_source_operator.cpp +++ b/be/src/pipeline/exec/aggregation_source_operator.cpp @@ -22,13 +22,10 @@ #include "common/exception.h" #include "pipeline/exec/operator.h" -#include "pipeline/exec/streaming_aggregation_source_operator.h" #include "vec//utils/util.hpp" namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(AggSourceOperator, SourceOperator) - AggLocalState::AggLocalState(RuntimeState* state, OperatorXBase* parent) : Base(state, parent), _get_results_timer(nullptr), diff --git a/be/src/pipeline/exec/aggregation_source_operator.h b/be/src/pipeline/exec/aggregation_source_operator.h index 1d1f564d41aa62..d43ec7db0b550d 100644 --- a/be/src/pipeline/exec/aggregation_source_operator.h +++ b/be/src/pipeline/exec/aggregation_source_operator.h @@ -21,32 +21,12 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vaggregation_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class AggSourceOperatorBuilder final : public OperatorBuilder { -public: - AggSourceOperatorBuilder(int32_t, ExecNode*); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class AggSourceOperator final : public SourceOperator { -public: - AggSourceOperator(OperatorBuilderBase*, ExecNode*); - // if exec node split to: sink, source operator. the source operator - // should skip `alloc_resource()` function call, only sink operator - // call the function - Status open(RuntimeState*) override { return Status::OK(); } -}; - class AggSourceOperatorX; class AggLocalState final : public PipelineXLocalState { diff --git a/be/src/pipeline/exec/analytic_sink_operator.cpp b/be/src/pipeline/exec/analytic_sink_operator.cpp index a1d3384edc6dde..12c4e7634e71a6 100644 --- a/be/src/pipeline/exec/analytic_sink_operator.cpp +++ b/be/src/pipeline/exec/analytic_sink_operator.cpp @@ -24,8 +24,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(AnalyticSinkOperator, StreamingOperator) - Status AnalyticSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(PipelineXSinkLocalState::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/analytic_sink_operator.h b/be/src/pipeline/exec/analytic_sink_operator.h index 3ae4a7b5cff5ca..0098c108e53d4e 100644 --- a/be/src/pipeline/exec/analytic_sink_operator.h +++ b/be/src/pipeline/exec/analytic_sink_operator.h @@ -23,10 +23,8 @@ #include "operator.h" #include "pipeline/pipeline_x/dependency.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vanalytic_eval_node.h" namespace doris { -class ExecNode; namespace pipeline { class AnalyticSinkOperatorBuilder final : public OperatorBuilder { diff --git a/be/src/pipeline/exec/analytic_source_operator.cpp b/be/src/pipeline/exec/analytic_source_operator.cpp index f6658583d4657a..85995732fff3d4 100644 --- a/be/src/pipeline/exec/analytic_source_operator.cpp +++ b/be/src/pipeline/exec/analytic_source_operator.cpp @@ -24,8 +24,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(AnalyticSourceOperator, SourceOperator) - AnalyticLocalState::AnalyticLocalState(RuntimeState* state, OperatorXBase* parent) : PipelineXLocalState(state, parent), _output_block_index(0), diff --git a/be/src/pipeline/exec/analytic_source_operator.h b/be/src/pipeline/exec/analytic_source_operator.h index 17a4d34ec739b3..075053a486600c 100644 --- a/be/src/pipeline/exec/analytic_source_operator.h +++ b/be/src/pipeline/exec/analytic_source_operator.h @@ -22,30 +22,12 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vanalytic_eval_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class AnalyticSourceOperatorBuilder final : public OperatorBuilder { -public: - AnalyticSourceOperatorBuilder(int32_t, ExecNode*); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class AnalyticSourceOperator final : public SourceOperator { -public: - AnalyticSourceOperator(OperatorBuilderBase*, ExecNode*); - - Status open(RuntimeState*) override { return Status::OK(); } -}; - class AnalyticSourceOperatorX; class AnalyticLocalState final : public PipelineXLocalState { public: diff --git a/be/src/pipeline/exec/assert_num_rows_operator.cpp b/be/src/pipeline/exec/assert_num_rows_operator.cpp index ef0efd3f86bf2b..4a51002beff389 100644 --- a/be/src/pipeline/exec/assert_num_rows_operator.cpp +++ b/be/src/pipeline/exec/assert_num_rows_operator.cpp @@ -22,10 +22,6 @@ namespace doris::pipeline { -OperatorPtr AssertNumRowsOperatorBuilder::build_operator() { - return std::make_shared(this, _node); -} - AssertNumRowsOperatorX::AssertNumRowsOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id, const DescriptorTbl& descs) : StreamingOperatorX(pool, tnode, operator_id, descs), diff --git a/be/src/pipeline/exec/assert_num_rows_operator.h b/be/src/pipeline/exec/assert_num_rows_operator.h index 4d6d835f815aa4..07a90d6b471144 100644 --- a/be/src/pipeline/exec/assert_num_rows_operator.h +++ b/be/src/pipeline/exec/assert_num_rows_operator.h @@ -19,25 +19,8 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vassert_num_rows_node.h" -namespace doris { - -namespace pipeline { - -class AssertNumRowsOperatorBuilder final : public OperatorBuilder { -public: - AssertNumRowsOperatorBuilder(int32_t id, ExecNode* node) - : OperatorBuilder(id, "AssertNumRowsOperator", node) {} - - OperatorPtr build_operator() override; -}; - -class AssertNumRowsOperator final : public StreamingOperator { -public: - AssertNumRowsOperator(OperatorBuilderBase* operator_builder, ExecNode* node) - : StreamingOperator(operator_builder, node) {} -}; +namespace doris::pipeline { class AssertNumRowsLocalState final : public PipelineXLocalState { public: @@ -70,5 +53,4 @@ class AssertNumRowsOperatorX final : public StreamingOperatorX { -public: - ConstValueOperatorBuilder(int32_t id, ExecNode* node) - : OperatorBuilder(id, "ConstValueOperator", node) {} - - OperatorPtr build_operator() override; - - bool is_source() const override { return true; } -}; - -class ConstValueOperator final : public SourceOperator { -public: - ConstValueOperator(OperatorBuilderBase* operator_builder, ExecNode* node) - : SourceOperator(operator_builder, node) {} - - bool can_read() override { return true; } -}; - -OperatorPtr ConstValueOperatorBuilder::build_operator() { - return std::make_shared(this, _node); -} - -} // namespace pipeline -} // namespace doris \ No newline at end of file diff --git a/be/src/pipeline/exec/datagen_operator.cpp b/be/src/pipeline/exec/datagen_operator.cpp index 4fbe21f71d5e32..95b284c94b4950 100644 --- a/be/src/pipeline/exec/datagen_operator.cpp +++ b/be/src/pipeline/exec/datagen_operator.cpp @@ -23,7 +23,6 @@ #include "util/runtime_profile.h" #include "vec/exec/data_gen_functions/vdata_gen_function_inf.h" #include "vec/exec/data_gen_functions/vnumbers_tvf.h" -#include "vec/exec/vdata_gen_scan_node.h" namespace doris { class RuntimeState; @@ -31,19 +30,6 @@ class RuntimeState; namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(DataGenOperator, SourceOperator) - -Status DataGenOperator::open(RuntimeState* state) { - RETURN_IF_ERROR(SourceOperator::open(state)); - return _node->open(state); -} - -Status DataGenOperator::close(RuntimeState* state) { - RETURN_IF_ERROR(SourceOperator::close(state)); - RETURN_IF_ERROR(_node->close(state)); - return Status::OK(); -} - DataGenSourceOperatorX::DataGenSourceOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id, const DescriptorTbl& descs) : OperatorX(pool, tnode, operator_id, descs), diff --git a/be/src/pipeline/exec/datagen_operator.h b/be/src/pipeline/exec/datagen_operator.h index af8eda179dac7c..edcc85c8342e12 100644 --- a/be/src/pipeline/exec/datagen_operator.h +++ b/be/src/pipeline/exec/datagen_operator.h @@ -25,30 +25,11 @@ #include "vec/exec/vdata_gen_scan_node.h" namespace doris { -class ExecNode; class RuntimeState; } // namespace doris namespace doris::pipeline { -class DataGenOperatorBuilder : public OperatorBuilder { -public: - DataGenOperatorBuilder(int32_t id, ExecNode* exec_node); - bool is_source() const override { return true; } - OperatorPtr build_operator() override; -}; - -class DataGenOperator : public SourceOperator { -public: - DataGenOperator(OperatorBuilderBase* operator_builder, ExecNode* datagen_node); - - bool can_read() override { return true; } - - Status open(RuntimeState* state) override; - - Status close(RuntimeState* state) override; -}; - class DataGenSourceOperatorX; class DataGenLocalState final : public PipelineXLocalState<> { public: diff --git a/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.cpp b/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.cpp deleted file mode 100644 index 3cb18168dcb215..00000000000000 --- a/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "distinct_streaming_aggregation_sink_operator.h" - -#include - -#include -#include - -#include "common/compiler_util.h" // IWYU pragma: keep -#include "pipeline/exec/data_queue.h" -#include "pipeline/exec/operator.h" -#include "vec/exec/distinct_vaggregation_node.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; -} // namespace doris - -namespace doris::pipeline { - -DistinctStreamingAggSinkOperator::DistinctStreamingAggSinkOperator( - OperatorBuilderBase* operator_builder, ExecNode* agg_node, std::shared_ptr queue) - : StreamingOperator(operator_builder, agg_node), _data_queue(std::move(queue)) {} - -bool DistinctStreamingAggSinkOperator::can_write() { - // sink and source in diff threads - return _data_queue->has_enough_space_to_push(); -} - -Status DistinctStreamingAggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, - SourceState source_state) { - if (in_block && in_block->rows() > 0) { - if (_output_block == nullptr) { - _output_block = _data_queue->get_free_block(); - } - RETURN_IF_ERROR( - _node->_distinct_pre_agg_with_serialized_key(in_block, _output_block.get())); - bool stop_emplace_flag = _node->is_stop_emplace_flag(); - // get enough data or reached limit rows, need push block to queue - if (!stop_emplace_flag && _node->limit() != -1 && - (_output_block->rows() + _output_distinct_rows) >= _node->limit()) { - auto limit_rows = _node->limit() - _output_distinct_rows; - _output_block->set_num_rows(limit_rows); - _output_distinct_rows += limit_rows; - _data_queue->push_block(std::move(_output_block)); - } else if (stop_emplace_flag || _output_block->rows() >= state->batch_size()) { - if (!stop_emplace_flag) { // if stop_emplace_flag = true, will be return rows directly, not get distinct - _output_distinct_rows += _output_block->rows(); - } - _data_queue->push_block(std::move(_output_block)); - } - } - - // reach limit or source finish - if ((UNLIKELY(source_state == SourceState::FINISHED)) || reached_limited_rows()) { - if (_output_block != nullptr) { //maybe the last block with eos - _output_distinct_rows += _output_block->rows(); - _data_queue->push_block(std::move(_output_block)); - } - _data_queue->set_finish(); - return Status::Error(""); - } - return Status::OK(); -} - -Status DistinctStreamingAggSinkOperator::close(RuntimeState* state) { - if (_data_queue && !_data_queue->is_finish()) { - // finish should be set, if not set here means error. - _data_queue->set_canceled(); - } - return StreamingOperator::close(state); -} - -DistinctStreamingAggSinkOperatorBuilder::DistinctStreamingAggSinkOperatorBuilder( - int32_t id, ExecNode* exec_node, std::shared_ptr queue) - : OperatorBuilder(id, "DistinctStreamingAggSinkOperator", exec_node), - _data_queue(std::move(queue)) {} - -OperatorPtr DistinctStreamingAggSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _node, _data_queue); -} - -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h b/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h deleted file mode 100644 index c872a2b299e14c..00000000000000 --- a/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include -#include - -#include "aggregation_sink_operator.h" -#include "common/status.h" -#include "operator.h" -#include "pipeline/exec/aggregation_sink_operator.h" -#include "pipeline/exec/aggregation_source_operator.h" -#include "util/runtime_profile.h" -#include "vec/core/block.h" -#include "vec/exec/distinct_vaggregation_node.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; - -namespace pipeline { -class DataQueue; - -class DistinctStreamingAggSinkOperatorBuilder final - : public OperatorBuilder { -public: - DistinctStreamingAggSinkOperatorBuilder(int32_t, ExecNode*, std::shared_ptr); - - OperatorPtr build_operator() override; - - bool is_sink() const override { return true; } - bool is_source() const override { return false; } - -private: - std::shared_ptr _data_queue; -}; - -class DistinctStreamingAggSinkOperator final - : public StreamingOperator { -public: - DistinctStreamingAggSinkOperator(OperatorBuilderBase* operator_builder, ExecNode*, - std::shared_ptr); - - Status sink(RuntimeState* state, vectorized::Block* block, SourceState source_state) override; - - bool can_write() override; - - Status close(RuntimeState* state) override; - - bool reached_limited_rows() { - return _node->limit() != -1 && _output_distinct_rows >= _node->limit(); - } - -private: - int64_t _output_distinct_rows = 0; - std::shared_ptr _data_queue; - std::unique_ptr _output_block = vectorized::Block::create_unique(); -}; - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.cpp b/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.cpp deleted file mode 100644 index 5ab8bd30bc845f..00000000000000 --- a/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "distinct_streaming_aggregation_source_operator.h" - -#include - -#include "pipeline/exec/data_queue.h" -#include "pipeline/exec/operator.h" -#include "runtime/descriptors.h" -#include "util/runtime_profile.h" -#include "vec/core/block.h" -#include "vec/exec/distinct_vaggregation_node.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; - -namespace pipeline { -DistinctStreamingAggSourceOperator::DistinctStreamingAggSourceOperator( - OperatorBuilderBase* templ, ExecNode* node, std::shared_ptr queue) - : SourceOperator(templ, node), _data_queue(std::move(queue)) {} - -bool DistinctStreamingAggSourceOperator::can_read() { - return _data_queue->has_data_or_finished(); -} - -Status DistinctStreamingAggSourceOperator::pull_data(RuntimeState* state, vectorized::Block* block, - bool* eos) { - std::unique_ptr agg_block; - RETURN_IF_ERROR(_data_queue->get_block_from_queue(&agg_block)); - if (agg_block != nullptr) { - block->swap(*agg_block); - agg_block->clear_column_data(block->columns()); - _data_queue->push_free_block(std::move(agg_block)); - } - if (_data_queue->data_exhausted()) { //the sink is eos or reached limit - *eos = true; - } - _node->_make_nullable_output_key(block); - if (_node->is_streaming_preagg() == false) { - // dispose the having clause, should not be execute in prestreaming agg - RETURN_IF_ERROR(vectorized::VExprContext::filter_block(_node->get_conjuncts(), block, - block->columns())); - } - - _node->add_num_rows_returned(block->rows()); - return Status::OK(); -} - -Status DistinctStreamingAggSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, - SourceState& source_state) { - bool eos = false; - RETURN_IF_ERROR(_node->get_next_after_projects( - state, block, &eos, - std::bind(&DistinctStreamingAggSourceOperator::pull_data, this, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3))); - if (UNLIKELY(eos)) { - source_state = SourceState::FINISHED; - } else { - source_state = SourceState::DEPEND_ON_SOURCE; - } - return Status::OK(); -} - -DistinctStreamingAggSourceOperatorBuilder::DistinctStreamingAggSourceOperatorBuilder( - int32_t id, ExecNode* exec_node, std::shared_ptr queue) - : OperatorBuilder(id, "DistinctStreamingAggSourceOperator", exec_node), - _data_queue(std::move(queue)) {} - -OperatorPtr DistinctStreamingAggSourceOperatorBuilder::build_operator() { - return std::make_shared(this, _node, _data_queue); -} - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h b/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h deleted file mode 100644 index e8fd21310bbd7b..00000000000000 --- a/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#pragma once - -#include - -#include -#include - -#include "common/status.h" -#include "operator.h" -#include "pipeline/exec/aggregation_source_operator.h" -#include "vec/exec/distinct_vaggregation_node.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; - -namespace vectorized { -class Block; -} // namespace vectorized -namespace pipeline { -class DataQueue; - -class DistinctStreamingAggSourceOperatorBuilder final - : public OperatorBuilder { -public: - DistinctStreamingAggSourceOperatorBuilder(int32_t, ExecNode*, std::shared_ptr); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; - -private: - std::shared_ptr _data_queue; -}; - -class DistinctStreamingAggSourceOperator final - : public SourceOperator { -public: - DistinctStreamingAggSourceOperator(OperatorBuilderBase*, ExecNode*, std::shared_ptr); - bool can_read() override; - Status get_block(RuntimeState*, vectorized::Block*, SourceState& source_state) override; - Status open(RuntimeState*) override { return Status::OK(); } - Status pull_data(RuntimeState* state, vectorized::Block* output_block, bool* eos); - -private: - std::shared_ptr _data_queue; -}; - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/empty_set_operator.cpp b/be/src/pipeline/exec/empty_set_operator.cpp index 02dc80258031b6..7233e46dfd1e52 100644 --- a/be/src/pipeline/exec/empty_set_operator.cpp +++ b/be/src/pipeline/exec/empty_set_operator.cpp @@ -23,8 +23,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(EmptySetSourceOperator, SourceOperator) - Status EmptySetSourceOperatorX::get_block(RuntimeState* state, vectorized::Block* block, bool* eos) { *eos = true; diff --git a/be/src/pipeline/exec/empty_set_operator.h b/be/src/pipeline/exec/empty_set_operator.h index b65139fb985a5f..3d5e9a72cf2bfe 100644 --- a/be/src/pipeline/exec/empty_set_operator.h +++ b/be/src/pipeline/exec/empty_set_operator.h @@ -21,27 +21,8 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vempty_set_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { - -class EmptySetSourceOperatorBuilder final : public OperatorBuilder { -public: - EmptySetSourceOperatorBuilder(int32_t id, ExecNode* empty_set_node); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class EmptySetSourceOperator final : public SourceOperator { -public: - EmptySetSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* empty_set_node); - bool can_read() override { return true; } -}; +namespace doris::pipeline { class EmptySetLocalState final : public PipelineXLocalState { public: @@ -63,5 +44,4 @@ class EmptySetSourceOperatorX final : public OperatorX { [[nodiscard]] bool is_source() const override { return true; } }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/empty_source_operator.cpp b/be/src/pipeline/exec/empty_source_operator.cpp deleted file mode 100644 index 78f5c946621e3b..00000000000000 --- a/be/src/pipeline/exec/empty_source_operator.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "empty_source_operator.h" - -#include "pipeline/exec/operator.h" - -namespace doris::pipeline { -OperatorPtr EmptySourceOperatorBuilder::build_operator() { - return std::make_shared(this, _exec_node); -} - -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/empty_source_operator.h b/be/src/pipeline/exec/empty_source_operator.h deleted file mode 100644 index b85d2b1a2ca9dc..00000000000000 --- a/be/src/pipeline/exec/empty_source_operator.h +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include - -#include "common/status.h" -#include "operator.h" -#include "runtime/descriptors.h" - -namespace doris { -class RuntimeState; - -namespace vectorized { -class Block; -} // namespace vectorized -} // namespace doris - -namespace doris::pipeline { - -class EmptySourceOperatorBuilder final : public OperatorBuilderBase { -public: - EmptySourceOperatorBuilder(int32_t id, const RowDescriptor& row_descriptor, ExecNode* exec_node) - : OperatorBuilderBase(id, "EmptySourceOperator"), - _row_descriptor(row_descriptor), - _exec_node(exec_node) {} - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; - - const RowDescriptor& row_desc() const override { return _row_descriptor; } - -private: - RowDescriptor _row_descriptor; - ExecNode* _exec_node = nullptr; -}; - -class EmptySourceOperator final : public OperatorBase { -public: - EmptySourceOperator(OperatorBuilderBase* builder, ExecNode* exec_node) - : OperatorBase(builder), _exec_node(exec_node) {} - - bool can_read() override { return true; } - bool is_pending_finish() const override { return false; } - - Status prepare(RuntimeState*) override { return Status::OK(); } - - Status open(RuntimeState*) override { return Status::OK(); } - - Status get_block(RuntimeState* /*runtime_state*/, vectorized::Block* /*block*/, - SourceState& result_state) override { - result_state = SourceState::FINISHED; - return Status::OK(); - } - - Status sink(RuntimeState*, vectorized::Block*, SourceState) override { return Status::OK(); } - - Status close(RuntimeState* state) override { - RETURN_IF_ERROR(_exec_node->close(state)); - return Status::OK(); - } - - [[nodiscard]] RuntimeProfile* get_runtime_profile() const override { - return _exec_node->runtime_profile(); - } - -private: - ExecNode* _exec_node = nullptr; -}; - -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/es_scan_operator.h b/be/src/pipeline/exec/es_scan_operator.h index cdbd6922454c11..c4e7772807181e 100644 --- a/be/src/pipeline/exec/es_scan_operator.h +++ b/be/src/pipeline/exec/es_scan_operator.h @@ -25,10 +25,8 @@ #include "operator.h" #include "pipeline/exec/scan_operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/scan/vscan_node.h" namespace doris { -class ExecNode; namespace vectorized { class NewEsScanner; diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp index 84381c6a8af746..7db22c98f9f6b7 100644 --- a/be/src/pipeline/exec/exchange_sink_operator.cpp +++ b/be/src/pipeline/exec/exchange_sink_operator.cpp @@ -30,7 +30,6 @@ #include "pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h" #include "vec/columns/column_const.h" #include "vec/exprs/vexpr.h" -#include "vec/sink/vdata_stream_sender.h" namespace doris { class DataSink; @@ -38,58 +37,6 @@ class DataSink; namespace doris::pipeline { -ExchangeSinkOperatorBuilder::ExchangeSinkOperatorBuilder(int32_t id, DataSink* sink, - int mult_cast_id) - : DataSinkOperatorBuilder(id, "ExchangeSinkOperator", sink), _mult_cast_id(mult_cast_id) {} - -OperatorPtr ExchangeSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink, _mult_cast_id); -} - -ExchangeSinkOperator::ExchangeSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink, - int mult_cast_id) - : DataSinkOperator(operator_builder, sink), _mult_cast_id(mult_cast_id) {} - -Status ExchangeSinkOperator::init(const TDataSink& tsink) { - // -1 means not the mult cast stream sender - if (_mult_cast_id == -1) { - _dest_node_id = tsink.stream_sink.dest_node_id; - } else { - _dest_node_id = tsink.multi_cast_stream_sink.sinks[_mult_cast_id].dest_node_id; - } - return Status::OK(); -} - -Status ExchangeSinkOperator::prepare(RuntimeState* state) { - _state = state; - PUniqueId id; - id.set_hi(_state->query_id().hi); - id.set_lo(_state->query_id().lo); - _sink_buffer = std::make_unique>( - id, _dest_node_id, _sink->_sender_id, _state->be_number(), state); - - RETURN_IF_ERROR(DataSinkOperator::prepare(state)); - _sink->register_pipeline_channels(_sink_buffer.get()); - return Status::OK(); -} - -bool ExchangeSinkOperator::can_write() { - return _sink_buffer->can_write() && _sink->channel_all_can_write(); -} - -bool ExchangeSinkOperator::is_pending_finish() const { - return _sink_buffer->is_pending_finish(); -} - -Status ExchangeSinkOperator::close(RuntimeState* state) { - RETURN_IF_ERROR(DataSinkOperator::close(state)); - if (_sink_buffer) { - _sink_buffer->update_profile(_sink->profile()); - _sink_buffer->close(); - } - return Status::OK(); -} - Status ExchangeSinkLocalState::serialize_block(vectorized::Block* src, PBlock* dest, int num_receivers) { return _parent->cast().serialize_block(*this, src, dest, num_receivers); diff --git a/be/src/pipeline/exec/exchange_sink_operator.h b/be/src/pipeline/exec/exchange_sink_operator.h index f275365c0e85a3..aaa89d246be656 100644 --- a/be/src/pipeline/exec/exchange_sink_operator.h +++ b/be/src/pipeline/exec/exchange_sink_operator.h @@ -35,36 +35,6 @@ class TDataSink; namespace pipeline { -class ExchangeSinkOperatorBuilder final - : public DataSinkOperatorBuilder { -public: - ExchangeSinkOperatorBuilder(int32_t id, DataSink* sink, int mult_cast_id = -1); - - OperatorPtr build_operator() override; - -private: - int _mult_cast_id = -1; -}; - -// Now local exchange is not supported since VDataStreamRecvr is considered as a pipeline broker. -class ExchangeSinkOperator final : public DataSinkOperator { -public: - ExchangeSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink, int mult_cast_id); - Status init(const TDataSink& tsink) override; - - Status prepare(RuntimeState* state) override; - bool can_write() override; - bool is_pending_finish() const override; - - Status close(RuntimeState* state) override; - -private: - std::unique_ptr> _sink_buffer = nullptr; - int _dest_node_id = -1; - RuntimeState* _state = nullptr; - int _mult_cast_id = -1; -}; - class ExchangeSinkLocalState final : public PipelineXSinkLocalState<> { ENABLE_FACTORY_CREATOR(ExchangeSinkLocalState); using Base = PipelineXSinkLocalState<>; diff --git a/be/src/pipeline/exec/exchange_source_operator.cpp b/be/src/pipeline/exec/exchange_source_operator.cpp index a23dae6dd62dd1..1a891655ee8daa 100644 --- a/be/src/pipeline/exec/exchange_source_operator.cpp +++ b/be/src/pipeline/exec/exchange_source_operator.cpp @@ -30,16 +30,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(ExchangeSourceOperator, SourceOperator) - -bool ExchangeSourceOperator::can_read() { - return _node->_stream_recvr->ready_to_read(); -} - -bool ExchangeSourceOperator::is_pending_finish() const { - return false; -} - ExchangeLocalState::ExchangeLocalState(RuntimeState* state, OperatorXBase* parent) : Base(state, parent), num_rows_skipped(0), is_ready(false) {} diff --git a/be/src/pipeline/exec/exchange_source_operator.h b/be/src/pipeline/exec/exchange_source_operator.h index 6176ad5b7f7190..de761d8391c94c 100644 --- a/be/src/pipeline/exec/exchange_source_operator.h +++ b/be/src/pipeline/exec/exchange_source_operator.h @@ -21,7 +21,6 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vexchange_node.h" namespace doris { class ExecNode; @@ -34,22 +33,6 @@ class Block; namespace doris::pipeline { -class ExchangeSourceOperatorBuilder final : public OperatorBuilder { -public: - ExchangeSourceOperatorBuilder(int32_t id, ExecNode* exec_node); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class ExchangeSourceOperator final : public SourceOperator { -public: - ExchangeSourceOperator(OperatorBuilderBase*, ExecNode*); - bool can_read() override; - bool is_pending_finish() const override; -}; - class ExchangeSourceOperatorX; class ExchangeLocalState final : public PipelineXLocalState<> { ENABLE_FACTORY_CREATOR(ExchangeLocalState); diff --git a/be/src/pipeline/exec/file_scan_operator.h b/be/src/pipeline/exec/file_scan_operator.h index e59dd8055b2949..f5c4f194bc5767 100644 --- a/be/src/pipeline/exec/file_scan_operator.h +++ b/be/src/pipeline/exec/file_scan_operator.h @@ -27,10 +27,8 @@ #include "pipeline/exec/scan_operator.h" #include "pipeline/pipeline_x/operator.h" #include "vec/exec/format/format_common.h" -#include "vec/exec/scan/vscan_node.h" namespace doris { -class ExecNode; namespace vectorized { class VFileScanner; } // namespace vectorized diff --git a/be/src/pipeline/exec/group_commit_block_sink_operator.cpp b/be/src/pipeline/exec/group_commit_block_sink_operator.cpp index 7c6abefc4a1902..4e9969d957049d 100644 --- a/be/src/pipeline/exec/group_commit_block_sink_operator.cpp +++ b/be/src/pipeline/exec/group_commit_block_sink_operator.cpp @@ -23,10 +23,6 @@ namespace doris::pipeline { -OperatorPtr GroupCommitBlockSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink); -} - GroupCommitBlockSinkLocalState::~GroupCommitBlockSinkLocalState() { if (_load_block_queue) { _remove_estimated_wal_bytes(); diff --git a/be/src/pipeline/exec/group_commit_block_sink_operator.h b/be/src/pipeline/exec/group_commit_block_sink_operator.h index 9d8aa372d6ea3d..ad03d607c15d2d 100644 --- a/be/src/pipeline/exec/group_commit_block_sink_operator.h +++ b/be/src/pipeline/exec/group_commit_block_sink_operator.h @@ -21,27 +21,7 @@ #include "pipeline/pipeline_x/operator.h" #include "vec/sink/group_commit_block_sink.h" -namespace doris { - -namespace pipeline { - -class GroupCommitBlockSinkOperatorBuilder final - : public DataSinkOperatorBuilder { -public: - GroupCommitBlockSinkOperatorBuilder(int32_t id, DataSink* sink) - : DataSinkOperatorBuilder(id, "GroupCommitBlockSinkOperator", sink) {} - - OperatorPtr build_operator() override; -}; - -class GroupCommitBlockSinkOperator final - : public DataSinkOperator { -public: - GroupCommitBlockSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink) - : DataSinkOperator(operator_builder, sink) {} - - bool can_write() override { return true; } // TODO: need use mem_limit -}; +namespace doris::pipeline { class GroupCommitBlockSinkOperatorX; class GroupCommitBlockSinkLocalState final : public PipelineXSinkLocalState { @@ -122,5 +102,4 @@ class GroupCommitBlockSinkOperatorX final TGroupCommitMode::type _group_commit_mode; }; -} // namespace pipeline -} // namespace doris \ No newline at end of file +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index da3614e4479fce..d583a827059111 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -28,8 +28,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(HashJoinBuildSink, StreamingOperator) - template struct Overload : Callables... { using Callables::operator()...; diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h index 0998884c99bb81..18c922eb19d5ea 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.h +++ b/be/src/pipeline/exec/hashjoin_build_sink.h @@ -22,12 +22,8 @@ #include "join_build_sink_operator.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/join/vhash_join_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { +namespace doris::pipeline { class HashJoinBuildSinkBuilder final : public OperatorBuilder { public: @@ -187,5 +183,4 @@ class HashJoinBuildSinkOperatorX final const bool _need_local_merge; }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp index fc6f81f41902a2..6e8e96e8d8a2ba 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp +++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp @@ -22,10 +22,7 @@ #include "common/logging.h" #include "pipeline/exec/operator.h" -namespace doris { -namespace pipeline { - -OPERATOR_CODE_GENERATOR(HashJoinProbeOperator, StatefulOperator) +namespace doris::pipeline { HashJoinProbeLocalState::HashJoinProbeLocalState(RuntimeState* state, OperatorXBase* parent) : JoinProbeLocalState(state, parent) {} @@ -634,5 +631,4 @@ Status HashJoinProbeOperatorX::open(RuntimeState* state) { return Status::OK(); } -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h index 1b45a2a258eb07..b5daefd735d984 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.h +++ b/be/src/pipeline/exec/hashjoin_probe_operator.h @@ -24,27 +24,10 @@ #include "pipeline/pipeline_x/operator.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class HashJoinProbeOperatorBuilder final : public OperatorBuilder { -public: - HashJoinProbeOperatorBuilder(int32_t, ExecNode*); - - OperatorPtr build_operator() override; -}; - -class HashJoinProbeOperator final : public StatefulOperator { -public: - HashJoinProbeOperator(OperatorBuilderBase*, ExecNode*); - // if exec node split to: sink, source operator. the source operator - // should skip `alloc_resource()` function call, only sink operator - // call the function - Status open(RuntimeState*) override { return Status::OK(); } -}; - class HashJoinProbeLocalState; using HashTableCtxVariants = std::variant< diff --git a/be/src/pipeline/exec/hive_table_sink_operator.cpp b/be/src/pipeline/exec/hive_table_sink_operator.cpp index 6b8eaa8c91e635..b931d48e832fca 100644 --- a/be/src/pipeline/exec/hive_table_sink_operator.cpp +++ b/be/src/pipeline/exec/hive_table_sink_operator.cpp @@ -21,10 +21,6 @@ namespace doris::pipeline { -OperatorPtr HiveTableSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink); -} - Status HiveTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/hive_table_sink_operator.h b/be/src/pipeline/exec/hive_table_sink_operator.h index 39b5df36567440..a489948268b9e4 100644 --- a/be/src/pipeline/exec/hive_table_sink_operator.h +++ b/be/src/pipeline/exec/hive_table_sink_operator.h @@ -21,26 +21,7 @@ #include "pipeline/pipeline_x/operator.h" #include "vec/sink/vhive_table_sink.h" -namespace doris { - -namespace pipeline { - -class HiveTableSinkOperatorBuilder final - : public DataSinkOperatorBuilder { -public: - HiveTableSinkOperatorBuilder(int32_t id, DataSink* sink) - : DataSinkOperatorBuilder(id, "HiveTableSinkOperator", sink) {} - - OperatorPtr build_operator() override; -}; - -class HiveTableSinkOperator final : public DataSinkOperator { -public: - HiveTableSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink) - : DataSinkOperator(operator_builder, sink) {} - - bool can_write() override { return _sink->can_write(); } -}; +namespace doris::pipeline { class HiveTableSinkOperatorX; @@ -111,5 +92,4 @@ class HiveTableSinkOperatorX final : public DataSinkOperatorX class JoinBuildSinkOperatorX; @@ -82,5 +79,4 @@ class JoinBuildSinkOperatorX : public DataSinkOperatorX { const std::vector _runtime_filter_descs; }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/join_probe_operator.h b/be/src/pipeline/exec/join_probe_operator.h index 679446147ef9ac..228a4140b13c34 100644 --- a/be/src/pipeline/exec/join_probe_operator.h +++ b/be/src/pipeline/exec/join_probe_operator.h @@ -19,11 +19,8 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/join/vjoin_node_base.h" -namespace doris { - -namespace pipeline { +namespace doris::pipeline { template class JoinProbeOperatorX; template @@ -127,5 +124,4 @@ class JoinProbeOperatorX : public StatefulOperatorX { const bool _use_specific_projections; }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/meta_scan_operator.h b/be/src/pipeline/exec/meta_scan_operator.h index e26af7dba5a6e1..440f489f2513e3 100644 --- a/be/src/pipeline/exec/meta_scan_operator.h +++ b/be/src/pipeline/exec/meta_scan_operator.h @@ -25,10 +25,8 @@ #include "operator.h" #include "pipeline/exec/scan_operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/scan/vscan_node.h" namespace doris { -class ExecNode; namespace vectorized { class NewOlapScanner; diff --git a/be/src/pipeline/exec/multi_cast_data_stream_sink.cpp b/be/src/pipeline/exec/multi_cast_data_stream_sink.cpp index de9cdeba04fae5..e39f60b356ccd9 100644 --- a/be/src/pipeline/exec/multi_cast_data_stream_sink.cpp +++ b/be/src/pipeline/exec/multi_cast_data_stream_sink.cpp @@ -19,10 +19,6 @@ namespace doris::pipeline { -OperatorPtr MultiCastDataStreamSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink); -} - std::string MultiCastDataStreamSinkLocalState::name_suffix() { auto& sinks = static_cast(_parent)->sink_node().sinks; std::string id_name = " (dst id : "; diff --git a/be/src/pipeline/exec/multi_cast_data_stream_sink.h b/be/src/pipeline/exec/multi_cast_data_stream_sink.h index b4886f089ef108..1a18772af4fd12 100644 --- a/be/src/pipeline/exec/multi_cast_data_stream_sink.h +++ b/be/src/pipeline/exec/multi_cast_data_stream_sink.h @@ -19,28 +19,9 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/sink/multi_cast_data_stream_sink.h" namespace doris::pipeline { -class MultiCastDataStreamSinkOperatorBuilder final - : public DataSinkOperatorBuilder { -public: - MultiCastDataStreamSinkOperatorBuilder(int32_t id, DataSink* sink) - : DataSinkOperatorBuilder(id, "MultiCastDataStreamSinkOperator", sink) {} - - OperatorPtr build_operator() override; -}; - -class MultiCastDataStreamSinkOperator final - : public DataSinkOperator { -public: - MultiCastDataStreamSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink) - : DataSinkOperator(operator_builder, sink) {} - - bool can_write() override { return _sink->can_write(); } -}; - class MultiCastDataStreamSinkOperatorX; class MultiCastDataStreamSinkLocalState final : public PipelineXSinkLocalState { diff --git a/be/src/pipeline/exec/multi_cast_data_stream_source.cpp b/be/src/pipeline/exec/multi_cast_data_stream_source.cpp index 90c809c535968d..b72125abbb3eb4 100644 --- a/be/src/pipeline/exec/multi_cast_data_stream_source.cpp +++ b/be/src/pipeline/exec/multi_cast_data_stream_source.cpp @@ -25,104 +25,6 @@ namespace doris::pipeline { -MultiCastDataStreamerSourceOperatorBuilder::MultiCastDataStreamerSourceOperatorBuilder( - int32_t id, const int consumer_id, std::shared_ptr& data_streamer, - const TDataStreamSink& sink) - : OperatorBuilderBase(id, "MultiCastDataStreamerSourceOperator"), - _consumer_id(consumer_id), - _multi_cast_data_streamer(data_streamer), - _t_data_stream_sink(sink) {} - -OperatorPtr MultiCastDataStreamerSourceOperatorBuilder::build_operator() { - return std::make_shared( - this, _consumer_id, _multi_cast_data_streamer, _t_data_stream_sink); -} - -const RowDescriptor& MultiCastDataStreamerSourceOperatorBuilder::row_desc() const { - return _multi_cast_data_streamer->row_desc(); -} - -MultiCastDataStreamerSourceOperator::MultiCastDataStreamerSourceOperator( - OperatorBuilderBase* operator_builder, const int consumer_id, - std::shared_ptr& data_streamer, const TDataStreamSink& sink) - : OperatorBase(operator_builder), - vectorized::RuntimeFilterConsumer(sink.dest_node_id, sink.runtime_filters, - data_streamer->row_desc(), _conjuncts), - _consumer_id(consumer_id), - _multi_cast_data_streamer(data_streamer), - _t_data_stream_sink(sink) {} - -Status MultiCastDataStreamerSourceOperator::prepare(doris::RuntimeState* state) { - RETURN_IF_ERROR(vectorized::RuntimeFilterConsumer::init(state)); - // init profile for runtime filter - RuntimeFilterConsumer::_init_profile(_multi_cast_data_streamer->profile()); - if (_t_data_stream_sink.__isset.output_exprs) { - RETURN_IF_ERROR(vectorized::VExpr::create_expr_trees(_t_data_stream_sink.output_exprs, - _output_expr_contexts)); - RETURN_IF_ERROR(vectorized::VExpr::prepare(_output_expr_contexts, state, row_desc())); - } - - if (_t_data_stream_sink.__isset.conjuncts) { - RETURN_IF_ERROR( - vectorized::VExpr::create_expr_trees(_t_data_stream_sink.conjuncts, _conjuncts)); - RETURN_IF_ERROR(vectorized::VExpr::prepare(_conjuncts, state, row_desc())); - } - return Status::OK(); -} - -Status MultiCastDataStreamerSourceOperator::open(doris::RuntimeState* state) { - if (_t_data_stream_sink.__isset.output_exprs) { - RETURN_IF_ERROR(vectorized::VExpr::open(_output_expr_contexts, state)); - } - if (_t_data_stream_sink.__isset.conjuncts) { - RETURN_IF_ERROR(vectorized::VExpr::open(_conjuncts, state)); - } - return _acquire_runtime_filter(false); -} - -bool MultiCastDataStreamerSourceOperator::runtime_filters_are_ready_or_timeout() { - return vectorized::RuntimeFilterConsumer::runtime_filters_are_ready_or_timeout(); -} - -bool MultiCastDataStreamerSourceOperator::can_read() { - return _multi_cast_data_streamer->can_read(_consumer_id); -} - -Status MultiCastDataStreamerSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, - SourceState& source_state) { - bool eos = false; - vectorized::Block tmp_block; - vectorized::Block* output_block = block; - if (!_output_expr_contexts.empty()) { - output_block = &tmp_block; - } - _multi_cast_data_streamer->pull(_consumer_id, output_block, &eos); - - if (!_conjuncts.empty()) { - RETURN_IF_ERROR(vectorized::VExprContext::filter_block(_conjuncts, output_block, - output_block->columns())); - } - - if (!_output_expr_contexts.empty() && output_block->rows() > 0) { - RETURN_IF_ERROR(vectorized::VExprContext::get_output_block_after_execute_exprs( - _output_expr_contexts, *output_block, block, true)); - vectorized::materialize_block_inplace(*block); - } - if (eos) { - source_state = SourceState::FINISHED; - } - return Status::OK(); -} - -Status MultiCastDataStreamerSourceOperator::close(doris::RuntimeState* state) { - _multi_cast_data_streamer->close_sender(_consumer_id); - return OperatorBase::close(state); -} - -RuntimeProfile* MultiCastDataStreamerSourceOperator::get_runtime_profile() const { - return _multi_cast_data_streamer->profile(); -} - MultiCastDataStreamSourceLocalState::MultiCastDataStreamSourceLocalState(RuntimeState* state, OperatorXBase* parent) : Base(state, parent), diff --git a/be/src/pipeline/exec/multi_cast_data_stream_source.h b/be/src/pipeline/exec/multi_cast_data_stream_source.h index 8d14b4f266bec9..c3404a873c6a9a 100644 --- a/be/src/pipeline/exec/multi_cast_data_stream_source.h +++ b/be/src/pipeline/exec/multi_cast_data_stream_source.h @@ -27,7 +27,6 @@ #include "vec/exec/runtime_filter_consumer.h" namespace doris { -class ExecNode; class RuntimeState; namespace vectorized { diff --git a/be/src/pipeline/exec/mysql_scan_operator.cpp b/be/src/pipeline/exec/mysql_scan_operator.cpp deleted file mode 100644 index 7ef6170d152683..00000000000000 --- a/be/src/pipeline/exec/mysql_scan_operator.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "mysql_scan_operator.h" - -#include "vec/exec/vmysql_scan_node.h" - -namespace doris::pipeline { - -OPERATOR_CODE_GENERATOR(MysqlScanOperator, SourceOperator) - -Status MysqlScanOperator::open(RuntimeState* state) { - RETURN_IF_ERROR(SourceOperator::open(state)); - return _node->open(state); -} - -Status MysqlScanOperator::close(RuntimeState* state) { - RETURN_IF_ERROR(SourceOperator::close(state)); - RETURN_IF_ERROR(_node->close(state)); - return Status::OK(); -} - -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/mysql_scan_operator.h b/be/src/pipeline/exec/mysql_scan_operator.h deleted file mode 100644 index 6e21d8d2ebef36..00000000000000 --- a/be/src/pipeline/exec/mysql_scan_operator.h +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "operator.h" -#include "vec/exec/vmysql_scan_node.h" - -namespace doris::pipeline { - -class MysqlScanOperatorBuilder : public OperatorBuilder { -public: - MysqlScanOperatorBuilder(int32_t id, ExecNode* exec_node); - bool is_source() const override { return true; } - OperatorPtr build_operator() override; -}; - -class MysqlScanOperator : public SourceOperator { -public: - MysqlScanOperator(OperatorBuilderBase* operator_builder, ExecNode* mysql_scan_node); - - bool can_read() override { return true; } - - Status open(RuntimeState* state) override; - - Status close(RuntimeState* state) override; -}; - -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp index 66612700fed37d..09a3a976567aae 100644 --- a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp +++ b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp @@ -20,11 +20,10 @@ #include #include "pipeline/exec/operator.h" +#include "vec/exec/join/vnested_loop_join_node.h" namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(NestLoopJoinBuildOperator, StreamingOperator) - NestedLoopJoinBuildSinkLocalState::NestedLoopJoinBuildSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : JoinBuildSinkLocalState( diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.h b/be/src/pipeline/exec/nested_loop_join_build_operator.h index da42e961f479a9..98377559f91b2e 100644 --- a/be/src/pipeline/exec/nested_loop_join_build_operator.h +++ b/be/src/pipeline/exec/nested_loop_join_build_operator.h @@ -22,27 +22,8 @@ #include "operator.h" #include "pipeline/exec/join_build_sink_operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/join/vnested_loop_join_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { - -class NestLoopJoinBuildOperatorBuilder final - : public OperatorBuilder { -public: - NestLoopJoinBuildOperatorBuilder(int32_t, ExecNode*); - - OperatorPtr build_operator() override; - bool is_sink() const override { return true; } -}; - -class NestLoopJoinBuildOperator final : public StreamingOperator { -public: - NestLoopJoinBuildOperator(OperatorBuilderBase* operator_builder, ExecNode* node); - bool can_write() override { return true; } -}; +namespace doris::pipeline { class NestedLoopJoinBuildSinkOperatorX; @@ -111,5 +92,4 @@ class NestedLoopJoinBuildSinkOperatorX final RowDescriptor _row_descriptor; }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp index c7afa2c399c249..819e8f67616bf6 100644 --- a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp +++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp @@ -22,7 +22,6 @@ #include "pipeline/exec/operator.h" #include "vec/columns/column_filter_helper.h" #include "vec/core/block.h" -#include "vec/exec/join/vnested_loop_join_node.h" namespace doris { class RuntimeState; @@ -30,18 +29,6 @@ class RuntimeState; namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(NestLoopJoinProbeOperator, StatefulOperator) - -Status NestLoopJoinProbeOperator::prepare(doris::RuntimeState* state) { - // just for speed up, the way is dangerous - _child_block = _node->get_left_block(); - return StatefulOperator::prepare(state); -} - -Status NestLoopJoinProbeOperator::close(doris::RuntimeState* state) { - return StatefulOperator::close(state); -} - NestedLoopJoinProbeLocalState::NestedLoopJoinProbeLocalState(RuntimeState* state, OperatorXBase* parent) : JoinProbeLocalState(state, diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.h b/be/src/pipeline/exec/nested_loop_join_probe_operator.h index de9f11b437eece..5c483a4348f87f 100644 --- a/be/src/pipeline/exec/nested_loop_join_probe_operator.h +++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.h @@ -24,33 +24,12 @@ #include "pipeline/exec/join_probe_operator.h" #include "pipeline/pipeline_x/operator.h" #include "util/simd/bits.h" -#include "vec/exec/join/vnested_loop_join_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class NestLoopJoinProbeOperatorBuilder final - : public OperatorBuilder { -public: - NestLoopJoinProbeOperatorBuilder(int32_t id, ExecNode* node); - - OperatorPtr build_operator() override; -}; - -class NestLoopJoinProbeOperator final : public StatefulOperator { -public: - NestLoopJoinProbeOperator(OperatorBuilderBase* operator_builder, ExecNode* node); - - Status prepare(RuntimeState* state) override; - - Status open(RuntimeState*) override { return Status::OK(); } - - Status close(RuntimeState* state) override; -}; - class NestedLoopJoinProbeOperatorX; class NestedLoopJoinProbeLocalState final : public JoinProbeLocalState { diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h index 8f546826c88fdd..f11cc6db0b982c 100644 --- a/be/src/pipeline/exec/olap_scan_operator.h +++ b/be/src/pipeline/exec/olap_scan_operator.h @@ -25,10 +25,8 @@ #include "operator.h" #include "pipeline/exec/scan_operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/scan/vscan_node.h" namespace doris { -class ExecNode; namespace vectorized { class NewOlapScanner; diff --git a/be/src/pipeline/exec/olap_table_sink_operator.cpp b/be/src/pipeline/exec/olap_table_sink_operator.cpp index faffaf99c112ed..60e6180469c764 100644 --- a/be/src/pipeline/exec/olap_table_sink_operator.cpp +++ b/be/src/pipeline/exec/olap_table_sink_operator.cpp @@ -19,16 +19,8 @@ #include "common/status.h" -namespace doris { -class DataSink; -} // namespace doris - namespace doris::pipeline { -OperatorPtr OlapTableSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink); -} - Status OlapTableSinkLocalState::close(RuntimeState* state, Status exec_status) { if (Base::_closed) { return Status::OK(); diff --git a/be/src/pipeline/exec/olap_table_sink_operator.h b/be/src/pipeline/exec/olap_table_sink_operator.h index 19c192160fd66e..ad8bbab3ee9b18 100644 --- a/be/src/pipeline/exec/olap_table_sink_operator.h +++ b/be/src/pipeline/exec/olap_table_sink_operator.h @@ -21,26 +21,7 @@ #include "pipeline/pipeline_x/operator.h" #include "vec/sink/volap_table_sink.h" -namespace doris { - -namespace pipeline { - -class OlapTableSinkOperatorBuilder final - : public DataSinkOperatorBuilder { -public: - OlapTableSinkOperatorBuilder(int32_t id, DataSink* sink) - : DataSinkOperatorBuilder(id, "OlapTableSinkOperator", sink) {} - - OperatorPtr build_operator() override; -}; - -class OlapTableSinkOperator final : public DataSinkOperator { -public: - OlapTableSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink) - : DataSinkOperator(operator_builder, sink) {} - - bool can_write() override { return _sink->can_write(); } -}; +namespace doris::pipeline { class OlapTableSinkOperatorX; @@ -102,5 +83,4 @@ class OlapTableSinkOperatorX final : public DataSinkOperatorX(this, _sink); -} - Status OlapTableSinkV2LocalState::close(RuntimeState* state, Status exec_status) { if (Base::_closed) { return Status::OK(); diff --git a/be/src/pipeline/exec/olap_table_sink_v2_operator.h b/be/src/pipeline/exec/olap_table_sink_v2_operator.h index 1fcd4716268119..da0700f8af3377 100644 --- a/be/src/pipeline/exec/olap_table_sink_v2_operator.h +++ b/be/src/pipeline/exec/olap_table_sink_v2_operator.h @@ -21,9 +21,7 @@ #include "pipeline/pipeline_x/operator.h" #include "vec/sink/volap_table_sink_v2.h" -namespace doris { - -namespace pipeline { +namespace doris::pipeline { class OlapTableSinkV2OperatorBuilder final : public DataSinkOperatorBuilder { @@ -104,5 +102,4 @@ class OlapTableSinkV2OperatorX final : public DataSinkOperatorX(this, _node); \ - } \ - \ - NAME::NAME(OperatorBuilderBase* operator_builder, ExecNode* node) \ - : SUBCLASS(operator_builder, node) {}; - namespace doris::pipeline { /** diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.cpp b/be/src/pipeline/exec/partition_sort_sink_operator.cpp index abe2fde555e164..77d9ca03e28a0b 100644 --- a/be/src/pipeline/exec/partition_sort_sink_operator.cpp +++ b/be/src/pipeline/exec/partition_sort_sink_operator.cpp @@ -23,10 +23,6 @@ namespace doris::pipeline { -OperatorPtr PartitionSortSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _node); -} - Status PartitionSortSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(PipelineXSinkLocalState::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.h b/be/src/pipeline/exec/partition_sort_sink_operator.h index 8602b096f516a2..4ac8a9d73f41a0 100644 --- a/be/src/pipeline/exec/partition_sort_sink_operator.h +++ b/be/src/pipeline/exec/partition_sort_sink_operator.h @@ -24,31 +24,8 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" #include "vec/common/sort/partition_sorter.h" -#include "vec/exec/vpartition_sort_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { - -class PartitionSortSinkOperatorBuilder final - : public OperatorBuilder { -public: - PartitionSortSinkOperatorBuilder(int32_t id, ExecNode* sort_node) - : OperatorBuilder(id, "PartitionSortSinkOperator", sort_node) {} - - bool is_sink() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class PartitionSortSinkOperator final : public StreamingOperator { -public: - PartitionSortSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node) - : StreamingOperator(operator_builder, sort_node) {}; - - bool can_write() override { return true; } -}; +namespace doris::pipeline { class PartitionSortSinkOperatorX; class PartitionSortSinkLocalState : public PipelineXSinkLocalState { @@ -128,5 +105,4 @@ class PartitionSortSinkOperatorX final : public DataSinkOperatorX(this, _node); -} - Status PartitionSortSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { RETURN_IF_ERROR(PipelineXLocalState::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/partition_sort_source_operator.h b/be/src/pipeline/exec/partition_sort_source_operator.h index 9d810db2039e70..5398a728e4ecf3 100644 --- a/be/src/pipeline/exec/partition_sort_source_operator.h +++ b/be/src/pipeline/exec/partition_sort_source_operator.h @@ -22,32 +22,12 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vpartition_sort_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class PartitionSortSourceOperatorBuilder final - : public OperatorBuilder { -public: - PartitionSortSourceOperatorBuilder(int32_t id, ExecNode* sort_node) - : OperatorBuilder(id, "PartitionSortSourceOperator", sort_node) {} - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class PartitionSortSourceOperator final : public SourceOperator { -public: - PartitionSortSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node) - : SourceOperator(operator_builder, sort_node) {} - Status open(RuntimeState*) override { return Status::OK(); } -}; - class PartitionSortSourceOperatorX; class PartitionSortSourceLocalState final : public PipelineXLocalState { diff --git a/be/src/pipeline/exec/partitioned_aggregation_source_operator.h b/be/src/pipeline/exec/partitioned_aggregation_source_operator.h index eff1e7179c8d0d..c1deb8af50db79 100644 --- a/be/src/pipeline/exec/partitioned_aggregation_source_operator.h +++ b/be/src/pipeline/exec/partitioned_aggregation_source_operator.h @@ -23,7 +23,6 @@ #include "pipeline/pipeline_x/operator.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h index 68c6b970163f24..5c6b7e1f74f930 100644 --- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h +++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.h @@ -29,7 +29,6 @@ #include "vec/sink/vdata_stream_sender.h" // ShuffleChannelIds namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/repeat_operator.cpp b/be/src/pipeline/exec/repeat_operator.cpp index 42d009f0e76b7d..fcf59f36543ba3 100644 --- a/be/src/pipeline/exec/repeat_operator.cpp +++ b/be/src/pipeline/exec/repeat_operator.cpp @@ -22,7 +22,6 @@ #include "common/logging.h" #include "pipeline/exec/operator.h" #include "vec/core/block.h" -#include "vec/exec/vrepeat_node.h" namespace doris { class RuntimeState; @@ -30,18 +29,6 @@ class RuntimeState; namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(RepeatOperator, StatefulOperator) - -Status RepeatOperator::prepare(doris::RuntimeState* state) { - // just for speed up, the way is dangerous - _child_block = _node->get_child_block(); - return StatefulOperator::prepare(state); -} - -Status RepeatOperator::close(doris::RuntimeState* state) { - return StatefulOperator::close(state); -} - RepeatLocalState::RepeatLocalState(RuntimeState* state, OperatorXBase* parent) : Base(state, parent), _child_block(vectorized::Block::create_unique()), diff --git a/be/src/pipeline/exec/repeat_operator.h b/be/src/pipeline/exec/repeat_operator.h index 208b3d1e00565c..922645d270d915 100644 --- a/be/src/pipeline/exec/repeat_operator.h +++ b/be/src/pipeline/exec/repeat_operator.h @@ -21,30 +21,12 @@ #include "common/status.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vrepeat_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class RepeatOperatorBuilder final : public OperatorBuilder { -public: - RepeatOperatorBuilder(int32_t id, ExecNode* repeat_node); - - OperatorPtr build_operator() override; -}; - -class RepeatOperator final : public StatefulOperator { -public: - RepeatOperator(OperatorBuilderBase* operator_builder, ExecNode* repeat_node); - - Status prepare(RuntimeState* state) override; - - Status close(RuntimeState* state) override; -}; - class RepeatOperatorX; class RepeatLocalState final : public PipelineXLocalState { diff --git a/be/src/pipeline/exec/result_file_sink_operator.cpp b/be/src/pipeline/exec/result_file_sink_operator.cpp index 65ad6599d5ddba..f8cadad1df7e1d 100644 --- a/be/src/pipeline/exec/result_file_sink_operator.cpp +++ b/be/src/pipeline/exec/result_file_sink_operator.cpp @@ -25,25 +25,9 @@ #include "runtime/buffer_control_block.h" #include "runtime/result_buffer_mgr.h" #include "vec/sink/vdata_stream_sender.h" -#include "vec/sink/vresult_file_sink.h" - -namespace doris { -class DataSink; -} // namespace doris namespace doris::pipeline { -ResultFileSinkOperatorBuilder::ResultFileSinkOperatorBuilder(int32_t id, DataSink* sink) - : DataSinkOperatorBuilder(id, "ResultSinkOperator", sink) {}; - -OperatorPtr ResultFileSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink); -} - -ResultFileSinkOperator::ResultFileSinkOperator(OperatorBuilderBase* operator_builder, - DataSink* sink) - : DataSinkOperator(operator_builder, sink) {}; - ResultFileSinkLocalState::ResultFileSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : AsyncWriterSink(parent, state), diff --git a/be/src/pipeline/exec/result_file_sink_operator.h b/be/src/pipeline/exec/result_file_sink_operator.h index 31b4b26206caaf..9dc91193510ce7 100644 --- a/be/src/pipeline/exec/result_file_sink_operator.h +++ b/be/src/pipeline/exec/result_file_sink_operator.h @@ -23,25 +23,7 @@ #include "pipeline/pipeline_x/operator.h" #include "vec/sink/vresult_file_sink.h" -namespace doris { -class DataSink; - -namespace pipeline { - -class ResultFileSinkOperatorBuilder final - : public DataSinkOperatorBuilder { -public: - ResultFileSinkOperatorBuilder(int32_t id, DataSink* sink); - - OperatorPtr build_operator() override; -}; - -class ResultFileSinkOperator final : public DataSinkOperator { -public: - ResultFileSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink); - - bool can_write() override { return true; } -}; +namespace doris::pipeline { class ResultFileSinkOperatorX; class ResultFileSinkLocalState final @@ -127,5 +109,4 @@ class ResultFileSinkOperatorX final : public DataSinkOperatorX(this, _sink); -} - -ResultSinkOperator::ResultSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink) - : DataSinkOperator(operator_builder, sink) {}; - -bool ResultSinkOperator::can_write() { - return _sink->_sender->can_sink(); -} - Status ResultSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/result_sink_operator.h b/be/src/pipeline/exec/result_sink_operator.h index aed9961a6d6771..71b8afce1719ec 100644 --- a/be/src/pipeline/exec/result_sink_operator.h +++ b/be/src/pipeline/exec/result_sink_operator.h @@ -21,28 +21,12 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/sink/vresult_sink.h" namespace doris { -class DataSink; class PipBufferControlBlock; namespace pipeline { -class ResultSinkOperatorBuilder final : public DataSinkOperatorBuilder { -public: - ResultSinkOperatorBuilder(int32_t id, DataSink* sink); - - OperatorPtr build_operator() override; -}; - -class ResultSinkOperator final : public DataSinkOperator { -public: - ResultSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink); - - bool can_write() override; -}; - class ResultSinkLocalState final : public PipelineXSinkLocalState { ENABLE_FACTORY_CREATOR(ResultSinkLocalState); using Base = PipelineXSinkLocalState; diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 9d32f0e25abf60..37da4d97288367 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -45,39 +45,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(ScanOperator, SourceOperator) - -bool ScanOperator::can_read() { - if (!_node->_opened) { - return _node->_should_create_scanner || _node->ready_to_open(); - } else { - // If scanner meet any error, done == true - if (_node->_eos || _node->_scanner_ctx->done()) { - // _eos: need eos - // _scanner_ctx->done(): need finish - // _scanner_ctx->no_schedule(): should schedule _scanner_ctx - return true; - } else { - return _node->ready_to_read(); // there are some blocks to process - } - } -} - -bool ScanOperator::runtime_filters_are_ready_or_timeout() { - return _node->runtime_filters_are_ready_or_timeout(); -} - -std::string ScanOperator::debug_string() const { - fmt::memory_buffer debug_string_buffer; - fmt::format_to(debug_string_buffer, "{}, scanner_ctx is null: {} ", - SourceOperator::debug_string(), _node->_scanner_ctx == nullptr); - if (_node->_scanner_ctx) { - fmt::format_to(debug_string_buffer, ", scanner ctx detail = {}", - _node->_scanner_ctx->debug_string()); - } - return fmt::to_string(debug_string_buffer); -} - #define RETURN_IF_PUSH_DOWN(stmt, status) \ if (pdt == vectorized::VScanNode::PushDownType::UNACCEPTABLE) { \ status = stmt; \ diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h index 3ebccb58a8c604..6b62af1eefd9ac 100644 --- a/be/src/pipeline/exec/scan_operator.h +++ b/be/src/pipeline/exec/scan_operator.h @@ -29,9 +29,6 @@ #include "runtime/descriptors.h" #include "vec/exec/scan/vscan_node.h" -namespace doris { -class ExecNode; -} // namespace doris namespace doris::vectorized { class ScannerDelegate; } @@ -39,24 +36,6 @@ class ScannerDelegate; namespace doris::pipeline { class PipScannerContext; -class ScanOperatorBuilder : public OperatorBuilder { -public: - ScanOperatorBuilder(int32_t id, ExecNode* exec_node); - bool is_source() const override { return true; } - OperatorPtr build_operator() override; -}; - -class ScanOperator : public SourceOperator { -public: - ScanOperator(OperatorBuilderBase* operator_builder, ExecNode* scan_node); - - bool can_read() override; // for source - - bool runtime_filters_are_ready_or_timeout() override; - - std::string debug_string() const override; -}; - class ScanLocalStateBase : public PipelineXLocalState<>, public vectorized::RuntimeFilterConsumer { public: ScanLocalStateBase(RuntimeState* state, OperatorXBase* parent) diff --git a/be/src/pipeline/exec/schema_scan_operator.cpp b/be/src/pipeline/exec/schema_scan_operator.cpp index 2d32e21d991960..e4a3a8c1ca2468 100644 --- a/be/src/pipeline/exec/schema_scan_operator.cpp +++ b/be/src/pipeline/exec/schema_scan_operator.cpp @@ -24,7 +24,6 @@ #include "pipeline/exec/operator.h" #include "util/runtime_profile.h" #include "vec/data_types/data_type_factory.hpp" -#include "vec/exec/vschema_scan_node.h" namespace doris { class RuntimeState; @@ -32,18 +31,6 @@ class RuntimeState; namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(SchemaScanOperator, SourceOperator) - -Status SchemaScanOperator::open(RuntimeState* state) { - return _node->open(state); -} - -Status SchemaScanOperator::close(RuntimeState* state) { - RETURN_IF_ERROR(SourceOperator::close(state)); - RETURN_IF_ERROR(_node->close(state)); - return Status::OK(); -} - Status SchemaScanLocalState::init(RuntimeState* state, LocalStateInfo& info) { RETURN_IF_ERROR(PipelineXLocalState<>::init(state, info)); diff --git a/be/src/pipeline/exec/schema_scan_operator.h b/be/src/pipeline/exec/schema_scan_operator.h index bd336132efb2b7..f07a0e854e70c0 100644 --- a/be/src/pipeline/exec/schema_scan_operator.h +++ b/be/src/pipeline/exec/schema_scan_operator.h @@ -25,30 +25,11 @@ #include "vec/exec/vschema_scan_node.h" namespace doris { -class ExecNode; class RuntimeState; } // namespace doris namespace doris::pipeline { -class SchemaScanOperatorBuilder : public OperatorBuilder { -public: - SchemaScanOperatorBuilder(int32_t id, ExecNode* exec_node); - bool is_source() const override { return true; } - OperatorPtr build_operator() override; -}; - -class SchemaScanOperator : public SourceOperator { -public: - SchemaScanOperator(OperatorBuilderBase* operator_builder, ExecNode* scan_node); - - bool can_read() override { return true; } - - Status open(RuntimeState* state) override; - - Status close(RuntimeState* state) override; -}; - class SchemaScanOperatorX; class SchemaScanLocalState final : public PipelineXLocalState<> { public: diff --git a/be/src/pipeline/exec/select_operator.cpp b/be/src/pipeline/exec/select_operator.cpp deleted file mode 100644 index b4eb27eace3ea1..00000000000000 --- a/be/src/pipeline/exec/select_operator.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "select_operator.h" - -#include - -#include "pipeline/exec/operator.h" - -namespace doris::pipeline { - -OPERATOR_CODE_GENERATOR(SelectOperator, StreamingOperator) - -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/select_operator.h b/be/src/pipeline/exec/select_operator.h index 4fd929e323ba7d..4bdc5a9e42d298 100644 --- a/be/src/pipeline/exec/select_operator.h +++ b/be/src/pipeline/exec/select_operator.h @@ -21,24 +21,8 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vselect_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { - -class SelectOperatorBuilder final : public OperatorBuilder { -public: - SelectOperatorBuilder(int32_t id, ExecNode* select_node); - - OperatorPtr build_operator() override; -}; - -class SelectOperator final : public StreamingOperator { -public: - SelectOperator(OperatorBuilderBase* operator_builder, ExecNode* select_node); -}; +namespace doris::pipeline { class SelectOperatorX; class SelectLocalState final : public PipelineXLocalState { @@ -72,5 +56,4 @@ class SelectOperatorX final : public StreamingOperatorX { [[nodiscard]] bool is_source() const override { return false; } }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp index 744c8b17e2257d..81c0cd463c1284 100644 --- a/be/src/pipeline/exec/set_probe_sink_operator.cpp +++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp @@ -23,10 +23,8 @@ #include "pipeline/exec/operator.h" #include "vec/common/hash_table/hash_table_set_probe.h" -#include "vec/exec/vset_operation_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace vectorized { @@ -36,41 +34,6 @@ class Block; namespace doris::pipeline { -template -SetProbeSinkOperatorBuilder::SetProbeSinkOperatorBuilder(int32_t id, int child_id, - ExecNode* set_node) - : OperatorBuilder>(id, builder_name, set_node), - _child_id(child_id) {} - -template -OperatorPtr SetProbeSinkOperatorBuilder::build_operator() { - return std::make_shared>(this, _child_id, this->_node); -} - -template -SetProbeSinkOperator::SetProbeSinkOperator(OperatorBuilderBase* operator_builder, - int child_id, ExecNode* set_node) - : StreamingOperator>(operator_builder, - set_node), - _child_id(child_id) {} - -template -Status SetProbeSinkOperator::sink(RuntimeState* state, vectorized::Block* block, - SourceState source_state) { - return this->_node->sink_probe(state, _child_id, block, source_state == SourceState::FINISHED); -} - -template -bool SetProbeSinkOperator::can_write() { - DCHECK_GT(_child_id, 0); - return this->_node->is_child_finished(_child_id - 1); -} - -template class SetProbeSinkOperatorBuilder; -template class SetProbeSinkOperatorBuilder; -template class SetProbeSinkOperator; -template class SetProbeSinkOperator; - template Status SetProbeSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* state) { DataSinkOperatorX>::_name = "SET_PROBE_SINK_OPERATOR"; diff --git a/be/src/pipeline/exec/set_probe_sink_operator.h b/be/src/pipeline/exec/set_probe_sink_operator.h index 9f80f03966b1f1..499eeee0d6205b 100644 --- a/be/src/pipeline/exec/set_probe_sink_operator.h +++ b/be/src/pipeline/exec/set_probe_sink_operator.h @@ -22,10 +22,8 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vset_operation_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace vectorized { @@ -36,37 +34,6 @@ struct HashTableProbe; namespace pipeline { -template -class SetProbeSinkOperatorBuilder final - : public OperatorBuilder> { -private: - constexpr static auto builder_name = - is_intersect ? "IntersectProbeSinkOperator" : "ExceptProbeSinkOperator"; - -public: - SetProbeSinkOperatorBuilder(int32_t id, int child_id, ExecNode* set_node); - [[nodiscard]] bool is_sink() const override { return true; } - - OperatorPtr build_operator() override; - -private: - int _child_id; -}; - -template -class SetProbeSinkOperator : public StreamingOperator> { -public: - SetProbeSinkOperator(OperatorBuilderBase* operator_builder, int child_id, ExecNode* set_node); - - bool can_write() override; - - Status sink(RuntimeState* state, vectorized::Block* block, SourceState source_state) override; - Status open(RuntimeState* /*state*/) override { return Status::OK(); } - -private: - int _child_id; -}; - template class SetProbeSinkOperatorX; diff --git a/be/src/pipeline/exec/set_sink_operator.cpp b/be/src/pipeline/exec/set_sink_operator.cpp index 2042e3eb1a1d51..796174d080a88e 100644 --- a/be/src/pipeline/exec/set_sink_operator.cpp +++ b/be/src/pipeline/exec/set_sink_operator.cpp @@ -22,34 +22,9 @@ #include "pipeline/exec/operator.h" #include "vec/common/hash_table/hash_table_set_build.h" #include "vec/core/materialize_block.h" -#include "vec/exec/vset_operation_node.h" - -namespace doris { -class ExecNode; -} // namespace doris namespace doris::pipeline { -template -SetSinkOperatorBuilder::SetSinkOperatorBuilder(int32_t id, ExecNode* set_node) - : OperatorBuilder>(id, builder_name, set_node) { -} - -template -OperatorPtr SetSinkOperatorBuilder::build_operator() { - return std::make_shared>(this, this->_node); -} - -template -SetSinkOperator::SetSinkOperator( - OperatorBuilderBase* builder, vectorized::VSetOperationNode* set_node) - : StreamingOperator>(builder, set_node) {} - -template class SetSinkOperatorBuilder; -template class SetSinkOperatorBuilder; -template class SetSinkOperator; -template class SetSinkOperator; - template Status SetSinkOperatorX::sink(RuntimeState* state, vectorized::Block* in_block, bool eos) { diff --git a/be/src/pipeline/exec/set_sink_operator.h b/be/src/pipeline/exec/set_sink_operator.h index 2a6bb63c02e815..8894b8b15f2252 100644 --- a/be/src/pipeline/exec/set_sink_operator.h +++ b/be/src/pipeline/exec/set_sink_operator.h @@ -22,10 +22,8 @@ #include "olap/olap_common.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vset_operation_node.h" namespace doris { -class ExecNode; namespace vectorized { template @@ -34,32 +32,6 @@ struct HashTableBuild; namespace pipeline { -template -class SetSinkOperatorBuilder final - : public OperatorBuilder> { -private: - constexpr static auto builder_name = - is_intersect ? "IntersectSinkOperator" : "ExceptSinkOperator"; - -public: - SetSinkOperatorBuilder(int32_t id, ExecNode* set_node); - [[nodiscard]] bool is_sink() const override { return true; } - - OperatorPtr build_operator() override; -}; - -template -class SetSinkOperator : public StreamingOperator> { -public: - SetSinkOperator(OperatorBuilderBase* operator_builder, - vectorized::VSetOperationNode* set_node); - - bool can_write() override { return true; } - -private: - vectorized::VSetOperationNode* _set_node = nullptr; -}; - template class SetSinkOperatorX; diff --git a/be/src/pipeline/exec/set_source_operator.cpp b/be/src/pipeline/exec/set_source_operator.cpp index 88d38d325af003..bdb844e70e8095 100644 --- a/be/src/pipeline/exec/set_source_operator.cpp +++ b/be/src/pipeline/exec/set_source_operator.cpp @@ -21,34 +21,9 @@ #include "common/status.h" #include "pipeline/exec/operator.h" -#include "vec/exec/vset_operation_node.h" - -namespace doris { -class ExecNode; -} // namespace doris namespace doris::pipeline { -template -SetSourceOperatorBuilder::SetSourceOperatorBuilder(int32_t id, ExecNode* set_node) - : OperatorBuilder>(id, builder_name, set_node) { -} - -template -OperatorPtr SetSourceOperatorBuilder::build_operator() { - return std::make_shared>(this, this->_node); -} - -template -SetSourceOperator::SetSourceOperator( - OperatorBuilderBase* builder, vectorized::VSetOperationNode* set_node) - : SourceOperator>(builder, set_node) {} - -template class SetSourceOperatorBuilder; -template class SetSourceOperatorBuilder; -template class SetSourceOperator; -template class SetSourceOperator; - template Status SetSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); diff --git a/be/src/pipeline/exec/set_source_operator.h b/be/src/pipeline/exec/set_source_operator.h index 1c5cf162940b40..94487507c26e3e 100644 --- a/be/src/pipeline/exec/set_source_operator.h +++ b/be/src/pipeline/exec/set_source_operator.h @@ -22,37 +22,12 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vset_operation_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -template -class SetSourceOperatorBuilder - : public OperatorBuilder> { -private: - constexpr static auto builder_name = - is_intersect ? "IntersectSourceOperator" : "ExceptSourceOperator"; - -public: - SetSourceOperatorBuilder(int32_t id, ExecNode* set_node); - [[nodiscard]] bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -template -class SetSourceOperator : public SourceOperator> { -public: - SetSourceOperator(OperatorBuilderBase* builder, - vectorized::VSetOperationNode* set_node); - - Status open(RuntimeState* /*state*/) override { return Status::OK(); } -}; - template class SetSourceOperatorX; diff --git a/be/src/pipeline/exec/sort_sink_operator.cpp b/be/src/pipeline/exec/sort_sink_operator.cpp index 91ae687510c985..7c9f40d1b5956f 100644 --- a/be/src/pipeline/exec/sort_sink_operator.cpp +++ b/be/src/pipeline/exec/sort_sink_operator.cpp @@ -26,8 +26,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(SortSinkOperator, StreamingOperator) - Status SortSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/sort_sink_operator.h b/be/src/pipeline/exec/sort_sink_operator.h index ad9c23401b4c69..8298dc980b6c7d 100644 --- a/be/src/pipeline/exec/sort_sink_operator.h +++ b/be/src/pipeline/exec/sort_sink_operator.h @@ -22,28 +22,8 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" #include "vec/core/field.h" -#include "vec/exec/vsort_node.h" -namespace doris { -class ExecNode; - -namespace pipeline { - -class SortSinkOperatorBuilder final : public OperatorBuilder { -public: - SortSinkOperatorBuilder(int32_t id, ExecNode* sort_node); - - bool is_sink() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class SortSinkOperator final : public StreamingOperator { -public: - SortSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node); - - bool can_write() override { return true; } -}; +namespace doris::pipeline { enum class SortAlgorithm { HEAP_SORT, TOPN_SORT, FULL_SORT }; @@ -132,5 +112,4 @@ class SortSinkOperatorX final : public DataSinkOperatorX { const std::vector _partition_exprs; }; -} // namespace pipeline -} // namespace doris +} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/sort_source_operator.cpp b/be/src/pipeline/exec/sort_source_operator.cpp index f7d6b6ae56c3c9..34bfffb8d9fd2e 100644 --- a/be/src/pipeline/exec/sort_source_operator.cpp +++ b/be/src/pipeline/exec/sort_source_operator.cpp @@ -23,8 +23,6 @@ namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(SortSourceOperator, SourceOperator) - SortLocalState::SortLocalState(RuntimeState* state, OperatorXBase* parent) : PipelineXLocalState(state, parent) {} diff --git a/be/src/pipeline/exec/sort_source_operator.h b/be/src/pipeline/exec/sort_source_operator.h index 1af30bfd05f49f..43c4934b977415 100644 --- a/be/src/pipeline/exec/sort_source_operator.h +++ b/be/src/pipeline/exec/sort_source_operator.h @@ -22,29 +22,12 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vsort_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -class SortSourceOperatorBuilder final : public OperatorBuilder { -public: - SortSourceOperatorBuilder(int32_t id, ExecNode* sort_node); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; -}; - -class SortSourceOperator final : public SourceOperator { -public: - SortSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node); - Status open(RuntimeState*) override { return Status::OK(); } -}; - class SortSourceOperatorX; class SortLocalState final : public PipelineXLocalState { public: diff --git a/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp b/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp deleted file mode 100644 index 5805b209f4f7f8..00000000000000 --- a/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "streaming_aggregation_sink_operator.h" - -#include - -#include - -#include "common/compiler_util.h" // IWYU pragma: keep -#include "pipeline/exec/data_queue.h" -#include "pipeline/exec/operator.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; -} // namespace doris - -namespace doris::pipeline { - -StreamingAggSinkOperator::StreamingAggSinkOperator(OperatorBuilderBase* operator_builder, - ExecNode* agg_node, - std::shared_ptr queue) - : StreamingOperator(operator_builder, agg_node), _data_queue(std::move(queue)) {} - -Status StreamingAggSinkOperator::prepare(RuntimeState* state) { - RETURN_IF_ERROR(StreamingOperator::prepare(state)); - _queue_byte_size_counter = - ADD_COUNTER(_node->runtime_profile(), "MaxSizeInBlockQueue", TUnit::BYTES); - _queue_size_counter = ADD_COUNTER(_node->runtime_profile(), "MaxSizeOfBlockQueue", TUnit::UNIT); - return Status::OK(); -} - -bool StreamingAggSinkOperator::can_write() { - // sink and source in diff threads - return _data_queue->has_enough_space_to_push(); -} - -Status StreamingAggSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, - SourceState source_state) { - Status ret = Status::OK(); - if (in_block && in_block->rows() > 0) { - auto block_from_ctx = _data_queue->get_free_block(); - RETURN_IF_ERROR(_node->do_pre_agg(in_block, block_from_ctx.get())); - if (block_from_ctx->rows() == 0) { - _data_queue->push_free_block(std::move(block_from_ctx)); - } else { - _data_queue->push_block(std::move(block_from_ctx)); - } - } - - if (UNLIKELY(source_state == SourceState::FINISHED)) { - _data_queue->set_finish(); - } - return Status::OK(); -} - -Status StreamingAggSinkOperator::close(RuntimeState* state) { - if (_data_queue && !_data_queue->is_finish()) { - // finish should be set, if not set here means error. - _data_queue->set_canceled(); - } - if (_data_queue) { - COUNTER_SET(_queue_size_counter, _data_queue->max_size_of_queue()); - COUNTER_SET(_queue_byte_size_counter, _data_queue->max_bytes_in_queue()); - } - return StreamingOperator::close(state); -} - -StreamingAggSinkOperatorBuilder::StreamingAggSinkOperatorBuilder(int32_t id, ExecNode* exec_node, - std::shared_ptr queue) - : OperatorBuilder(id, "StreamingAggSinkOperator", exec_node), - _data_queue(std::move(queue)) {} - -OperatorPtr StreamingAggSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _node, _data_queue); -} -} // namespace doris::pipeline diff --git a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h deleted file mode 100644 index 99e94e2d585957..00000000000000 --- a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include - -#include "aggregation_sink_operator.h" -#include "aggregation_source_operator.h" -#include "common/status.h" -#include "operator.h" -#include "pipeline/pipeline_x/operator.h" -#include "util/runtime_profile.h" -#include "vec/core/block.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; - -namespace pipeline { -class DataQueue; - -class StreamingAggSinkOperatorBuilder final : public OperatorBuilder { -public: - StreamingAggSinkOperatorBuilder(int32_t, ExecNode*, std::shared_ptr); - - OperatorPtr build_operator() override; - - bool is_sink() const override { return true; } - bool is_source() const override { return false; } - -private: - std::shared_ptr _data_queue; -}; - -class StreamingAggSinkOperator final : public StreamingOperator { -public: - StreamingAggSinkOperator(OperatorBuilderBase* operator_builder, ExecNode*, - std::shared_ptr); - - Status prepare(RuntimeState*) override; - - Status sink(RuntimeState* state, vectorized::Block* block, SourceState source_state) override; - - bool can_write() override; - - Status close(RuntimeState* state) override; - -private: - vectorized::Block _preagg_block = vectorized::Block(); - - RuntimeProfile::Counter* _queue_byte_size_counter = nullptr; - RuntimeProfile::Counter* _queue_size_counter = nullptr; - - std::shared_ptr _data_queue; -}; - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp b/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp deleted file mode 100644 index 9c9a014046a043..00000000000000 --- a/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "streaming_aggregation_source_operator.h" - -#include - -#include "pipeline/exec/data_queue.h" -#include "pipeline/exec/operator.h" -#include "runtime/descriptors.h" -#include "util/runtime_profile.h" -#include "vec/core/block.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; - -namespace pipeline { - -StreamingAggSourceOperator::StreamingAggSourceOperator(OperatorBuilderBase* templ, ExecNode* node, - std::shared_ptr queue) - : SourceOperator(templ, node), _data_queue(std::move(queue)) {} - -bool StreamingAggSourceOperator::can_read() { - return _data_queue->has_data_or_finished(); -} - -Status StreamingAggSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, - SourceState& source_state) { - bool eos = false; - if (!_data_queue->data_exhausted()) { - std::unique_ptr agg_block; - RETURN_IF_ERROR(_data_queue->get_block_from_queue(&agg_block)); - - if (_data_queue->data_exhausted()) { - RETURN_IF_ERROR(_node->pull(state, block, &eos)); - } else { - block->swap(*agg_block); - agg_block->clear_column_data(_node->row_desc().num_materialized_slots()); - _data_queue->push_free_block(std::move(agg_block)); - } - } else { - RETURN_IF_ERROR(_node->pull(state, block, &eos)); - } - - source_state = eos ? SourceState::FINISHED : SourceState::DEPEND_ON_SOURCE; - - return Status::OK(); -} - -StreamingAggSourceOperatorBuilder::StreamingAggSourceOperatorBuilder( - int32_t id, ExecNode* exec_node, std::shared_ptr queue) - : OperatorBuilder(id, "StreamingAggSourceOperator", exec_node), - _data_queue(std::move(queue)) {} - -OperatorPtr StreamingAggSourceOperatorBuilder::build_operator() { - return std::make_shared(this, _node, _data_queue); -} - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/streaming_aggregation_source_operator.h b/be/src/pipeline/exec/streaming_aggregation_source_operator.h deleted file mode 100644 index 89dbaab058ac82..00000000000000 --- a/be/src/pipeline/exec/streaming_aggregation_source_operator.h +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#pragma once - -#include - -#include - -#include "common/status.h" -#include "operator.h" -#include "pipeline/exec/aggregation_source_operator.h" -#include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vaggregation_node.h" - -namespace doris { -class ExecNode; -class RuntimeState; - -namespace vectorized { -class Block; -} // namespace vectorized -namespace pipeline { -class DataQueue; - -class StreamingAggSourceOperatorBuilder final - : public OperatorBuilder { -public: - StreamingAggSourceOperatorBuilder(int32_t, ExecNode*, std::shared_ptr); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; - -private: - std::shared_ptr _data_queue; -}; - -class StreamingAggSourceOperator final : public SourceOperator { -public: - StreamingAggSourceOperator(OperatorBuilderBase*, ExecNode*, std::shared_ptr); - bool can_read() override; - Status get_block(RuntimeState*, vectorized::Block*, SourceState& source_state) override; - Status open(RuntimeState*) override { return Status::OK(); } - -private: - std::shared_ptr _data_queue; -}; - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/table_function_operator.cpp b/be/src/pipeline/exec/table_function_operator.cpp index 9256d1deb2b072..cb547688595de6 100644 --- a/be/src/pipeline/exec/table_function_operator.cpp +++ b/be/src/pipeline/exec/table_function_operator.cpp @@ -29,18 +29,6 @@ class RuntimeState; namespace doris::pipeline { -OPERATOR_CODE_GENERATOR(TableFunctionOperator, StatefulOperator) - -Status TableFunctionOperator::prepare(doris::RuntimeState* state) { - // just for speed up, the way is dangerous - _child_block = _node->get_child_block(); - return StatefulOperator::prepare(state); -} - -Status TableFunctionOperator::close(doris::RuntimeState* state) { - return StatefulOperator::close(state); -} - TableFunctionLocalState::TableFunctionLocalState(RuntimeState* state, OperatorXBase* parent) : PipelineXLocalState<>(state, parent), _child_block(vectorized::Block::create_unique()) {} diff --git a/be/src/pipeline/exec/table_function_operator.h b/be/src/pipeline/exec/table_function_operator.h index 8a7b7bd43d45d1..5b9457a474b79b 100644 --- a/be/src/pipeline/exec/table_function_operator.h +++ b/be/src/pipeline/exec/table_function_operator.h @@ -25,28 +25,11 @@ #include "vec/exec/vtable_function_node.h" namespace doris { -class ExecNode; class RuntimeState; } // namespace doris namespace doris::pipeline { -class TableFunctionOperatorBuilder final : public OperatorBuilder { -public: - TableFunctionOperatorBuilder(int32_t id, ExecNode* node); - - OperatorPtr build_operator() override; -}; - -class TableFunctionOperator final : public StatefulOperator { -public: - TableFunctionOperator(OperatorBuilderBase* operator_builder, ExecNode* node); - - Status prepare(RuntimeState* state) override; - - Status close(RuntimeState* state) override; -}; - class TableFunctionOperatorX; class TableFunctionLocalState final : public PipelineXLocalState<> { public: diff --git a/be/src/pipeline/exec/table_sink_operator.h b/be/src/pipeline/exec/table_sink_operator.h deleted file mode 100644 index 46843c23f870b3..00000000000000 --- a/be/src/pipeline/exec/table_sink_operator.h +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "exec/data_sink.h" -#include "operator.h" - -namespace doris { - -namespace pipeline { - -// used for VMysqlTableSink, VJdbcTableSink and VOdbcTableSink. -class TableSinkOperatorBuilder final : public DataSinkOperatorBuilder { -public: - TableSinkOperatorBuilder(int32_t id, DataSink* sink) - : DataSinkOperatorBuilder(id, "TableSinkOperator", sink) {} - - OperatorPtr build_operator() override; -}; - -class TableSinkOperator final : public DataSinkOperator { -public: - TableSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink) - : DataSinkOperator(operator_builder, sink) {} - - bool can_write() override { return _sink->can_write(); } -}; - -OperatorPtr TableSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _sink); -} - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/exec/union_sink_operator.cpp b/be/src/pipeline/exec/union_sink_operator.cpp index 40344882a84e53..aa10468fc754d7 100644 --- a/be/src/pipeline/exec/union_sink_operator.cpp +++ b/be/src/pipeline/exec/union_sink_operator.cpp @@ -26,73 +26,8 @@ #include "runtime/runtime_state.h" #include "util/runtime_profile.h" -namespace doris { -class ExecNode; -} // namespace doris - namespace doris::pipeline { -UnionSinkOperatorBuilder::UnionSinkOperatorBuilder(int32_t id, int child_id, ExecNode* node, - std::shared_ptr queue) - : OperatorBuilder(id, "UnionSinkOperator", node), - _cur_child_id(child_id), - _data_queue(queue) {}; - -UnionSinkOperator::UnionSinkOperator(OperatorBuilderBase* operator_builder, int child_id, - ExecNode* node, std::shared_ptr queue) - : StreamingOperator(operator_builder, node), _cur_child_id(child_id), _data_queue(queue) {}; - -OperatorPtr UnionSinkOperatorBuilder::build_operator() { - return std::make_shared(this, _cur_child_id, _node, _data_queue); -} - -Status UnionSinkOperator::sink(RuntimeState* state, vectorized::Block* in_block, - SourceState source_state) { - if (_output_block == nullptr) { - _output_block = _data_queue->get_free_block(_cur_child_id); - } - - if (_cur_child_id < _node->get_first_materialized_child_idx()) { //pass_through - if (in_block->rows() > 0) { - _output_block->swap(*in_block); - _data_queue->push_block(std::move(_output_block), _cur_child_id); - } - } else if (_node->get_first_materialized_child_idx() != _node->children_count() && - _cur_child_id < _node->children_count()) { //need materialized - RETURN_IF_ERROR(this->_node->materialize_child_block(state, _cur_child_id, in_block, - _output_block.get())); - } else { - return Status::InternalError("maybe can't reach here, execute const expr: {}, {}, {}", - _cur_child_id, _node->get_first_materialized_child_idx(), - _node->children_count()); - } - - if (UNLIKELY(source_state == SourceState::FINISHED)) { - //if _cur_child_id eos, need check to push block - //Now here can't check _output_block rows, even it's row==0, also need push block - //because maybe sink is eos and queue have none data, if not push block - //the source can't can_read again and can't set source finished - if (_output_block) { - _data_queue->push_block(std::move(_output_block), _cur_child_id); - } - _data_queue->set_finish(_cur_child_id); - return Status::OK(); - } - // not eos and block rows is enough to output,so push block - if (_output_block && (_output_block->rows() >= state->batch_size())) { - _data_queue->push_block(std::move(_output_block), _cur_child_id); - } - return Status::OK(); -} - -Status UnionSinkOperator::close(RuntimeState* state) { - if (_data_queue && !_data_queue->is_finish(_cur_child_id)) { - // finish should be set, if not set here means error. - _data_queue->set_canceled(_cur_child_id); - } - return StreamingOperator::close(state); -} - Status UnionSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/union_sink_operator.h b/be/src/pipeline/exec/union_sink_operator.h index 97b704078c63ec..c11465da134b60 100644 --- a/be/src/pipeline/exec/union_sink_operator.h +++ b/be/src/pipeline/exec/union_sink_operator.h @@ -25,47 +25,13 @@ #include "operator.h" #include "pipeline/pipeline_x/operator.h" #include "vec/core/block.h" -#include "vec/exec/vunion_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { class DataQueue; -class UnionSinkOperatorBuilder final : public OperatorBuilder { -public: - UnionSinkOperatorBuilder(int32_t id, int child_id, ExecNode* node, - std::shared_ptr queue); - - OperatorPtr build_operator() override; - - bool is_sink() const override { return true; } - -private: - int _cur_child_id; - std::shared_ptr _data_queue; -}; - -class UnionSinkOperator final : public StreamingOperator { -public: - UnionSinkOperator(OperatorBuilderBase* operator_builder, int child_id, ExecNode* node, - std::shared_ptr queue); - - bool can_write() override { return true; } - - Status sink(RuntimeState* state, vectorized::Block* in_block, - SourceState source_state) override; - - Status close(RuntimeState* state) override; - -private: - int _cur_child_id; - std::shared_ptr _data_queue; - std::unique_ptr _output_block; -}; - class UnionSinkOperatorX; class UnionSinkLocalState final : public PipelineXSinkLocalState { public: diff --git a/be/src/pipeline/exec/union_source_operator.cpp b/be/src/pipeline/exec/union_source_operator.cpp index 10f98a8d1cbc49..6393ffcccd312b 100644 --- a/be/src/pipeline/exec/union_source_operator.cpp +++ b/be/src/pipeline/exec/union_source_operator.cpp @@ -30,83 +30,10 @@ #include "vec/core/block.h" namespace doris { -class ExecNode; class RuntimeState; namespace pipeline { -UnionSourceOperatorBuilder::UnionSourceOperatorBuilder(int32_t id, ExecNode* node, - std::shared_ptr queue) - : OperatorBuilder(id, "UnionSourceOperator", node), _data_queue(queue) {}; - -OperatorPtr UnionSourceOperatorBuilder::build_operator() { - return std::make_shared(this, _node, _data_queue); -} - -UnionSourceOperator::UnionSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* node, - std::shared_ptr queue) - : SourceOperator(operator_builder, node), - _data_queue(queue), - _need_read_for_const_expr(true) {}; - -bool UnionSourceOperator::_has_data() { - return _need_read_for_const_expr || _data_queue->remaining_has_data(); -} - -// we assumed it can read to process const expr， Although we don't know whether there is -// ,and queue have data, could read also -// The source operator's run dependences on Node's alloc_resource, which is called in Sink's open. -// So hang until SinkOperator was scheduled to open. -bool UnionSourceOperator::can_read() { - return _node->resource_allocated() && (_has_data() || _data_queue->is_all_finish()); -} - -Status UnionSourceOperator::pull_data(RuntimeState* state, vectorized::Block* block, bool* eos) { - // here we precess const expr firstly - if (_need_read_for_const_expr) { - if (_node->has_more_const(state)) { - RETURN_IF_ERROR(_node->get_next_const(state, block)); - } - _need_read_for_const_expr = _node->has_more_const(state); - } else { - std::unique_ptr output_block; - int child_idx = 0; - RETURN_IF_ERROR(_data_queue->get_block_from_queue(&output_block, &child_idx)); - if (!output_block) { - return Status::OK(); - } - block->swap(*output_block); - output_block->clear_column_data(_node->intermediate_row_desc().num_materialized_slots()); - _data_queue->push_free_block(std::move(output_block), child_idx); - } - - _node->reached_limit(block, eos); - return Status::OK(); -} - -Status UnionSourceOperator::get_block(RuntimeState* state, vectorized::Block* block, - SourceState& source_state) { - bool eos = false; - RETURN_IF_ERROR(_node->get_next_after_projects( - state, block, &eos, - std::bind(&UnionSourceOperator::pull_data, this, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3))); - //have executing const expr, queue have no data anymore, and child could be closed. - if (eos) { // reach limit - source_state = SourceState::FINISHED; - } else if (_has_data()) { - source_state = SourceState::MORE_DATA; - } else if (_data_queue->is_all_finish()) { - // Here, check the value of `_has_data(state)` again after `data_queue.is_all_finish()` is TRUE - // as there may be one or more blocks when `data_queue.is_all_finish()` is TRUE. - source_state = _has_data() ? SourceState::MORE_DATA : SourceState::FINISHED; - } else { - source_state = SourceState::DEPEND_ON_SOURCE; - } - - return Status::OK(); -} - Status UnionSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); SCOPED_TIMER(exec_time_counter()); diff --git a/be/src/pipeline/exec/union_source_operator.h b/be/src/pipeline/exec/union_source_operator.h index 60530521ec0a82..ee1d35d73cff04 100644 --- a/be/src/pipeline/exec/union_source_operator.h +++ b/be/src/pipeline/exec/union_source_operator.h @@ -23,10 +23,8 @@ #include "common/status.h" #include "operator.h" #include "pipeline/pipeline_x/operator.h" -#include "vec/exec/vunion_node.h" namespace doris { -class ExecNode; class RuntimeState; namespace vectorized { @@ -36,39 +34,6 @@ class Block; namespace pipeline { class DataQueue; -class UnionSourceOperatorBuilder final : public OperatorBuilder { -public: - UnionSourceOperatorBuilder(int32_t id, ExecNode* node, std::shared_ptr); - - bool is_source() const override { return true; } - - OperatorPtr build_operator() override; - -private: - std::shared_ptr _data_queue; -}; - -class UnionSourceOperator final : public SourceOperator { -public: - UnionSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* node, - std::shared_ptr); - - // this operator in source open directly return, do this work in sink - Status open(RuntimeState* /*state*/) override { return Status::OK(); } - - Status get_block(RuntimeState* state, vectorized::Block* block, - SourceState& source_state) override; - bool can_read() override; - - Status pull_data(RuntimeState* state, vectorized::Block* output_block, bool* eos); - -private: - bool _has_data(); - - std::shared_ptr _data_queue; - bool _need_read_for_const_expr; -}; - class UnionSourceOperatorX; class UnionSourceLocalState final : public PipelineXLocalState { public: diff --git a/be/src/pipeline/pipeline.h b/be/src/pipeline/pipeline.h index ab5b7e36bc2ad5..1bab4f1fd5003b 100644 --- a/be/src/pipeline/pipeline.h +++ b/be/src/pipeline/pipeline.h @@ -42,7 +42,7 @@ using PipelineId = uint32_t; class Pipeline : public std::enable_shared_from_this { friend class PipelineTask; friend class PipelineXTask; - friend class PipelineXFragmentContext; + friend class PipelineFragmentContext; public: Pipeline() = delete; diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index 3b094062074e34..03de71f3c8d829 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -48,46 +48,52 @@ #include "pipeline/exec/analytic_sink_operator.h" #include "pipeline/exec/analytic_source_operator.h" #include "pipeline/exec/assert_num_rows_operator.h" -#include "pipeline/exec/const_value_operator.h" -#include "pipeline/exec/data_queue.h" #include "pipeline/exec/datagen_operator.h" -#include "pipeline/exec/distinct_streaming_aggregation_sink_operator.h" -#include "pipeline/exec/distinct_streaming_aggregation_source_operator.h" +#include "pipeline/exec/distinct_streaming_aggregation_operator.h" #include "pipeline/exec/empty_set_operator.h" -#include "pipeline/exec/empty_source_operator.h" +#include "pipeline/exec/es_scan_operator.h" #include "pipeline/exec/exchange_sink_operator.h" #include "pipeline/exec/exchange_source_operator.h" +#include "pipeline/exec/file_scan_operator.h" #include "pipeline/exec/group_commit_block_sink_operator.h" #include "pipeline/exec/hashjoin_build_sink.h" #include "pipeline/exec/hashjoin_probe_operator.h" #include "pipeline/exec/hive_table_sink_operator.h" +#include "pipeline/exec/jdbc_scan_operator.h" +#include "pipeline/exec/jdbc_table_sink_operator.h" +#include "pipeline/exec/meta_scan_operator.h" #include "pipeline/exec/multi_cast_data_stream_sink.h" #include "pipeline/exec/multi_cast_data_stream_source.h" -#include "pipeline/exec/mysql_scan_operator.h" // IWYU pragma: keep #include "pipeline/exec/nested_loop_join_build_operator.h" #include "pipeline/exec/nested_loop_join_probe_operator.h" +#include "pipeline/exec/olap_scan_operator.h" #include "pipeline/exec/olap_table_sink_operator.h" #include "pipeline/exec/olap_table_sink_v2_operator.h" -#include "pipeline/exec/operator.h" #include "pipeline/exec/partition_sort_sink_operator.h" #include "pipeline/exec/partition_sort_source_operator.h" +#include "pipeline/exec/partitioned_aggregation_sink_operator.h" +#include "pipeline/exec/partitioned_aggregation_source_operator.h" +#include "pipeline/exec/partitioned_hash_join_probe_operator.h" +#include "pipeline/exec/partitioned_hash_join_sink_operator.h" #include "pipeline/exec/repeat_operator.h" #include "pipeline/exec/result_file_sink_operator.h" #include "pipeline/exec/result_sink_operator.h" -#include "pipeline/exec/scan_operator.h" #include "pipeline/exec/schema_scan_operator.h" #include "pipeline/exec/select_operator.h" -#include "pipeline/exec/set_probe_sink_operator.h" // IWYU pragma: keep -#include "pipeline/exec/set_sink_operator.h" // IWYU pragma: keep -#include "pipeline/exec/set_source_operator.h" // IWYU pragma: keep +#include "pipeline/exec/set_probe_sink_operator.h" +#include "pipeline/exec/set_sink_operator.h" +#include "pipeline/exec/set_source_operator.h" #include "pipeline/exec/sort_sink_operator.h" #include "pipeline/exec/sort_source_operator.h" -#include "pipeline/exec/streaming_aggregation_sink_operator.h" -#include "pipeline/exec/streaming_aggregation_source_operator.h" +#include "pipeline/exec/spill_sort_sink_operator.h" +#include "pipeline/exec/spill_sort_source_operator.h" +#include "pipeline/exec/streaming_aggregation_operator.h" #include "pipeline/exec/table_function_operator.h" -#include "pipeline/exec/table_sink_operator.h" #include "pipeline/exec/union_sink_operator.h" #include "pipeline/exec/union_source_operator.h" +#include "pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h" +#include "pipeline/pipeline_x/local_exchange/local_exchange_source_operator.h" +#include "pipeline/task_scheduler.h" #include "pipeline_task.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" @@ -119,14 +125,11 @@ namespace doris::pipeline { bvar::Adder g_pipeline_tasks_count("doris_pipeline_tasks_count"); PipelineFragmentContext::PipelineFragmentContext( - const TUniqueId& query_id, const TUniqueId& instance_id, int fragment_id, int backend_num, - std::shared_ptr query_ctx, ExecEnv* exec_env, - const std::function& call_back, - report_status_callback report_status_cb) + const TUniqueId& query_id, const int fragment_id, std::shared_ptr query_ctx, + ExecEnv* exec_env, const std::function& call_back, + const report_status_callback& report_status_cb) : _query_id(query_id), - _fragment_instance_id(instance_id), _fragment_id(fragment_id), - _backend_num(backend_num), _exec_env(exec_env), _query_ctx(std::move(query_ctx)), _call_back(call_back), @@ -144,14 +147,19 @@ PipelineFragmentContext::~PipelineFragmentContext() { auto st = _query_ctx->exec_status(); _query_ctx.reset(); _tasks.clear(); - if (_runtime_state != nullptr) { - _call_back(_runtime_state.get(), &st); - _runtime_state.reset(); + if (!_task_runtime_states.empty()) { + for (auto& runtime_state : _task_runtime_states) { + _call_back(runtime_state.get(), &st); + runtime_state.reset(); + } } - _root_pipeline.reset(); _pipelines.clear(); _sink.reset(); - _multi_cast_stream_sink_senders.clear(); + _root_op.reset(); + _runtime_state.reset(); + _runtime_filter_states.clear(); + _runtime_filter_mgr_map.clear(); + _op_id_to_le_state.clear(); } bool PipelineFragmentContext::is_timeout(const VecDateTimeValue& now) const { @@ -171,26 +179,21 @@ bool PipelineFragmentContext::is_timeout(const VecDateTimeValue& now) const { void PipelineFragmentContext::cancel(const PPlanFragmentCancelReason& reason, const std::string& msg) { LOG_INFO("PipelineFragmentContext::cancel") - .tag("query_id", print_id(_query_ctx->query_id())) + .tag("query_id", print_id(_query_id)) .tag("fragment_id", _fragment_id) - .tag("instance_id", print_id(_runtime_state->fragment_instance_id())) - .tag("reason", PPlanFragmentCancelReason_Name(reason)) - .tag("message", msg); - // TODO(zhiqiang): may be not need to check if query is already cancelled. - // Dont cancel in this situation may lead to bug. For example, result sink node - // can not be cancelled if other fragments set the query_ctx cancelled, this will - // make result receiver on fe be stocked on rpc forever until timeout... - // We need a more detail discussion. - _query_ctx->cancel(msg, Status::Cancelled(msg)); + .tag("reason", reason) + .tag("error message", msg); + if (reason == PPlanFragmentCancelReason::TIMEOUT) { + LOG(WARNING) << "PipelineFragmentContext is cancelled due to timeout : " << debug_string(); + } + _query_ctx->cancel(msg, Status::Cancelled(msg), _fragment_id); if (reason == PPlanFragmentCancelReason::LIMIT_REACH) { _is_report_on_cancel = false; } else { - LOG(WARNING) << "PipelineFragmentContext " - << PrintInstanceStandardInfo(_query_id, _fragment_instance_id) - << " is canceled, cancel message: " << msg; + for (auto& id : _fragment_instance_ids) { + LOG(WARNING) << "PipelineFragmentContext cancel instance: " << print_id(id); + } } - - _runtime_state->set_process_status(_query_ctx->exec_status()); // Get pipe from new load stream manager and send cancel to it or the fragment may hang to wait read from pipe // For stream load the fragment's query_id == load id, it is set in FE. auto stream_load_ctx = _exec_env->new_load_stream_mgr()->get(_query_id); @@ -198,12 +201,14 @@ void PipelineFragmentContext::cancel(const PPlanFragmentCancelReason& reason, stream_load_ctx->pipe->cancel(msg); } - // must close stream_mgr to avoid dead lock in Exchange Node - // TODO bug llj fix this other instance will not cancel - _exec_env->vstream_mgr()->cancel(_fragment_instance_id, Status::Cancelled(msg)); // Cancel the result queue manager used by spark doris connector // TODO pipeline incomp // _exec_env->result_queue_mgr()->update_queue_status(id, Status::Aborted(msg)); + for (auto& tasks : _tasks) { + for (auto& task : tasks) { + task->clear_blocking_state(); + } + } } PipelinePtr PipelineFragmentContext::add_pipeline() { @@ -231,36 +236,33 @@ PipelinePtr PipelineFragmentContext::add_pipeline(PipelinePtr parent, int idx) { return pipeline; } -Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& request, size_t idx) { +Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& request) { if (_prepared) { return Status::InternalError("Already prepared"); } - if (request.__isset.query_options && request.query_options.__isset.execution_timeout) { - _timeout = request.query_options.execution_timeout; - } - const auto& local_params = request.local_params[idx]; + _num_instances = request.local_params.size(); + _total_instances = request.__isset.total_instances ? request.total_instances : _num_instances; _runtime_profile = std::make_unique("PipelineContext"); - _start_timer = ADD_TIMER(_runtime_profile, "StartTime"); - COUNTER_UPDATE(_start_timer, _fragment_watcher.elapsed_time()); _prepare_timer = ADD_TIMER(_runtime_profile, "PrepareTime"); SCOPED_TIMER(_prepare_timer); auto* fragment_context = this; - LOG_INFO("Preparing instance {}|{}, backend_num {}", print_id(_query_id), - print_id(local_params.fragment_instance_id), local_params.backend_num); + LOG_INFO("PipelineFragmentContext::prepare") + .tag("query_id", print_id(_query_id)) + .tag("fragment_id", _fragment_id) + .tag("pthread_id", (uintptr_t)pthread_self()); - // 1. init _runtime_state - _runtime_state = RuntimeState::create_unique( - local_params.fragment_instance_id, request.query_id, request.fragment_id, - request.query_options, _query_ctx->query_globals, _exec_env, _query_ctx.get()); + if (request.query_options.__isset.is_report_success) { + fragment_context->set_is_report_success(request.query_options.is_report_success); + } - _runtime_state->set_task_execution_context(shared_from_this()); + // 1. Set up the global runtime state. + _runtime_state = RuntimeState::create_unique(request.query_id, request.fragment_id, + request.query_options, _query_ctx->query_globals, + _exec_env, _query_ctx.get()); - // TODO should be combine with plan_fragment_executor.prepare funciton SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_runtime_state->query_mem_tracker()); - _runtime_state->set_be_number(local_params.backend_num); - if (request.__isset.backend_id) { _runtime_state->set_backend_id(request.backend_id); } @@ -274,10 +276,6 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re _runtime_state->set_load_job_id(request.load_job_id); } - if (request.query_options.__isset.is_report_success) { - fragment_context->set_is_report_success(request.query_options.is_report_success); - } - if (request.is_simplified_param) { _desc_tbl = _query_ctx->desc_tbl; } else { @@ -286,91 +284,58 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re DescriptorTbl::create(_runtime_state->obj_pool(), request.desc_tbl, &_desc_tbl)); } _runtime_state->set_desc_tbl(_desc_tbl); - - // 2. Create ExecNode to build pipeline with PipelineFragmentContext - RETURN_IF_ERROR_OR_CATCH_EXCEPTION( - ExecNode::create_tree(_runtime_state.get(), _runtime_state->obj_pool(), - request.fragment.plan, *_desc_tbl, &_root_plan)); - - // Set senders of exchange nodes before pipeline build - std::vector exch_nodes; - _root_plan->collect_nodes(TPlanNodeType::EXCHANGE_NODE, &exch_nodes); - for (ExecNode* exch_node : exch_nodes) { - DCHECK_EQ(exch_node->type(), TPlanNodeType::EXCHANGE_NODE); - int num_senders = find_with_default(request.per_exch_num_senders, exch_node->id(), 0); - DCHECK_GT(num_senders, 0); - static_cast(exch_node)->set_num_senders(num_senders); - } - - // All prepare work do in exec node tree - RETURN_IF_ERROR(_root_plan->prepare(_runtime_state.get())); - // set scan ranges - std::vector scan_nodes; - std::vector no_scan_ranges; - _root_plan->collect_scan_nodes(&scan_nodes); - VLOG_CRITICAL << "query " << print_id(get_query_id()) - << " scan_nodes.size()=" << scan_nodes.size(); - VLOG_CRITICAL << "query " << print_id(get_query_id()) << " params.per_node_scan_ranges.size()=" - << local_params.per_node_scan_ranges.size(); - - // set scan range in ScanNode - for (auto& i : scan_nodes) { - // TODO(cmy): this "if...else" should be removed once all ScanNode are derived from VScanNode. - ExecNode* node = i; - if (typeid(*node) == typeid(vectorized::NewOlapScanNode) || - typeid(*node) == typeid(vectorized::NewFileScanNode) || - typeid(*node) == typeid(vectorized::NewOdbcScanNode) || - typeid(*node) == typeid(vectorized::NewEsScanNode) || - typeid(*node) == typeid(vectorized::VMetaScanNode) || - typeid(*node) == typeid(vectorized::NewJdbcScanNode)) { - auto* scan_node = static_cast(i); - auto scan_ranges = find_with_default(local_params.per_node_scan_ranges, scan_node->id(), - no_scan_ranges); - const bool shared_scan = - find_with_default(local_params.per_node_shared_scans, scan_node->id(), false); - scan_node->set_scan_ranges(_runtime_state.get(), scan_ranges); - scan_node->set_shared_scan(_runtime_state.get(), shared_scan); - } else { - auto* scan_node = static_cast(node); - auto scan_ranges = find_with_default(local_params.per_node_scan_ranges, scan_node->id(), - no_scan_ranges); - RETURN_IF_ERROR(scan_node->set_scan_ranges(_runtime_state.get(), scan_ranges)); - VLOG_CRITICAL << "query " << print_id(get_query_id()) - << " scan_node_id=" << scan_node->id() - << " size=" << scan_ranges.get().size(); - } - } - - _runtime_state->set_per_fragment_instance_idx(local_params.sender_id); _runtime_state->set_num_per_fragment_instances(request.num_senders); _runtime_state->set_load_stream_per_node(request.load_stream_per_node); _runtime_state->set_total_load_streams(request.total_load_streams); _runtime_state->set_num_local_sink(request.num_local_sink); - if (request.fragment.__isset.output_sink) { - // Here we build a DataSink object, which will be hold by DataSinkOperator - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(DataSink::create_data_sink( - _runtime_state->obj_pool(), request.fragment.output_sink, - request.fragment.output_exprs, request, idx, _root_plan->row_desc(), - _runtime_state.get(), &_sink, *_desc_tbl)); + const auto& local_params = request.local_params[0]; + if (local_params.__isset.runtime_filter_params) { + _query_ctx->runtime_filter_mgr()->set_runtime_filter_params( + local_params.runtime_filter_params); + } + if (local_params.__isset.topn_filter_source_node_ids) { + _query_ctx->init_runtime_predicates(local_params.topn_filter_source_node_ids); + } else { + _query_ctx->init_runtime_predicates({0}); } - _root_pipeline = fragment_context->add_pipeline(); - _root_pipeline->set_is_root_pipeline(); - RETURN_IF_ERROR(_build_pipelines(_root_plan, _root_pipeline)); - if (_sink) { - // DataSinkOperator is builded here - RETURN_IF_ERROR(_create_sink(request.local_params[idx].sender_id, - request.fragment.output_sink, _runtime_state.get())); + _need_local_merge = request.__isset.parallel_instances; + + // 2. Build pipelines with operators in this fragment. + auto root_pipeline = add_pipeline(); + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_build_pipelines( + _runtime_state->obj_pool(), request, *_query_ctx->desc_tbl, &_root_op, root_pipeline)); + + // 3. Create sink operator + if (!request.fragment.__isset.output_sink) { + return Status::InternalError("No output sink in this fragment!"); } - RETURN_IF_ERROR(_build_pipeline_tasks(request)); - if (_sink) { - _runtime_state->runtime_profile()->add_child(_sink->profile(), true, nullptr); - _sink->profile()->add_child(_root_plan->runtime_profile(), true, nullptr); - } else { - _runtime_state->runtime_profile()->add_child(_root_plan->runtime_profile(), true, nullptr); + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_create_data_sink( + _runtime_state->obj_pool(), request.fragment.output_sink, request.fragment.output_exprs, + request, root_pipeline->output_row_desc(), _runtime_state.get(), *_desc_tbl, + root_pipeline->id())); + RETURN_IF_ERROR(_sink->init(request.fragment.output_sink)); + RETURN_IF_ERROR(root_pipeline->set_sink(_sink)); + + for (PipelinePtr& pipeline : _pipelines) { + DCHECK(pipeline->sink_x() != nullptr) << pipeline->operator_xs().size(); + RETURN_IF_ERROR(pipeline->sink_x()->set_child(pipeline->operator_xs().back())); + } + if (_enable_local_shuffle()) { + RETURN_IF_ERROR(_plan_local_exchange(request.num_buckets, + request.bucket_seq_to_instance_idx, + request.shuffle_idx_to_instance_idx)); } - _runtime_state->runtime_profile()->add_child(_runtime_profile.get(), true, nullptr); + + // 4. Initialize global states in pipelines. + for (PipelinePtr& pipeline : _pipelines) { + pipeline->children().clear(); + RETURN_IF_ERROR(pipeline->prepare(_runtime_state.get())); + } + + // 5. Build pipeline tasks and initialize local state. + RETURN_IF_ERROR(_build_pipeline_tasks(request)); _init_next_report_time(); @@ -381,29 +346,189 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re Status PipelineFragmentContext::_build_pipeline_tasks( const doris::TPipelineFragmentParams& request) { _total_tasks = 0; - for (PipelinePtr& pipeline : _pipelines) { - // if sink - auto sink_operator = pipeline->get_sink_builder()->build_operator(); - // TODO pipeline 1 need to add new interface for exec node and operator - RETURN_IF_ERROR(sink_operator->init(request.fragment.output_sink)); - - RETURN_IF_ERROR(pipeline->build_operators()); - auto task = - std::make_unique(pipeline, _total_tasks++, _runtime_state.get(), - sink_operator, this, pipeline->pipeline_profile()); - RETURN_IF_ERROR(sink_operator->set_child(task->get_root())); - _tasks.emplace_back(std::move(task)); - _runtime_profile->add_child(pipeline->pipeline_profile(), true, nullptr); - } - g_pipeline_tasks_count << _total_tasks; - for (auto& task : _tasks) { - RETURN_IF_ERROR(task->prepare(_runtime_state.get())); + int target_size = request.local_params.size(); + _tasks.resize(target_size); + auto& pipeline_id_to_profile = _runtime_state->pipeline_id_to_profile(); + DCHECK(pipeline_id_to_profile.empty()); + pipeline_id_to_profile.resize(_pipelines.size()); + { + size_t pip_idx = 0; + for (auto& pipeline_profile : pipeline_id_to_profile) { + pipeline_profile = + std::make_unique("Pipeline : " + std::to_string(pip_idx)); + pip_idx++; + } } - // register the profile of child data stream sender - for (auto& sender : _multi_cast_stream_sink_senders) { - _sink->profile()->add_child(sender->profile(), true, nullptr); + for (size_t i = 0; i < target_size; i++) { + const auto& local_params = request.local_params[i]; + auto fragment_instance_id = local_params.fragment_instance_id; + _fragment_instance_ids.push_back(fragment_instance_id); + std::unique_ptr runtime_filter_mgr; + auto init_runtime_state = [&](std::unique_ptr& runtime_state) { + runtime_state->set_query_mem_tracker(_query_ctx->query_mem_tracker); + + runtime_state->set_task_execution_context(shared_from_this()); + runtime_state->set_be_number(local_params.backend_num); + + if (request.__isset.backend_id) { + runtime_state->set_backend_id(request.backend_id); + } + if (request.__isset.import_label) { + runtime_state->set_import_label(request.import_label); + } + if (request.__isset.db_name) { + runtime_state->set_db_name(request.db_name); + } + if (request.__isset.load_job_id) { + runtime_state->set_load_job_id(request.load_job_id); + } + + runtime_state->set_desc_tbl(_desc_tbl); + runtime_state->set_per_fragment_instance_idx(local_params.sender_id); + runtime_state->set_num_per_fragment_instances(request.num_senders); + runtime_state->resize_op_id_to_local_state(max_operator_id()); + runtime_state->set_max_operator_id(max_operator_id()); + runtime_state->set_load_stream_per_node(request.load_stream_per_node); + runtime_state->set_total_load_streams(request.total_load_streams); + runtime_state->set_num_local_sink(request.num_local_sink); + DCHECK(runtime_filter_mgr); + runtime_state->set_pipeline_x_runtime_filter_mgr(runtime_filter_mgr.get()); + }; + + auto filterparams = std::make_unique(); + + { + filterparams->runtime_filter_wait_infinitely = + _runtime_state->runtime_filter_wait_infinitely(); + filterparams->runtime_filter_wait_time_ms = + _runtime_state->runtime_filter_wait_time_ms(); + filterparams->enable_pipeline_exec = _runtime_state->enable_pipeline_x_exec(); + filterparams->execution_timeout = _runtime_state->execution_timeout(); + + filterparams->exec_env = ExecEnv::GetInstance(); + filterparams->query_id.set_hi(_runtime_state->query_id().hi); + filterparams->query_id.set_lo(_runtime_state->query_id().lo); + + filterparams->be_exec_version = _runtime_state->be_exec_version(); + filterparams->query_ctx = _query_ctx.get(); + } + + // build local_runtime_filter_mgr for each instance + runtime_filter_mgr = std::make_unique( + request.query_id, filterparams.get(), _query_ctx->query_mem_tracker); + + filterparams->runtime_filter_mgr = runtime_filter_mgr.get(); + + _runtime_filter_states.push_back(std::move(filterparams)); + std::map pipeline_id_to_task; + auto get_local_exchange_state = [&](PipelinePtr pipeline) + -> std::map, + std::shared_ptr>> { + std::map, + std::shared_ptr>> + le_state_map; + auto source_id = pipeline->operator_xs().front()->operator_id(); + if (auto iter = _op_id_to_le_state.find(source_id); iter != _op_id_to_le_state.end()) { + le_state_map.insert({source_id, iter->second}); + } + for (auto sink_to_source_id : pipeline->sink_x()->dests_id()) { + if (auto iter = _op_id_to_le_state.find(sink_to_source_id); + iter != _op_id_to_le_state.end()) { + le_state_map.insert({sink_to_source_id, iter->second}); + } + } + return le_state_map; + }; + auto get_task_runtime_state = [&](int task_id) -> RuntimeState* { + DCHECK(_task_runtime_states[task_id]); + return _task_runtime_states[task_id].get(); + }; + for (size_t pip_idx = 0; pip_idx < _pipelines.size(); pip_idx++) { + auto& pipeline = _pipelines[pip_idx]; + if (pipeline->need_to_create_task()) { + // build task runtime state + _task_runtime_states.push_back(RuntimeState::create_unique( + this, local_params.fragment_instance_id, request.query_id, + request.fragment_id, request.query_options, _query_ctx->query_globals, + _exec_env, _query_ctx.get())); + auto& task_runtime_state = _task_runtime_states.back(); + init_runtime_state(task_runtime_state); + auto cur_task_id = _total_tasks++; + task_runtime_state->set_task_id(cur_task_id); + task_runtime_state->set_task_num(pipeline->num_tasks()); + auto task = std::make_unique( + pipeline, cur_task_id, get_task_runtime_state(cur_task_id), this, + pipeline_id_to_profile[pip_idx].get(), get_local_exchange_state(pipeline), + i); + pipeline_id_to_task.insert({pipeline->id(), task.get()}); + _tasks[i].emplace_back(std::move(task)); + } + } + + /** + * Build DAG for pipeline tasks. + * For example, we have + * + * ExchangeSink (Pipeline1) JoinBuildSink (Pipeline2) + * \ / + * JoinProbeOperator1 (Pipeline1) JoinBuildSink (Pipeline3) + * \ / + * JoinProbeOperator2 (Pipeline1) + * + * In this fragment, we have three pipelines and pipeline 1 depends on pipeline 2 and pipeline 3. + * To build this DAG, `_dag` manage dependencies between pipelines by pipeline ID and + * `pipeline_id_to_task` is used to find the task by a unique pipeline ID. + * + * Finally, we have two upstream dependencies in Pipeline1 corresponding to JoinProbeOperator1 + * and JoinProbeOperator2. + */ + + // First, set up the parent profile,task runtime state + + auto prepare_and_set_parent_profile = [&](PipelineXTask* task, size_t pip_idx) { + DCHECK(pipeline_id_to_profile[pip_idx]); + RETURN_IF_ERROR( + task->prepare(local_params, request.fragment.output_sink, _query_ctx.get())); + return Status::OK(); + }; + + for (auto& _pipeline : _pipelines) { + if (pipeline_id_to_task.contains(_pipeline->id())) { + auto* task = pipeline_id_to_task[_pipeline->id()]; + DCHECK(task != nullptr); + + // if this task has upstream dependency, then record them. + if (_dag.find(_pipeline->id()) != _dag.end()) { + auto& deps = _dag[_pipeline->id()]; + for (auto& dep : deps) { + if (pipeline_id_to_task.contains(dep)) { + auto ss = pipeline_id_to_task[dep]->get_sink_shared_state(); + if (ss) { + task->inject_shared_state(ss); + } else { + pipeline_id_to_task[dep]->inject_shared_state( + task->get_source_shared_state()); + } + } + } + } + } + } + for (size_t pip_idx = 0; pip_idx < _pipelines.size(); pip_idx++) { + if (pipeline_id_to_task.contains(_pipelines[pip_idx]->id())) { + auto* task = pipeline_id_to_task[_pipelines[pip_idx]->id()]; + RETURN_IF_ERROR(prepare_and_set_parent_profile(task, pip_idx)); + } + } + { + std::lock_guard l(_state_map_lock); + _runtime_filter_mgr_map[fragment_instance_id] = std::move(runtime_filter_mgr); + } } + _pipeline_parent_map.clear(); + _dag.clear(); + _op_id_to_le_state.clear(); return Status::OK(); } @@ -467,8 +592,7 @@ void PipelineFragmentContext::trigger_report_if_necessary() { } VLOG_FILE << "Query " << print_id(this->get_query_id()) << " fragment " - << this->get_fragment_id() << " instance " - << print_id(this->get_fragment_instance_id()) << " profile:\n" + << this->get_fragment_id() << " profile:\n" << ss.str(); } auto st = send_report(false); @@ -481,265 +605,840 @@ void PipelineFragmentContext::trigger_report_if_necessary() { } // TODO: use virtual function to do abstruct -Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur_pipe) { - auto node_type = node->type(); - switch (node_type) { - // for source - case TPlanNodeType::OLAP_SCAN_NODE: - case TPlanNodeType::JDBC_SCAN_NODE: - case TPlanNodeType::ODBC_SCAN_NODE: - case TPlanNodeType::FILE_SCAN_NODE: - case TPlanNodeType::META_SCAN_NODE: - case TPlanNodeType::GROUP_COMMIT_SCAN_NODE: - case TPlanNodeType::ES_HTTP_SCAN_NODE: - case TPlanNodeType::ES_SCAN_NODE: { - OperatorBuilderPtr operator_t = std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(operator_t)); - break; - } - case TPlanNodeType::MYSQL_SCAN_NODE: { -#ifdef DORIS_WITH_MYSQL - OperatorBuilderPtr operator_t = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(operator_t)); - break; -#else +Status PipelineFragmentContext::_build_pipelines(ObjectPool* pool, + const doris::TPipelineFragmentParams& request, + const DescriptorTbl& descs, OperatorXPtr* root, + PipelinePtr cur_pipe) { + if (request.fragment.plan.nodes.empty()) { + throw Exception(ErrorCode::INTERNAL_ERROR, "Invalid plan which has no plan node!"); + } + + int node_idx = 0; + + cur_pipe->_name.append(std::to_string(cur_pipe->id())); + + RETURN_IF_ERROR(_create_tree_helper(pool, request.fragment.plan.nodes, request, descs, nullptr, + &node_idx, root, cur_pipe, 0)); + + if (node_idx + 1 != request.fragment.plan.nodes.size()) { + // TODO: print thrift msg for diagnostic purposes. return Status::InternalError( - "Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON"); -#endif + "Plan tree only partially reconstructed. Not all thrift nodes were used."); } - case TPlanNodeType::SCHEMA_SCAN_NODE: { - OperatorBuilderPtr operator_t = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(operator_t)); + return Status::OK(); +} + +Status PipelineFragmentContext::_create_tree_helper(ObjectPool* pool, + const std::vector& tnodes, + const doris::TPipelineFragmentParams& request, + const DescriptorTbl& descs, OperatorXPtr parent, + int* node_idx, OperatorXPtr* root, + PipelinePtr& cur_pipe, int child_idx) { + // propagate error case + if (*node_idx >= tnodes.size()) { + // TODO: print thrift msg + return Status::InternalError( + "Failed to reconstruct plan tree from thrift. Node id: {}, number of nodes: {}", + *node_idx, tnodes.size()); + } + const TPlanNode& tnode = tnodes[*node_idx]; + + int num_children = tnodes[*node_idx].num_children; + OperatorXPtr op = nullptr; + RETURN_IF_ERROR(_create_operator(pool, tnodes[*node_idx], request, descs, op, cur_pipe, + parent == nullptr ? -1 : parent->node_id(), child_idx)); + + // assert(parent != nullptr || (node_idx == 0 && root_expr != nullptr)); + if (parent != nullptr) { + // add to parent's child(s) + RETURN_IF_ERROR(parent->set_child(op)); + } else { + *root = op; + } + + cur_pipe->_name.push_back('-'); + cur_pipe->_name.append(std::to_string(op->id())); + cur_pipe->_name.append(op->get_name()); + + // rely on that tnodes is preorder of the plan + for (int i = 0; i < num_children; i++) { + ++*node_idx; + RETURN_IF_ERROR(_create_tree_helper(pool, tnodes, request, descs, op, node_idx, nullptr, + cur_pipe, i)); + + // we are expecting a child, but have used all nodes + // this means we have been given a bad tree and must fail + if (*node_idx >= tnodes.size()) { + // TODO: print thrift msg + return Status::InternalError( + "Failed to reconstruct plan tree from thrift. Node id: {}, number of nodes: {}", + *node_idx, tnodes.size()); + } + } + + RETURN_IF_ERROR(op->init(tnode, _runtime_state.get())); + + return Status::OK(); +} + +void PipelineFragmentContext::_inherit_pipeline_properties( + const DataDistribution& data_distribution, PipelinePtr pipe_with_source, + PipelinePtr pipe_with_sink) { + pipe_with_sink->set_num_tasks(pipe_with_source->num_tasks()); + pipe_with_source->set_num_tasks(_num_instances); + pipe_with_source->set_data_distribution(data_distribution); +} + +Status PipelineFragmentContext::_add_local_exchange_impl( + int idx, ObjectPool* pool, PipelinePtr cur_pipe, PipelinePtr new_pip, + DataDistribution data_distribution, bool* do_local_exchange, int num_buckets, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx, + const bool ignore_data_hash_distribution) { + auto& operator_xs = cur_pipe->operator_xs(); + const auto downstream_pipeline_id = cur_pipe->id(); + auto local_exchange_id = next_operator_id(); + // 1. Create a new pipeline with local exchange sink. + DataSinkOperatorXPtr sink; + auto sink_id = next_sink_operator_id(); + const bool is_shuffled_hash_join = operator_xs.size() > idx + ? operator_xs[idx]->is_shuffled_hash_join() + : cur_pipe->sink_x()->is_shuffled_hash_join(); + sink.reset(new LocalExchangeSinkOperatorX( + sink_id, local_exchange_id, is_shuffled_hash_join ? _total_instances : _num_instances, + data_distribution.partition_exprs, bucket_seq_to_instance_idx)); + RETURN_IF_ERROR(new_pip->set_sink(sink)); + RETURN_IF_ERROR(new_pip->sink_x()->init(data_distribution.distribution_type, num_buckets, + is_shuffled_hash_join, shuffle_idx_to_instance_idx)); + + // 2. Create and initialize LocalExchangeSharedState. + auto shared_state = LocalExchangeSharedState::create_shared(_num_instances); + switch (data_distribution.distribution_type) { + case ExchangeType::HASH_SHUFFLE: + shared_state->exchanger = ShuffleExchanger::create_unique( + std::max(cur_pipe->num_tasks(), _num_instances), + is_shuffled_hash_join ? _total_instances : _num_instances); + break; + case ExchangeType::BUCKET_HASH_SHUFFLE: + shared_state->exchanger = BucketShuffleExchanger::create_unique( + std::max(cur_pipe->num_tasks(), _num_instances), _num_instances, num_buckets, + ignore_data_hash_distribution); break; + case ExchangeType::PASSTHROUGH: + shared_state->exchanger = + PassthroughExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); + break; + case ExchangeType::BROADCAST: + shared_state->exchanger = + BroadcastExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); + break; + case ExchangeType::PASS_TO_ONE: + shared_state->exchanger = + BroadcastExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); + break; + case ExchangeType::ADAPTIVE_PASSTHROUGH: + shared_state->exchanger = + AdaptivePassthroughExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); + break; + default: + return Status::InternalError("Unsupported local exchange type : " + + std::to_string((int)data_distribution.distribution_type)); + } + auto sink_dep = std::make_shared(sink_id, local_exchange_id, + "LOCAL_EXCHANGE_SINK_DEPENDENCY", true, + _runtime_state->get_query_ctx()); + sink_dep->set_shared_state(shared_state.get()); + shared_state->sink_deps.push_back(sink_dep); + _op_id_to_le_state.insert({local_exchange_id, {shared_state, sink_dep}}); + + // 3. Set two pipelines' operator list. For example, split pipeline [Scan - AggSink] to + // pipeline1 [Scan - LocalExchangeSink] and pipeline2 [LocalExchangeSource - AggSink]. + + // 3.1 Initialize new pipeline's operator list. + std::copy(operator_xs.begin(), operator_xs.begin() + idx, + std::inserter(new_pip->operator_xs(), new_pip->operator_xs().end())); + + // 3.2 Erase unused operators in previous pipeline. + operator_xs.erase(operator_xs.begin(), operator_xs.begin() + idx); + + // 4. Initialize LocalExchangeSource and insert it into this pipeline. + OperatorXPtr source_op; + source_op.reset(new LocalExchangeSourceOperatorX(pool, local_exchange_id)); + RETURN_IF_ERROR(source_op->set_child(new_pip->operator_xs().back())); + RETURN_IF_ERROR(source_op->init(data_distribution.distribution_type)); + if (!operator_xs.empty()) { + RETURN_IF_ERROR(operator_xs.front()->set_child(source_op)); + } + operator_xs.insert(operator_xs.begin(), source_op); + + shared_state->create_source_dependencies(source_op->operator_id(), source_op->node_id(), + _query_ctx.get()); + + // 5. Set children for two pipelines separately. + std::vector> new_children; + std::vector edges_with_source; + for (auto child : cur_pipe->children()) { + bool found = false; + for (auto op : new_pip->operator_xs()) { + if (child->sink_x()->node_id() == op->node_id()) { + new_pip->set_children(child); + found = true; + }; + } + if (!found) { + new_children.push_back(child); + edges_with_source.push_back(child->id()); + } } - case TPlanNodeType::EXCHANGE_NODE: { - OperatorBuilderPtr operator_t = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(operator_t)); + new_children.push_back(new_pip); + edges_with_source.push_back(new_pip->id()); + + // 6. Set DAG for new pipelines. + if (!new_pip->children().empty()) { + std::vector edges_with_sink; + for (auto child : new_pip->children()) { + edges_with_sink.push_back(child->id()); + } + _dag.insert({new_pip->id(), edges_with_sink}); + } + cur_pipe->set_children(new_children); + _dag[downstream_pipeline_id] = edges_with_source; + RETURN_IF_ERROR(new_pip->sink_x()->set_child(new_pip->operator_xs().back())); + RETURN_IF_ERROR(cur_pipe->sink_x()->set_child(cur_pipe->operator_xs().back())); + + // 7. Inherit properties from current pipeline. + _inherit_pipeline_properties(data_distribution, cur_pipe, new_pip); + return Status::OK(); +} + +Status PipelineFragmentContext::_add_local_exchange( + int pip_idx, int idx, int node_id, ObjectPool* pool, PipelinePtr cur_pipe, + DataDistribution data_distribution, bool* do_local_exchange, int num_buckets, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx, + const bool ignore_data_distribution) { + DCHECK(_enable_local_shuffle()); + if (_num_instances <= 1) { + return Status::OK(); + } + + if (!cur_pipe->need_to_local_exchange(data_distribution)) { + return Status::OK(); + } + *do_local_exchange = true; + + auto& operator_xs = cur_pipe->operator_xs(); + auto total_op_num = operator_xs.size(); + auto new_pip = add_pipeline(cur_pipe, pip_idx + 1); + RETURN_IF_ERROR(_add_local_exchange_impl( + idx, pool, cur_pipe, new_pip, data_distribution, do_local_exchange, num_buckets, + bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, ignore_data_distribution)); + + CHECK(total_op_num + 1 == cur_pipe->operator_xs().size() + new_pip->operator_xs().size()) + << "total_op_num: " << total_op_num + << " cur_pipe->operator_xs().size(): " << cur_pipe->operator_xs().size() + << " new_pip->operator_xs().size(): " << new_pip->operator_xs().size(); + + // Add passthrough local exchanger if necessary + if (cur_pipe->num_tasks() > 1 && new_pip->num_tasks() == 1 && + Pipeline::is_hash_exchange(data_distribution.distribution_type)) { + RETURN_IF_ERROR(_add_local_exchange_impl( + new_pip->operator_xs().size(), pool, new_pip, add_pipeline(new_pip, pip_idx + 2), + DataDistribution(ExchangeType::PASSTHROUGH), do_local_exchange, num_buckets, + bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, ignore_data_distribution)); + } + return Status::OK(); +} + +Status PipelineFragmentContext::_plan_local_exchange( + int num_buckets, const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx) { + for (int pip_idx = _pipelines.size() - 1; pip_idx >= 0; pip_idx--) { + _pipelines[pip_idx]->init_data_distribution(); + // Set property if child pipeline is not join operator's child. + if (!_pipelines[pip_idx]->children().empty()) { + for (auto& child : _pipelines[pip_idx]->children()) { + if (child->sink_x()->node_id() == + _pipelines[pip_idx]->operator_xs().front()->node_id()) { + RETURN_IF_ERROR(_pipelines[pip_idx]->operator_xs().front()->set_child( + child->operator_xs().back())); + _pipelines[pip_idx]->set_data_distribution(child->data_distribution()); + } + } + } + + RETURN_IF_ERROR(_plan_local_exchange( + _pipelines[pip_idx]->operator_xs().front()->ignore_data_hash_distribution() + ? _num_instances + : num_buckets, + pip_idx, _pipelines[pip_idx], bucket_seq_to_instance_idx, + shuffle_idx_to_instance_idx, + _pipelines[pip_idx]->operator_xs().front()->ignore_data_hash_distribution())); + } + return Status::OK(); +} + +Status PipelineFragmentContext::_plan_local_exchange( + int num_buckets, int pip_idx, PipelinePtr pip, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx, + const bool ignore_data_hash_distribution) { + int idx = 1; + bool do_local_exchange = false; + do { + auto& ops = pip->operator_xs(); + do_local_exchange = false; + // Plan local exchange for each operator. + for (; idx < ops.size();) { + if (ops[idx]->required_data_distribution().need_local_exchange()) { + RETURN_IF_ERROR(_add_local_exchange( + pip_idx, idx, ops[idx]->node_id(), _runtime_state->obj_pool(), pip, + ops[idx]->required_data_distribution(), &do_local_exchange, num_buckets, + bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, + ignore_data_hash_distribution)); + } + if (do_local_exchange) { + // If local exchange is needed for current operator, we will split this pipeline to + // two pipelines by local exchange sink/source. And then we need to process remaining + // operators in this pipeline so we set idx to 2 (0 is local exchange source and 1 + // is current operator was already processed) and continue to plan local exchange. + idx = 2; + break; + } + idx++; + } + } while (do_local_exchange); + if (pip->sink_x()->required_data_distribution().need_local_exchange()) { + RETURN_IF_ERROR(_add_local_exchange( + pip_idx, idx, pip->sink_x()->node_id(), _runtime_state->obj_pool(), pip, + pip->sink_x()->required_data_distribution(), &do_local_exchange, num_buckets, + bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, + ignore_data_hash_distribution)); + } + return Status::OK(); +} + +Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink, + const std::vector& output_exprs, + const TPipelineFragmentParams& params, + const RowDescriptor& row_desc, + RuntimeState* state, DescriptorTbl& desc_tbl, + PipelineId cur_pipeline_id) { + switch (thrift_sink.type) { + case TDataSinkType::DATA_STREAM_SINK: { + if (!thrift_sink.__isset.stream_sink) { + return Status::InternalError("Missing data stream sink."); + } + _sink.reset(new ExchangeSinkOperatorX(state, row_desc, next_sink_operator_id(), + thrift_sink.stream_sink, params.destinations)); break; } - case TPlanNodeType::EMPTY_SET_NODE: { - OperatorBuilderPtr operator_t = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(operator_t)); + case TDataSinkType::RESULT_SINK: { + if (!thrift_sink.__isset.result_sink) { + return Status::InternalError("Missing data buffer sink."); + } + + // TODO: figure out good buffer size based on size of output row + _sink.reset(new ResultSinkOperatorX(next_sink_operator_id(), row_desc, output_exprs, + thrift_sink.result_sink)); break; } - case TPlanNodeType::DATA_GEN_SCAN_NODE: { - OperatorBuilderPtr operator_t = std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(operator_t)); + case TDataSinkType::GROUP_COMMIT_OLAP_TABLE_SINK: + case TDataSinkType::OLAP_TABLE_SINK: { + if (state->query_options().enable_memtable_on_sink_node && + !_has_inverted_index_or_partial_update(thrift_sink.olap_table_sink) && + !config::is_cloud_mode()) { + _sink.reset(new OlapTableSinkV2OperatorX(pool, next_sink_operator_id(), row_desc, + output_exprs)); + } else { + _sink.reset(new OlapTableSinkOperatorX(pool, next_sink_operator_id(), row_desc, + output_exprs)); + } break; } - case TPlanNodeType::UNION_NODE: { - auto* union_node = assert_cast(node); - if (union_node->children_count() == 0 && - union_node->get_first_materialized_child_idx() == 0) { // only have const expr - OperatorBuilderPtr builder = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(builder)); + case TDataSinkType::GROUP_COMMIT_BLOCK_SINK: { + DCHECK(thrift_sink.__isset.olap_table_sink); + _sink.reset(new GroupCommitBlockSinkOperatorX(next_sink_operator_id(), row_desc)); + break; + } + case TDataSinkType::HIVE_TABLE_SINK: { + if (!thrift_sink.__isset.hive_table_sink) { + return Status::InternalError("Missing hive table sink."); + } + _sink.reset( + new HiveTableSinkOperatorX(pool, next_sink_operator_id(), row_desc, output_exprs)); + break; + } + case TDataSinkType::JDBC_TABLE_SINK: { + if (!thrift_sink.__isset.jdbc_table_sink) { + return Status::InternalError("Missing data jdbc sink."); + } + if (config::enable_java_support) { + _sink.reset( + new JdbcTableSinkOperatorX(row_desc, next_sink_operator_id(), output_exprs)); } else { - int child_count = union_node->children_count(); - auto data_queue = std::make_shared(child_count); - data_queue->set_max_blocks_in_sub_queue(_runtime_state->data_queue_max_blocks()); - for (int child_id = 0; child_id < child_count; ++child_id) { - auto new_child_pipeline = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(union_node->child(child_id), new_child_pipeline)); - OperatorBuilderPtr child_sink_builder = std::make_shared( - union_node->id(), child_id, union_node, data_queue); - RETURN_IF_ERROR(new_child_pipeline->set_sink_builder(child_sink_builder)); - } - OperatorBuilderPtr source_builder = std::make_shared( - node->id(), union_node, data_queue); - RETURN_IF_ERROR(cur_pipe->add_operator(source_builder)); + return Status::InternalError( + "Jdbc table sink is not enabled, you can change be config " + "enable_java_support to true and restart be."); } break; } - case TPlanNodeType::AGGREGATION_NODE: { - auto* agg_node = static_cast(node); - auto new_pipe = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(0), new_pipe)); - if (agg_node->is_probe_expr_ctxs_empty() && agg_node->agg_output_desc()->slots().empty()) { - return Status::InternalError("Illegal aggregate node " + - std::to_string(agg_node->id()) + - ": group by and output is empty"); + case TDataSinkType::RESULT_FILE_SINK: { + if (!thrift_sink.__isset.result_file_sink) { + return Status::InternalError("Missing result file sink."); } - const int64_t data_queue_max_blocks = _runtime_state->data_queue_max_blocks(); - if (agg_node->is_aggregate_evaluators_empty() && !agg_node->is_probe_expr_ctxs_empty()) { - auto data_queue = std::make_shared(1); - data_queue->set_max_blocks_in_sub_queue(data_queue_max_blocks); - OperatorBuilderPtr pre_agg_sink = - std::make_shared(node->id(), agg_node, - data_queue); - RETURN_IF_ERROR(new_pipe->set_sink_builder(pre_agg_sink)); - - OperatorBuilderPtr pre_agg_source = - std::make_shared( - node->id(), agg_node, data_queue); - RETURN_IF_ERROR(cur_pipe->add_operator(pre_agg_source)); - } else if (agg_node->is_streaming_preagg() && !agg_node->is_probe_expr_ctxs_empty()) { - auto data_queue = std::make_shared(1); - data_queue->set_max_blocks_in_sub_queue(data_queue_max_blocks); - OperatorBuilderPtr pre_agg_sink = std::make_shared( - node->id(), agg_node, data_queue); - RETURN_IF_ERROR(new_pipe->set_sink_builder(pre_agg_sink)); - - OperatorBuilderPtr pre_agg_source = std::make_shared( - node->id(), agg_node, data_queue); - RETURN_IF_ERROR(cur_pipe->add_operator(pre_agg_source)); + // TODO: figure out good buffer size based on size of output row + // Result file sink is not the top sink + if (params.__isset.destinations && !params.destinations.empty()) { + _sink.reset(new ResultFileSinkOperatorX(next_sink_operator_id(), row_desc, + thrift_sink.result_file_sink, + params.destinations, output_exprs, desc_tbl)); } else { - OperatorBuilderPtr agg_sink = - std::make_shared(node->id(), agg_node); - RETURN_IF_ERROR(new_pipe->set_sink_builder(agg_sink)); - - OperatorBuilderPtr agg_source = - std::make_shared(node->id(), agg_node); - RETURN_IF_ERROR(cur_pipe->add_operator(agg_source)); + _sink.reset( + new ResultFileSinkOperatorX(next_sink_operator_id(), row_desc, output_exprs)); } break; } - case TPlanNodeType::SORT_NODE: { - auto new_pipeline = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(0), new_pipeline)); + case TDataSinkType::MULTI_CAST_DATA_STREAM_SINK: { + DCHECK(thrift_sink.__isset.multi_cast_stream_sink); + DCHECK_GT(thrift_sink.multi_cast_stream_sink.sinks.size(), 0); + // TODO: figure out good buffer size based on size of output row + auto sink_id = next_sink_operator_id(); + auto sender_size = thrift_sink.multi_cast_stream_sink.sinks.size(); + // one sink has multiple sources. + std::vector sources; + for (int i = 0; i < sender_size; ++i) { + auto source_id = next_operator_id(); + sources.push_back(source_id); + } - OperatorBuilderPtr sort_sink = std::make_shared(node->id(), node); - RETURN_IF_ERROR(new_pipeline->set_sink_builder(sort_sink)); + _sink.reset(new MultiCastDataStreamSinkOperatorX( + sink_id, sources, thrift_sink.multi_cast_stream_sink.sinks.size(), pool, + thrift_sink.multi_cast_stream_sink, row_desc)); + for (int i = 0; i < sender_size; ++i) { + auto new_pipeline = add_pipeline(); + RowDescriptor* _row_desc = nullptr; + { + const auto& tmp_row_desc = + !thrift_sink.multi_cast_stream_sink.sinks[i].output_exprs.empty() + ? RowDescriptor(state->desc_tbl(), + {thrift_sink.multi_cast_stream_sink.sinks[i] + .output_tuple_id}, + {false}) + : _sink->row_desc(); + _row_desc = pool->add(new RowDescriptor(tmp_row_desc)); + } + auto source_id = sources[i]; + OperatorXPtr source_op; + // 1. create and set the source operator of multi_cast_data_stream_source for new pipeline + source_op.reset(new MultiCastDataStreamerSourceOperatorX( + i, pool, thrift_sink.multi_cast_stream_sink.sinks[i], row_desc, source_id)); + RETURN_IF_ERROR(new_pipeline->add_operator(source_op)); + // 2. create and set sink operator of data stream sender for new pipeline + + DataSinkOperatorXPtr sink_op; + sink_op.reset( + new ExchangeSinkOperatorX(state, *_row_desc, next_sink_operator_id(), + thrift_sink.multi_cast_stream_sink.sinks[i], + thrift_sink.multi_cast_stream_sink.destinations[i])); + + RETURN_IF_ERROR(new_pipeline->set_sink(sink_op)); + { + TDataSink* t = pool->add(new TDataSink()); + t->stream_sink = thrift_sink.multi_cast_stream_sink.sinks[i]; + RETURN_IF_ERROR(sink_op->init(*t)); + } - OperatorBuilderPtr sort_source = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(sort_source)); + // 3. set dependency dag + _dag[new_pipeline->id()].push_back(cur_pipeline_id); + } + if (sources.empty()) { + return Status::InternalError("size of sources must be greater than 0"); + } break; } - case TPlanNodeType::PARTITION_SORT_NODE: { - auto new_pipeline = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(0), new_pipeline)); - - OperatorBuilderPtr partition_sort_sink = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(new_pipeline->set_sink_builder(partition_sort_sink)); + default: + return Status::InternalError("Unsuported sink type in pipeline: {}", thrift_sink.type); + } + return Status::OK(); +} - OperatorBuilderPtr partition_sort_source = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(partition_sort_source)); +// NOLINTBEGIN(readability-function-size) +// NOLINTBEGIN(readability-function-cognitive-complexity) +Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNode& tnode, + const doris::TPipelineFragmentParams& request, + const DescriptorTbl& descs, OperatorXPtr& op, + PipelinePtr& cur_pipe, int parent_idx, + int child_idx) { + // We directly construct the operator from Thrift because the given array is in the order of preorder traversal. + // Therefore, here we need to use a stack-like structure. + _pipeline_parent_map.pop(cur_pipe, parent_idx, child_idx); + std::stringstream error_msg; + + switch (tnode.node_type) { + case TPlanNodeType::OLAP_SCAN_NODE: { + op.reset(new OlapScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + if (request.__isset.parallel_instances) { + cur_pipe->set_num_tasks(request.parallel_instances); + op->set_ignore_data_distribution(); + } break; } - case TPlanNodeType::ANALYTIC_EVAL_NODE: { - auto new_pipeline = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(0), new_pipeline)); - - OperatorBuilderPtr analytic_sink = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(new_pipeline->set_sink_builder(analytic_sink)); - - OperatorBuilderPtr analytic_source = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(analytic_source)); + case doris::TPlanNodeType::JDBC_SCAN_NODE: { + if (config::enable_java_support) { + op.reset(new JDBCScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + } else { + return Status::InternalError( + "Jdbc scan node is disabled, you can change be config enable_java_support " + "to true and restart be."); + } + if (request.__isset.parallel_instances) { + cur_pipe->set_num_tasks(request.parallel_instances); + op->set_ignore_data_distribution(); + } break; } - case TPlanNodeType::REPEAT_NODE: { - RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe)); - OperatorBuilderPtr builder = std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(builder)); + case doris::TPlanNodeType::FILE_SCAN_NODE: { + op.reset(new FileScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + if (request.__isset.parallel_instances) { + cur_pipe->set_num_tasks(request.parallel_instances); + op->set_ignore_data_distribution(); + } break; } - case TPlanNodeType::ASSERT_NUM_ROWS_NODE: { - RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe)); - OperatorBuilderPtr builder = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(builder)); + case TPlanNodeType::ES_SCAN_NODE: + case TPlanNodeType::ES_HTTP_SCAN_NODE: { + op.reset(new EsScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + if (request.__isset.parallel_instances) { + cur_pipe->set_num_tasks(request.parallel_instances); + op->set_ignore_data_distribution(); + } break; } - case TPlanNodeType::TABLE_FUNCTION_NODE: { - RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe)); - OperatorBuilderPtr builder = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(builder)); + case TPlanNodeType::EXCHANGE_NODE: { + int num_senders = find_with_default(request.per_exch_num_senders, tnode.node_id, 0); + DCHECK_GT(num_senders, 0); + op.reset(new ExchangeSourceOperatorX(pool, tnode, next_operator_id(), descs, num_senders)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + if (request.__isset.parallel_instances) { + op->set_ignore_data_distribution(); + cur_pipe->set_num_tasks(request.parallel_instances); + } + break; + } + case TPlanNodeType::AGGREGATION_NODE: { + if (tnode.agg_node.grouping_exprs.empty() && + descs.get_tuple_descriptor(tnode.agg_node.output_tuple_id)->slots().empty()) { + return Status::InternalError("Illegal aggregate node " + std::to_string(tnode.node_id) + + ": group by and output is empty"); + } + if (tnode.agg_node.aggregate_functions.empty() && !_runtime_state->enable_agg_spill() && + request.query_options.__isset.enable_distinct_streaming_aggregation && + request.query_options.enable_distinct_streaming_aggregation && + !tnode.agg_node.grouping_exprs.empty()) { + op.reset(new DistinctStreamingAggOperatorX(pool, next_operator_id(), tnode, descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + } else if (tnode.agg_node.__isset.use_streaming_preaggregation && + tnode.agg_node.use_streaming_preaggregation && + !tnode.agg_node.grouping_exprs.empty()) { + op.reset(new StreamingAggOperatorX(pool, next_operator_id(), tnode, descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + } else { + if (_runtime_state->enable_agg_spill() && !tnode.agg_node.grouping_exprs.empty()) { + op.reset(new PartitionedAggSourceOperatorX(pool, tnode, next_operator_id(), descs)); + } else { + op.reset(new AggSourceOperatorX(pool, tnode, next_operator_id(), descs)); + } + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + cur_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(cur_pipe->id()); + + DataSinkOperatorXPtr sink; + if (_runtime_state->enable_agg_spill() && !tnode.agg_node.grouping_exprs.empty()) { + sink.reset(new PartitionedAggSinkOperatorX(pool, next_sink_operator_id(), tnode, + descs, _require_bucket_distribution)); + } else { + sink.reset(new AggSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, + _require_bucket_distribution)); + } + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(cur_pipe->set_sink(sink)); + RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); + } break; } case TPlanNodeType::HASH_JOIN_NODE: { - auto* join_node = assert_cast(node); - auto new_pipe = add_pipeline(); - if (join_node->should_build_hash_table()) { - RETURN_IF_ERROR(_build_pipelines(node->child(1), new_pipe)); + const auto is_broadcast_join = tnode.hash_join_node.__isset.is_broadcast_join && + tnode.hash_join_node.is_broadcast_join; + const auto enable_join_spill = _runtime_state->enable_join_spill(); + if (enable_join_spill && !is_broadcast_join) { + auto tnode_ = tnode; + /// TODO: support rf in partitioned hash join + tnode_.runtime_filters.clear(); + const uint32_t partition_count = 32; + auto inner_probe_operator = + std::make_shared(pool, tnode_, 0, descs); + auto inner_sink_operator = std::make_shared( + pool, 0, tnode_, descs, _need_local_merge); + + RETURN_IF_ERROR(inner_probe_operator->init(tnode_, _runtime_state.get())); + RETURN_IF_ERROR(inner_sink_operator->init(tnode_, _runtime_state.get())); + + auto probe_operator = std::make_shared( + pool, tnode_, next_operator_id(), descs, partition_count); + probe_operator->set_inner_operators(inner_sink_operator, inner_probe_operator); + op = std::move(probe_operator); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + PipelinePtr build_side_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); + + auto sink_operator = std::make_shared( + pool, next_sink_operator_id(), tnode_, descs, _need_local_merge, + partition_count); + sink_operator->set_inner_operators(inner_sink_operator, inner_probe_operator); + DataSinkOperatorXPtr sink = std::move(sink_operator); + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); + RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode_, _runtime_state.get())); + + _pipeline_parent_map.push(op->node_id(), cur_pipe); + _pipeline_parent_map.push(op->node_id(), build_side_pipe); + } else { + op.reset(new HashJoinProbeOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + PipelinePtr build_side_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); + + DataSinkOperatorXPtr sink; + sink.reset(new HashJoinBuildSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, + _need_local_merge)); + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); + RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode, _runtime_state.get())); + + _pipeline_parent_map.push(op->node_id(), cur_pipe); + _pipeline_parent_map.push(op->node_id(), build_side_pipe); + } + break; + } + case TPlanNodeType::CROSS_JOIN_NODE: { + op.reset(new NestedLoopJoinProbeOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + PipelinePtr build_side_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); + + DataSinkOperatorXPtr sink; + sink.reset(new NestedLoopJoinBuildSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, + _need_local_merge)); + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); + RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode, _runtime_state.get())); + _pipeline_parent_map.push(op->node_id(), cur_pipe); + _pipeline_parent_map.push(op->node_id(), build_side_pipe); + break; + } + case TPlanNodeType::UNION_NODE: { + int child_count = tnode.num_children; + op.reset(new UnionSourceOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + for (int i = 0; i < child_count; i++) { + PipelinePtr build_side_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); + DataSinkOperatorXPtr sink; + sink.reset(new UnionSinkOperatorX(i, next_sink_operator_id(), pool, tnode, descs)); + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); + RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode, _runtime_state.get())); + // preset children pipelines. if any pipeline found this as its father, will use the prepared pipeline to build. + _pipeline_parent_map.push(op->node_id(), build_side_pipe); + } + break; + } + case TPlanNodeType::SORT_NODE: { + if (_runtime_state->enable_sort_spill()) { + op.reset(new SpillSortSourceOperatorX(pool, tnode, next_operator_id(), descs)); } else { - OperatorBuilderPtr builder = std::make_shared( - node->child(1)->id(), node->child(1)->row_desc(), node->child(1)); - RETURN_IF_ERROR(new_pipe->add_operator(builder)); + op.reset(new SortSourceOperatorX(pool, tnode, next_operator_id(), descs)); } - OperatorBuilderPtr join_sink = - std::make_shared(node->id(), join_node); - RETURN_IF_ERROR(new_pipe->set_sink_builder(join_sink)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); - RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe)); - OperatorBuilderPtr join_source = - std::make_shared(node->id(), join_node); - RETURN_IF_ERROR(cur_pipe->add_operator(join_source)); + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + cur_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(cur_pipe->id()); - cur_pipe->add_dependency(new_pipe); + DataSinkOperatorXPtr sink; + if (_runtime_state->enable_sort_spill()) { + sink.reset(new SpillSortSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); + } else { + sink.reset(new SortSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); + } + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(cur_pipe->set_sink(sink)); + RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); break; } - case TPlanNodeType::CROSS_JOIN_NODE: { - auto new_pipe = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(1), new_pipe)); - OperatorBuilderPtr join_sink = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(new_pipe->set_sink_builder(join_sink)); + case doris::TPlanNodeType::PARTITION_SORT_NODE: { + op.reset(new PartitionSortSourceOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); - RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe)); - OperatorBuilderPtr join_source = - std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(join_source)); + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + cur_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(cur_pipe->id()); + + DataSinkOperatorXPtr sink; + sink.reset(new PartitionSortSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(cur_pipe->set_sink(sink)); + RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); + break; + } + case TPlanNodeType::ANALYTIC_EVAL_NODE: { + op.reset(new AnalyticSourceOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); - cur_pipe->add_dependency(new_pipe); + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + cur_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(cur_pipe->id()); + + DataSinkOperatorXPtr sink; + sink.reset(new AnalyticSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(cur_pipe->set_sink(sink)); + RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); break; } case TPlanNodeType::INTERSECT_NODE: { - RETURN_IF_ERROR(_build_operators_for_set_operation_node(node, cur_pipe)); + RETURN_IF_ERROR(_build_operators_for_set_operation_node( + pool, tnode, descs, op, cur_pipe, parent_idx, child_idx)); break; } case TPlanNodeType::EXCEPT_NODE: { - RETURN_IF_ERROR(_build_operators_for_set_operation_node(node, cur_pipe)); + RETURN_IF_ERROR(_build_operators_for_set_operation_node( + pool, tnode, descs, op, cur_pipe, parent_idx, child_idx)); + break; + } + case TPlanNodeType::REPEAT_NODE: { + op.reset(new RepeatOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + break; + } + case TPlanNodeType::TABLE_FUNCTION_NODE: { + op.reset(new TableFunctionOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + break; + } + case TPlanNodeType::ASSERT_NUM_ROWS_NODE: { + op.reset(new AssertNumRowsOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + break; + } + case TPlanNodeType::EMPTY_SET_NODE: { + op.reset(new EmptySetSourceOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + break; + } + case TPlanNodeType::DATA_GEN_SCAN_NODE: { + op.reset(new DataGenSourceOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + break; + } + case TPlanNodeType::SCHEMA_SCAN_NODE: { + op.reset(new SchemaScanOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + break; + } + case TPlanNodeType::META_SCAN_NODE: { + op.reset(new MetaScanOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); break; } case TPlanNodeType::SELECT_NODE: { - RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe)); - OperatorBuilderPtr builder = std::make_shared(node->id(), node); - RETURN_IF_ERROR(cur_pipe->add_operator(builder)); + op.reset(new SelectOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); break; } default: - return Status::InternalError("Unsupported exec type in pipeline: {}", - print_plan_node_type(node_type)); + return Status::InternalError("Unsupported exec type in pipelineX: {}", + print_plan_node_type(tnode.node_type)); } + _require_bucket_distribution = true; + return Status::OK(); } +// NOLINTEND(readability-function-cognitive-complexity) +// NOLINTEND(readability-function-size) template -Status PipelineFragmentContext::_build_operators_for_set_operation_node(ExecNode* node, - PipelinePtr cur_pipe) { - auto build_pipeline = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(0), build_pipeline)); - OperatorBuilderPtr sink_builder = - std::make_shared>(node->id(), node); - RETURN_IF_ERROR(build_pipeline->set_sink_builder(sink_builder)); - std::vector all_pipelines; - all_pipelines.emplace_back(build_pipeline); - for (int child_id = 1; child_id < node->children_count(); ++child_id) { - auto probe_pipeline = add_pipeline(); - RETURN_IF_ERROR(_build_pipelines(node->child(child_id), probe_pipeline)); - OperatorBuilderPtr probe_sink_builder = - std::make_shared>(node->id(), child_id, - node); - RETURN_IF_ERROR(probe_pipeline->set_sink_builder(probe_sink_builder)); - //eg: These sinks must be completed one by one in order, child(1) must wait child(0) build finish - probe_pipeline->add_dependency(all_pipelines[child_id - 1]); - all_pipelines.emplace_back(probe_pipeline); - } - - OperatorBuilderPtr source_builder = - std::make_shared>(node->id(), node); - return cur_pipe->add_operator(source_builder); +Status PipelineFragmentContext::_build_operators_for_set_operation_node( + ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs, OperatorXPtr& op, + PipelinePtr& cur_pipe, int parent_idx, int child_idx) { + op.reset(new SetSourceOperatorX(pool, tnode, next_operator_id(), descs)); + RETURN_IF_ERROR(cur_pipe->add_operator(op)); + + const auto downstream_pipeline_id = cur_pipe->id(); + if (_dag.find(downstream_pipeline_id) == _dag.end()) { + _dag.insert({downstream_pipeline_id, {}}); + } + + for (int child_id = 0; child_id < tnode.num_children; child_id++) { + PipelinePtr probe_side_pipe = add_pipeline(cur_pipe); + _dag[downstream_pipeline_id].push_back(probe_side_pipe->id()); + + DataSinkOperatorXPtr sink; + if (child_id == 0) { + sink.reset(new SetSinkOperatorX(child_id, next_sink_operator_id(), pool, + tnode, descs)); + } else { + sink.reset(new SetProbeSinkOperatorX(child_id, next_sink_operator_id(), + pool, tnode, descs)); + } + sink->set_dests_id({op->operator_id()}); + RETURN_IF_ERROR(probe_side_pipe->set_sink(sink)); + RETURN_IF_ERROR(probe_side_pipe->sink_x()->init(tnode, _runtime_state.get())); + // prepare children pipelines. if any pipeline found this as its father, will use the prepared pipeline to build. + _pipeline_parent_map.push(op->node_id(), probe_side_pipe); + } + + return Status::OK(); } Status PipelineFragmentContext::submit() { @@ -752,23 +1451,16 @@ Status PipelineFragmentContext::submit() { Status st; auto* scheduler = _query_ctx->get_pipe_exec_scheduler(); for (auto& task : _tasks) { - st = scheduler->schedule_task(task.get()); - if (!st) { - { + for (auto& t : task) { + st = scheduler->schedule_task(t.get()); + if (!st) { std::lock_guard l(_status_lock); cancel(PPlanFragmentCancelReason::INTERNAL_ERROR, "submit context fail"); - } - { - // The fragment instance may contains 10 tasks, maybe 8 of them is in scheduler running - // and the 9th failed to add to scheduler, then it will update total_tasks. - // But the previous 8 tasks may finished and try to access total_tasks. So that - // has to use a lock to protect it. - std::lock_guard l(_task_mutex); _total_tasks = submit_tasks; + break; } - break; + submit_tasks++; } - submit_tasks++; } if (!st.ok()) { std::lock_guard l(_task_mutex); @@ -783,134 +1475,25 @@ Status PipelineFragmentContext::submit() { } void PipelineFragmentContext::close_sink() { - if (_sink) { - if (_prepared) { - static_cast( - _sink->close(_runtime_state.get(), Status::RuntimeError("prepare failed"))); - } else { - static_cast(_sink->close(_runtime_state.get(), Status::OK())); + for (auto& tasks : _tasks) { + auto& root_task = *tasks.begin(); + auto st = root_task->close_sink(_prepared ? Status::RuntimeError("prepare failed") + : Status::OK()); + if (!st.ok()) { + LOG_WARNING("PipelineFragmentContext::close_sink() error").tag("msg", st.msg()); } } } -void PipelineFragmentContext::close_if_prepare_failed(Status /*st*/) { - if (_tasks.empty()) { - if (_root_plan) { - static_cast(_root_plan->close(_runtime_state.get())); - } - if (_sink) { - static_cast( - _sink->close(_runtime_state.get(), Status::RuntimeError("prepare failed"))); - } - } +void PipelineFragmentContext::close_if_prepare_failed(Status st) { for (auto& task : _tasks) { - DCHECK(!task->is_pending_finish()); - WARN_IF_ERROR(task->close(Status::OK()), - fmt::format("Query {} closed since prepare failed", print_id(_query_id))); - close_a_pipeline(); - } -} - -// construct sink operator -Status PipelineFragmentContext::_create_sink(int sender_id, const TDataSink& thrift_sink, - RuntimeState* state) { - OperatorBuilderPtr sink_; - switch (thrift_sink.type) { - case TDataSinkType::DATA_STREAM_SINK: { - sink_ = std::make_shared(thrift_sink.stream_sink.dest_node_id, - _sink.get()); - break; - } - case TDataSinkType::RESULT_SINK: { - sink_ = std::make_shared(next_operator_builder_id(), - _sink.get()); - break; - } - case TDataSinkType::GROUP_COMMIT_OLAP_TABLE_SINK: - case TDataSinkType::OLAP_TABLE_SINK: { - DCHECK(thrift_sink.__isset.olap_table_sink); - if (state->query_options().enable_memtable_on_sink_node && - !_has_inverted_index_or_partial_update(thrift_sink.olap_table_sink) && - !config::is_cloud_mode()) { - sink_ = std::make_shared(next_operator_builder_id(), - _sink.get()); - } else { - sink_ = std::make_shared(next_operator_builder_id(), - _sink.get()); - } - break; - } - case TDataSinkType::GROUP_COMMIT_BLOCK_SINK: { - sink_ = std::make_shared(next_operator_builder_id(), - _sink.get()); - break; - } - case TDataSinkType::HIVE_TABLE_SINK: { - sink_ = std::make_shared(next_operator_builder_id(), - _sink.get()); - break; - } - case TDataSinkType::MYSQL_TABLE_SINK: - case TDataSinkType::JDBC_TABLE_SINK: - case TDataSinkType::ODBC_TABLE_SINK: - case TDataSinkType::RESULT_FILE_SINK: { - sink_ = std::make_shared( - thrift_sink.result_file_sink.dest_node_id, _sink.get()); - break; - } - case TDataSinkType::MULTI_CAST_DATA_STREAM_SINK: { - sink_ = std::make_shared(next_operator_builder_id(), - _sink.get()); - RETURN_IF_ERROR(_root_pipeline->set_sink_builder(sink_)); - - auto& multi_cast_data_streamer = - assert_cast(_sink.get()) - ->get_multi_cast_data_streamer(); - DCHECK_EQ(thrift_sink.multi_cast_stream_sink.sinks.size(), - thrift_sink.multi_cast_stream_sink.destinations.size()); - auto sender_size = thrift_sink.multi_cast_stream_sink.sinks.size(); - _multi_cast_stream_sink_senders.resize(sender_size); - for (int i = 0; i < sender_size; ++i) { - auto new_pipeline = add_pipeline(); - - auto row_desc = - !thrift_sink.multi_cast_stream_sink.sinks[i].output_exprs.empty() - ? RowDescriptor( - _runtime_state->desc_tbl(), - {thrift_sink.multi_cast_stream_sink.sinks[i].output_tuple_id}, - {false}) - : sink_->row_desc(); - // 1. create the data stream sender sink - _multi_cast_stream_sink_senders[i] = std::make_unique( - _runtime_state.get(), _runtime_state->obj_pool(), sender_id, row_desc, - thrift_sink.multi_cast_stream_sink.sinks[i], - thrift_sink.multi_cast_stream_sink.destinations[i]); - - // 2. create and set the source operator of multi_cast_data_stream_source for new pipeline - OperatorBuilderPtr source_op = - std::make_shared( - next_operator_builder_id(), i, multi_cast_data_streamer, - thrift_sink.multi_cast_stream_sink.sinks[i]); - static_cast(new_pipeline->add_operator(source_op)); - - // 3. create and set sink operator of data stream sender for new pipeline - OperatorBuilderPtr sink_op_builder = std::make_shared( - next_operator_builder_id(), _multi_cast_stream_sink_senders[i].get(), i); - static_cast(new_pipeline->set_sink_builder(sink_op_builder)); - - // 4. init and prepare the data_stream_sender of diff exchange - TDataSink t; - t.stream_sink = thrift_sink.multi_cast_stream_sink.sinks[i]; - RETURN_IF_ERROR(_multi_cast_stream_sink_senders[i]->init(t)); - RETURN_IF_ERROR(_multi_cast_stream_sink_senders[i]->prepare(state)); + for (auto& t : task) { + DCHECK(!t->is_pending_finish()); + WARN_IF_ERROR(t->close(st), "close_if_prepare_failed failed: "); + close_a_pipeline(); } - - return Status::OK(); - } - default: - return Status::InternalError("Unsuported sink type in pipeline: {}", thrift_sink.type); } - return _root_pipeline->set_sink_builder(sink_); + _query_ctx->cancel(st.to_string(), st, _fragment_id); } // If all pipeline tasks binded to the fragment instance are finished, then we could @@ -921,8 +1504,6 @@ void PipelineFragmentContext::_close_fragment_instance() { } Defer defer_op {[&]() { _is_fragment_instance_closed = true; }}; _runtime_profile->total_time_counter()->update(_fragment_watcher.elapsed_time()); - _runtime_state->runtime_profile()->total_time_counter()->update( - _fragment_watcher.elapsed_time()); static_cast(send_report(true)); if (_runtime_state->enable_profile()) { std::stringstream ss; @@ -938,9 +1519,15 @@ void PipelineFragmentContext::_close_fragment_instance() { _runtime_state->load_channel_profile()->pretty_print(&ss); } - LOG_INFO("Query {} fragment {} instance {} profile:\n {}", print_id(this->_query_id), - this->_fragment_id, print_id(this->get_fragment_instance_id()), ss.str()); + LOG_INFO("Query {} fragment {} profile:\n {}", print_id(this->_query_id), + this->_fragment_id, ss.str()); } + + if (_query_ctx->enable_profile()) { + _query_ctx->add_fragment_profile_x(_fragment_id, collect_realtime_profile_x(), + collect_realtime_load_channel_profile_x()); + } + // all submitted tasks done _exec_env->fragment_mgr()->remove_pipeline_context( std::dynamic_pointer_cast(shared_from_this())); @@ -976,31 +1563,100 @@ Status PipelineFragmentContext::send_report(bool done) { return Status::NeedSendAgain(""); } + std::vector runtime_states; + + for (auto& task_state : _task_runtime_states) { + runtime_states.push_back(task_state.get()); + } + + ReportStatusRequest req {true, + exec_status, + runtime_states, + nullptr, + _runtime_state->load_channel_profile(), + done || !exec_status.ok(), + _query_ctx->coord_addr, + _query_id, + _fragment_id, + TUniqueId(), + -1, + _runtime_state.get(), + [this](Status st) { return update_status(st); }, + [this](const PPlanFragmentCancelReason& reason, + const std::string& msg) { cancel(reason, msg); }}; + return _report_status_cb( - {false, - exec_status, - {}, - _runtime_state->enable_profile() ? _runtime_state->runtime_profile() : nullptr, - _runtime_state->enable_profile() ? _runtime_state->load_channel_profile() : nullptr, - done || !exec_status.ok(), - _query_ctx->coord_addr, - _query_id, - _fragment_id, - _fragment_instance_id, - _backend_num, - _runtime_state.get(), - [this](Status st) { return update_status(st); }, - [this](const PPlanFragmentCancelReason& reason, const std::string& msg) { - cancel(reason, msg); - }}, - std::dynamic_pointer_cast(shared_from_this())); + req, std::dynamic_pointer_cast(shared_from_this())); } std::string PipelineFragmentContext::debug_string() { fmt::memory_buffer debug_string_buffer; - fmt::format_to(debug_string_buffer, "PipelineFragmentContext Info: QueryId = {}\n", - print_id(_query_ctx->query_id())); + fmt::format_to(debug_string_buffer, "PipelineFragmentContext Info:\n"); + for (size_t j = 0; j < _tasks.size(); j++) { + fmt::format_to(debug_string_buffer, "Tasks in instance {}:\n", j); + for (size_t i = 0; i < _tasks[j].size(); i++) { + fmt::format_to(debug_string_buffer, "Task {}: {}\n", i, _tasks[j][i]->debug_string()); + } + } + return fmt::to_string(debug_string_buffer); } +std::vector> +PipelineFragmentContext::collect_realtime_profile_x() const { + std::vector> res; + DCHECK(_query_ctx->enable_pipeline_x_exec() == true) + << fmt::format("Query {} calling a pipeline X function, but its pipeline X is disabled", + print_id(this->_query_id)); + + // we do not have mutex to protect pipeline_id_to_profile + // so we need to make sure this funciton is invoked after fragment context + // has already been prepared. + if (!this->_prepared) { + std::string msg = + "Query " + print_id(this->_query_id) + " collecting profile, but its not prepared"; + DCHECK(false) << msg; + LOG_ERROR(msg); + return res; + } + + // pipeline_id_to_profile is initialized in prepare stage + for (auto& pipeline_profile : _runtime_state->pipeline_id_to_profile()) { + auto profile_ptr = std::make_shared(); + pipeline_profile->to_thrift(profile_ptr.get()); + res.push_back(profile_ptr); + } + + return res; +} + +std::shared_ptr +PipelineFragmentContext::collect_realtime_load_channel_profile_x() const { + // we do not have mutex to protect pipeline_id_to_profile + // so we need to make sure this funciton is invoked after fragment context + // has already been prepared. + if (!this->_prepared) { + std::string msg = + "Query " + print_id(this->_query_id) + " collecting profile, but its not prepared"; + DCHECK(false) << msg; + LOG_ERROR(msg); + return nullptr; + } + + for (auto& runtime_state : _task_runtime_states) { + if (runtime_state->runtime_profile() == nullptr) { + continue; + } + + auto tmp_load_channel_profile = std::make_shared(); + + runtime_state->runtime_profile()->to_thrift(tmp_load_channel_profile.get()); + this->_runtime_state->load_channel_profile()->update(*tmp_load_channel_profile); + } + + auto load_channel_profile = std::make_shared(); + this->_runtime_state->load_channel_profile()->to_thrift(load_channel_profile.get()); + return load_channel_profile; +} + } // namespace doris::pipeline diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h index b9bfcb28f68af2..0c3af6733bccba 100644 --- a/be/src/pipeline/pipeline_fragment_context.h +++ b/be/src/pipeline/pipeline_fragment_context.h @@ -31,7 +31,10 @@ #include "common/status.h" #include "pipeline/pipeline.h" +#include "pipeline/pipeline_fragment_context.h" #include "pipeline/pipeline_task.h" +#include "pipeline/pipeline_x/local_exchange/local_exchanger.h" +#include "pipeline/pipeline_x/pipeline_x_task.h" #include "runtime/query_context.h" #include "runtime/runtime_state.h" #include "runtime/task_execution_context.h" @@ -39,8 +42,6 @@ #include "util/stopwatch.hpp" namespace doris { -class ExecNode; -class DataSink; struct ReportStatusRequest; class ExecEnv; class RuntimeFilterMergeControllerEntity; @@ -49,6 +50,8 @@ class TPipelineFragmentParams; namespace pipeline { +class Dependency; + class PipelineFragmentContext : public TaskExecutionContext { public: ENABLE_FACTORY_CREATOR(PipelineFragmentContext); @@ -61,13 +64,15 @@ class PipelineFragmentContext : public TaskExecutionContext { using report_status_callback = std::function&&)>; PipelineFragmentContext() = default; - PipelineFragmentContext(const TUniqueId& query_id, const TUniqueId& instance_id, - int fragment_id, int backend_num, + PipelineFragmentContext(const TUniqueId& query_id, const int fragment_id, std::shared_ptr query_ctx, ExecEnv* exec_env, const std::function& call_back, - report_status_callback report_status_cb); + const report_status_callback& report_status_cb); + + ~PipelineFragmentContext(); - ~PipelineFragmentContext() override; + std::vector> collect_realtime_profile_x() const; + std::shared_ptr collect_realtime_load_channel_profile_x() const; bool is_timeout(const VecDateTimeValue& now) const; @@ -75,32 +80,23 @@ class PipelineFragmentContext : public TaskExecutionContext { PipelinePtr add_pipeline(PipelinePtr parent, int idx = -1); - TUniqueId get_fragment_instance_id() const { return _fragment_instance_id; } - RuntimeState* get_runtime_state() { return _runtime_state.get(); } QueryContext* get_query_ctx() { return _query_ctx.get(); } // should be protected by lock? [[nodiscard]] bool is_canceled() const { return _runtime_state->is_cancelled(); } - int32_t next_operator_builder_id() { return _next_operator_builder_id++; } - - Status prepare(const doris::TPipelineFragmentParams& request, size_t idx); - - virtual Status prepare(const doris::TPipelineFragmentParams& request) { - return Status::InternalError("Pipeline fragment context do not implement prepare"); - } + Status prepare(const doris::TPipelineFragmentParams& request); - virtual Status submit(); + Status submit(); - virtual void close_if_prepare_failed(Status st); - virtual void close_sink(); + void close_if_prepare_failed(Status st); + void close_sink(); void set_is_report_success(bool is_report_success) { _is_report_success = is_report_success; } - virtual void cancel( - const PPlanFragmentCancelReason& reason = PPlanFragmentCancelReason::INTERNAL_ERROR, - const std::string& msg = ""); + void cancel(const PPlanFragmentCancelReason& reason = PPlanFragmentCancelReason::INTERNAL_ERROR, + const std::string& msg = ""); // TODO: Support pipeline runtime filter @@ -110,10 +106,7 @@ class PipelineFragmentContext : public TaskExecutionContext { void close_a_pipeline(); - virtual void add_merge_controller_handler( - std::shared_ptr& handler) {} - - virtual Status send_report(bool); + Status send_report(bool); Status update_status(Status status) { std::lock_guard l(_status_lock); @@ -124,36 +117,94 @@ class PipelineFragmentContext : public TaskExecutionContext { } void trigger_report_if_necessary(); - virtual void instance_ids(std::vector& ins_ids) const { - ins_ids.resize(1); - ins_ids[0] = _fragment_instance_id; - } - virtual void instance_ids(std::vector& ins_ids) const { - ins_ids.resize(1); - ins_ids[0] = print_id(_fragment_instance_id); - } void refresh_next_report_time(); - virtual std::string debug_string(); + std::string debug_string(); uint64_t create_time() const { return _create_time; } -protected: - Status _create_sink(int sender_id, const TDataSink& t_data_sink, RuntimeState* state); - Status _build_pipelines(ExecNode*, PipelinePtr); - virtual Status _build_pipeline_tasks(const doris::TPipelineFragmentParams& request); + [[nodiscard]] int next_operator_id() { return _operator_id--; } + + [[nodiscard]] int max_operator_id() const { return _operator_id; } + + [[nodiscard]] int next_sink_operator_id() { return _sink_operator_id--; } + + [[nodiscard]] int max_sink_operator_id() const { return _sink_operator_id; } + + void instance_ids(std::vector& ins_ids) const { + ins_ids.resize(_fragment_instance_ids.size()); + for (size_t i = 0; i < _fragment_instance_ids.size(); i++) { + ins_ids[i] = _fragment_instance_ids[i]; + } + } + + void instance_ids(std::vector& ins_ids) const { + ins_ids.resize(_fragment_instance_ids.size()); + for (size_t i = 0; i < _fragment_instance_ids.size(); i++) { + ins_ids[i] = print_id(_fragment_instance_ids[i]); + } + } + + void add_merge_controller_handler( + std::shared_ptr& handler) { + _merge_controller_handlers.emplace_back(handler); + } + +private: + Status _build_pipelines(ObjectPool* pool, const doris::TPipelineFragmentParams& request, + const DescriptorTbl& descs, OperatorXPtr* root, PipelinePtr cur_pipe); + Status _create_tree_helper(ObjectPool* pool, const std::vector& tnodes, + const doris::TPipelineFragmentParams& request, + const DescriptorTbl& descs, OperatorXPtr parent, int* node_idx, + OperatorXPtr* root, PipelinePtr& cur_pipe, int child_idx); + + Status _create_operator(ObjectPool* pool, const TPlanNode& tnode, + const doris::TPipelineFragmentParams& request, + const DescriptorTbl& descs, OperatorXPtr& op, PipelinePtr& cur_pipe, + int parent_idx, int child_idx); template - Status _build_operators_for_set_operation_node(ExecNode*, PipelinePtr); - virtual void _close_fragment_instance(); + Status _build_operators_for_set_operation_node(ObjectPool* pool, const TPlanNode& tnode, + const DescriptorTbl& descs, OperatorXPtr& op, + PipelinePtr& cur_pipe, int parent_idx, + int child_idx); + + Status _create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink, + const std::vector& output_exprs, + const TPipelineFragmentParams& params, const RowDescriptor& row_desc, + RuntimeState* state, DescriptorTbl& desc_tbl, + PipelineId cur_pipeline_id); + Status _plan_local_exchange(int num_buckets, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx); + Status _plan_local_exchange(int num_buckets, int pip_idx, PipelinePtr pip, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx, + const bool ignore_data_distribution); + void _inherit_pipeline_properties(const DataDistribution& data_distribution, + PipelinePtr pipe_with_source, PipelinePtr pipe_with_sink); + Status _add_local_exchange(int pip_idx, int idx, int node_id, ObjectPool* pool, + PipelinePtr cur_pipe, DataDistribution data_distribution, + bool* do_local_exchange, int num_buckets, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx, + const bool ignore_data_distribution); + Status _add_local_exchange_impl(int idx, ObjectPool* pool, PipelinePtr cur_pipe, + PipelinePtr new_pip, DataDistribution data_distribution, + bool* do_local_exchange, int num_buckets, + const std::map& bucket_seq_to_instance_idx, + const std::map& shuffle_idx_to_instance_idx, + const bool ignore_data_hash_distribution); + + bool _enable_local_shuffle() const { return _runtime_state->enable_local_shuffle(); } + + Status _build_pipeline_tasks(const doris::TPipelineFragmentParams& request); + void _close_fragment_instance(); void _init_next_report_time(); // Id of this query TUniqueId _query_id; - TUniqueId _fragment_instance_id; int _fragment_id; - int _backend_num; - ExecEnv* _exec_env = nullptr; std::atomic_bool _prepared = false; @@ -169,27 +220,16 @@ class PipelineFragmentContext : public TaskExecutionContext { // When submit fail, `_total_tasks` is equal to the number of tasks submitted. int _total_tasks = 0; - int32_t _next_operator_builder_id = 10000; - - PipelinePtr _root_pipeline; - std::unique_ptr _runtime_profile; bool _is_report_success = false; std::unique_ptr _runtime_state; - ExecNode* _root_plan = nullptr; // lives in _runtime_state->obj_pool() - // TODO: remove the _sink and _multi_cast_stream_sink_senders to set both - // of it in pipeline task not the fragment_context - std::unique_ptr _sink; - std::vector> _multi_cast_stream_sink_senders; - std::shared_ptr _query_ctx; QueryThreadContext _query_thread_context; MonotonicStopWatch _fragment_watcher; - RuntimeProfile::Counter* _start_timer = nullptr; RuntimeProfile::Counter* _prepare_timer = nullptr; std::function _call_back; @@ -212,10 +252,78 @@ class PipelineFragmentContext : public TaskExecutionContext { VecDateTimeValue _start_time; int _timeout = -1; -private: - std::vector> _tasks; + OperatorXPtr _root_op = nullptr; + // this is a [n * m] matrix. n is parallelism of pipeline engine and m is the number of pipelines. + std::vector>> _tasks; + + bool _need_local_merge = false; + + // It is used to manage the lifecycle of RuntimeFilterMergeController + std::vector> _merge_controller_handlers; + + // TODO: remove the _sink and _multi_cast_stream_sink_senders to set both + // of it in pipeline task not the fragment_context +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif + DataSinkOperatorXPtr _sink = nullptr; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + + // `_dag` manage dependencies between pipelines by pipeline ID. the indices will be blocked by members + std::map> _dag; + + // We use preorder traversal to create an operator tree. When we meet a join node, we should + // build probe operator and build operator in separate pipelines. To do this, we should build + // ProbeSide first, and use `_pipelines_to_build` to store which pipeline the build operator + // is in, so we can build BuildSide once we complete probe side. + struct pipeline_parent_map { + std::map> _build_side_pipelines; + void push(int parent_node_id, PipelinePtr pipeline) { + if (!_build_side_pipelines.contains(parent_node_id)) { + _build_side_pipelines.insert({parent_node_id, {pipeline}}); + } else { + _build_side_pipelines[parent_node_id].push_back(pipeline); + } + } + void pop(PipelinePtr& cur_pipe, int parent_node_id, int child_idx) { + if (!_build_side_pipelines.contains(parent_node_id)) { + return; + } + DCHECK(_build_side_pipelines.contains(parent_node_id)); + auto& child_pipeline = _build_side_pipelines[parent_node_id]; + DCHECK(child_idx < child_pipeline.size()); + cur_pipe = child_pipeline[child_idx]; + } + void clear() { _build_side_pipelines.clear(); } + } _pipeline_parent_map; + + std::mutex _state_map_lock; + + int _operator_id = 0; + int _sink_operator_id = 0; + std::map, std::shared_ptr>> + _op_id_to_le_state; + + // UniqueId -> runtime mgr + std::map> _runtime_filter_mgr_map; + + //Here are two types of runtime states: + // - _runtime state is at the Fragment level. + // - _task_runtime_states is at the task level, unique to each task. + + std::vector _fragment_instance_ids; + // Local runtime states for each task + std::vector> _task_runtime_states; + + std::vector> _runtime_filter_states; + // Total instance num running on all BEs + int _total_instances = -1; uint64_t _create_time; + bool _require_bucket_distribution = false; }; } // namespace pipeline } // namespace doris \ No newline at end of file diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp index f31a39df31a79d..167ff01fe695ed 100644 --- a/be/src/pipeline/pipeline_task.cpp +++ b/be/src/pipeline/pipeline_task.cpp @@ -391,39 +391,7 @@ void PipelineTask::set_state(PipelineTaskState state) { } std::string PipelineTask::debug_string() { - fmt::memory_buffer debug_string_buffer; - - fmt::format_to(debug_string_buffer, "QueryId: {}\n", print_id(query_context()->query_id())); - fmt::format_to(debug_string_buffer, "InstanceId: {}\n", - print_id(fragment_context()->get_fragment_instance_id())); - - fmt::format_to(debug_string_buffer, "RuntimeUsage: {}\n", - PrettyPrinter::print(get_runtime_ns(), TUnit::TIME_NS)); - { - std::stringstream profile_ss; - _fresh_profile_counter(); - _task_profile->pretty_print(&profile_ss, ""); - fmt::format_to(debug_string_buffer, "Profile: {}\n", profile_ss.str()); - } - fmt::format_to(debug_string_buffer, - "PipelineTask[this = {}, state = {}]\noperators: ", (void*)this, - get_state_name(_cur_state)); - for (size_t i = 0; i < _operators.size(); i++) { - fmt::format_to(debug_string_buffer, "\n{}{}", std::string(i * 2, ' '), - _operators[i]->debug_string()); - std::stringstream profile_ss; - _operators[i]->get_runtime_profile()->pretty_print(&profile_ss, std::string(i * 2, ' ')); - fmt::format_to(debug_string_buffer, "\n{}", profile_ss.str()); - } - fmt::format_to(debug_string_buffer, "\n{}{}", std::string(_operators.size() * 2, ' '), - _sink->debug_string()); - { - std::stringstream profile_ss; - _sink->get_runtime_profile()->pretty_print(&profile_ss, - std::string(_operators.size() * 2, ' ')); - fmt::format_to(debug_string_buffer, "\n{}", profile_ss.str()); - } - return fmt::to_string(debug_string_buffer); + return ""; } } // namespace doris::pipeline diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h index b9a5cb06ff872d..aabfc91af37a39 100644 --- a/be/src/pipeline/pipeline_task.h +++ b/be/src/pipeline/pipeline_task.h @@ -254,8 +254,6 @@ class PipelineTask { void set_parent_profile(RuntimeProfile* profile) { _parent_profile = profile; } - virtual bool is_pipelineX() const { return false; } - bool is_running() { return _running.load(); } void set_running(bool running) { _running = running; } diff --git a/be/src/pipeline/pipeline_x/operator.cpp b/be/src/pipeline/pipeline_x/operator.cpp index d5afad15fa0ec0..8b1fb15857107a 100644 --- a/be/src/pipeline/pipeline_x/operator.cpp +++ b/be/src/pipeline/pipeline_x/operator.cpp @@ -70,8 +70,6 @@ #include "pipeline/exec/spill_sort_sink_operator.h" #include "pipeline/exec/spill_sort_source_operator.h" #include "pipeline/exec/streaming_aggregation_operator.h" -#include "pipeline/exec/streaming_aggregation_sink_operator.h" -#include "pipeline/exec/streaming_aggregation_source_operator.h" #include "pipeline/exec/table_function_operator.h" #include "pipeline/exec/union_sink_operator.h" #include "pipeline/exec/union_source_operator.h" diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp deleted file mode 100644 index fc0234c62904b6..00000000000000 --- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp +++ /dev/null @@ -1,1522 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "pipeline_x_fragment_context.h" - -#include -#include -#include -#include -#include -#include -#include - -// IWYU pragma: no_include -#include // IWYU pragma: keep -#include -#include -#include -#include - -#include "cloud/config.h" -#include "common/config.h" -#include "common/logging.h" -#include "exec/data_sink.h" -#include "exec/exec_node.h" -#include "exec/scan_node.h" -#include "io/fs/stream_load_pipe.h" -#include "pipeline/exec/aggregation_sink_operator.h" -#include "pipeline/exec/aggregation_source_operator.h" -#include "pipeline/exec/analytic_sink_operator.h" -#include "pipeline/exec/analytic_source_operator.h" -#include "pipeline/exec/assert_num_rows_operator.h" -#include "pipeline/exec/datagen_operator.h" -#include "pipeline/exec/distinct_streaming_aggregation_operator.h" -#include "pipeline/exec/empty_set_operator.h" -#include "pipeline/exec/es_scan_operator.h" -#include "pipeline/exec/exchange_sink_operator.h" -#include "pipeline/exec/exchange_source_operator.h" -#include "pipeline/exec/file_scan_operator.h" -#include "pipeline/exec/group_commit_block_sink_operator.h" -#include "pipeline/exec/hashjoin_build_sink.h" -#include "pipeline/exec/hashjoin_probe_operator.h" -#include "pipeline/exec/hive_table_sink_operator.h" -#include "pipeline/exec/jdbc_scan_operator.h" -#include "pipeline/exec/jdbc_table_sink_operator.h" -#include "pipeline/exec/meta_scan_operator.h" -#include "pipeline/exec/multi_cast_data_stream_sink.h" -#include "pipeline/exec/multi_cast_data_stream_source.h" -#include "pipeline/exec/nested_loop_join_build_operator.h" -#include "pipeline/exec/nested_loop_join_probe_operator.h" -#include "pipeline/exec/olap_scan_operator.h" -#include "pipeline/exec/olap_table_sink_operator.h" -#include "pipeline/exec/olap_table_sink_v2_operator.h" -#include "pipeline/exec/partition_sort_sink_operator.h" -#include "pipeline/exec/partition_sort_source_operator.h" -#include "pipeline/exec/partitioned_aggregation_sink_operator.h" -#include "pipeline/exec/partitioned_aggregation_source_operator.h" -#include "pipeline/exec/partitioned_hash_join_probe_operator.h" -#include "pipeline/exec/partitioned_hash_join_sink_operator.h" -#include "pipeline/exec/repeat_operator.h" -#include "pipeline/exec/result_file_sink_operator.h" -#include "pipeline/exec/result_sink_operator.h" -#include "pipeline/exec/schema_scan_operator.h" -#include "pipeline/exec/select_operator.h" -#include "pipeline/exec/set_probe_sink_operator.h" -#include "pipeline/exec/set_sink_operator.h" -#include "pipeline/exec/set_source_operator.h" -#include "pipeline/exec/sort_sink_operator.h" -#include "pipeline/exec/sort_source_operator.h" -#include "pipeline/exec/spill_sort_sink_operator.h" -#include "pipeline/exec/spill_sort_source_operator.h" -#include "pipeline/exec/streaming_aggregation_operator.h" -#include "pipeline/exec/table_function_operator.h" -#include "pipeline/exec/union_sink_operator.h" -#include "pipeline/exec/union_source_operator.h" -#include "pipeline/pipeline_x/local_exchange/local_exchange_sink_operator.h" -#include "pipeline/pipeline_x/local_exchange/local_exchange_source_operator.h" -#include "pipeline/task_scheduler.h" -#include "runtime/exec_env.h" -#include "runtime/fragment_mgr.h" -#include "runtime/runtime_filter_mgr.h" -#include "runtime/runtime_state.h" -#include "runtime/stream_load/new_load_stream_mgr.h" -#include "runtime/stream_load/stream_load_context.h" -#include "runtime/thread_context.h" -#include "service/backend_options.h" -#include "util/container_util.hpp" -#include "util/debug_util.h" -#include "util/uid_util.h" -#include "vec/runtime/vdata_stream_mgr.h" - -namespace doris::pipeline { - -PipelineXFragmentContext::PipelineXFragmentContext( - const TUniqueId& query_id, const int fragment_id, std::shared_ptr query_ctx, - ExecEnv* exec_env, const std::function& call_back, - const report_status_callback& report_status_cb) - : PipelineFragmentContext(query_id, TUniqueId(), fragment_id, -1, query_ctx, exec_env, - call_back, report_status_cb) {} - -PipelineXFragmentContext::~PipelineXFragmentContext() { - // The memory released by the query end is recorded in the query mem tracker. - SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_thread_context.query_mem_tracker); - auto st = _query_ctx->exec_status(); - _tasks.clear(); - if (!_task_runtime_states.empty()) { - for (auto& runtime_state : _task_runtime_states) { - _call_back(runtime_state.get(), &st); - runtime_state.reset(); - } - } - _runtime_state.reset(); - _runtime_filter_states.clear(); - _runtime_filter_mgr_map.clear(); - _op_id_to_le_state.clear(); -} - -void PipelineXFragmentContext::cancel(const PPlanFragmentCancelReason& reason, - const std::string& msg) { - { - std::lock_guard l(_task_mutex); - if (_closed_tasks == _total_tasks) { - // All tasks in this PipelineXFragmentContext already closed. - return; - } - } - LOG_INFO("PipelineXFragmentContext::cancel") - .tag("query_id", print_id(_query_id)) - .tag("fragment_id", _fragment_id) - .tag("reason", reason) - .tag("error message", msg); - if (reason == PPlanFragmentCancelReason::TIMEOUT) { - LOG(WARNING) << "PipelineXFragmentContext is cancelled due to timeout : " << debug_string(); - } - _query_ctx->cancel(msg, Status::Cancelled(msg), _fragment_id); - if (reason == PPlanFragmentCancelReason::LIMIT_REACH) { - _is_report_on_cancel = false; - } else { - for (auto& id : _fragment_instance_ids) { - LOG(WARNING) << "PipelineXFragmentContext cancel instance: " << print_id(id); - } - } - // Get pipe from new load stream manager and send cancel to it or the fragment may hang to wait read from pipe - // For stream load the fragment's query_id == load id, it is set in FE. - auto stream_load_ctx = _exec_env->new_load_stream_mgr()->get(_query_id); - if (stream_load_ctx != nullptr) { - stream_load_ctx->pipe->cancel(msg); - } - - // Cancel the result queue manager used by spark doris connector - // TODO pipeline incomp - // _exec_env->result_queue_mgr()->update_queue_status(id, Status::Aborted(msg)); - for (auto& tasks : _tasks) { - for (auto& task : tasks) { - if (task->is_finished()) { - continue; - } - task->clear_blocking_state(); - } - } -} - -Status PipelineXFragmentContext::prepare(const doris::TPipelineFragmentParams& request) { - if (_prepared) { - return Status::InternalError("Already prepared"); - } - _num_instances = request.local_params.size(); - _total_instances = request.__isset.total_instances ? request.total_instances : _num_instances; - _runtime_profile = std::make_unique("PipelineContext"); - _prepare_timer = ADD_TIMER(_runtime_profile, "PrepareTime"); - SCOPED_TIMER(_prepare_timer); - - auto* fragment_context = this; - - LOG_INFO("PipelineXFragmentContext::prepare") - .tag("query_id", print_id(_query_id)) - .tag("fragment_id", _fragment_id) - .tag("pthread_id", (uintptr_t)pthread_self()); - - if (request.query_options.__isset.is_report_success) { - fragment_context->set_is_report_success(request.query_options.is_report_success); - } - - // 1. Set up the global runtime state. - _runtime_state = RuntimeState::create_unique(request.query_id, request.fragment_id, - request.query_options, _query_ctx->query_globals, - _exec_env, _query_ctx.get()); - - SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_runtime_state->query_mem_tracker()); - if (request.__isset.backend_id) { - _runtime_state->set_backend_id(request.backend_id); - } - if (request.__isset.import_label) { - _runtime_state->set_import_label(request.import_label); - } - if (request.__isset.db_name) { - _runtime_state->set_db_name(request.db_name); - } - if (request.__isset.load_job_id) { - _runtime_state->set_load_job_id(request.load_job_id); - } - - if (request.is_simplified_param) { - _desc_tbl = _query_ctx->desc_tbl; - } else { - DCHECK(request.__isset.desc_tbl); - RETURN_IF_ERROR( - DescriptorTbl::create(_runtime_state->obj_pool(), request.desc_tbl, &_desc_tbl)); - } - _runtime_state->set_desc_tbl(_desc_tbl); - _runtime_state->set_num_per_fragment_instances(request.num_senders); - _runtime_state->set_load_stream_per_node(request.load_stream_per_node); - _runtime_state->set_total_load_streams(request.total_load_streams); - _runtime_state->set_num_local_sink(request.num_local_sink); - - const auto& local_params = request.local_params[0]; - if (local_params.__isset.runtime_filter_params) { - _query_ctx->runtime_filter_mgr()->set_runtime_filter_params( - local_params.runtime_filter_params); - } - if (local_params.__isset.topn_filter_source_node_ids) { - _query_ctx->init_runtime_predicates(local_params.topn_filter_source_node_ids); - } else { - _query_ctx->init_runtime_predicates({0}); - } - - _need_local_merge = request.__isset.parallel_instances; - - // 2. Build pipelines with operators in this fragment. - auto root_pipeline = add_pipeline(); - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_build_pipelines( - _runtime_state->obj_pool(), request, *_query_ctx->desc_tbl, &_root_op, root_pipeline)); - - // 3. Create sink operator - if (!request.fragment.__isset.output_sink) { - return Status::InternalError("No output sink in this fragment!"); - } - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_create_data_sink( - _runtime_state->obj_pool(), request.fragment.output_sink, request.fragment.output_exprs, - request, root_pipeline->output_row_desc(), _runtime_state.get(), *_desc_tbl, - root_pipeline->id())); - RETURN_IF_ERROR(_sink->init(request.fragment.output_sink)); - RETURN_IF_ERROR(root_pipeline->set_sink(_sink)); - - for (PipelinePtr& pipeline : _pipelines) { - DCHECK(pipeline->sink_x() != nullptr) << pipeline->operator_xs().size(); - RETURN_IF_ERROR(pipeline->sink_x()->set_child(pipeline->operator_xs().back())); - } - if (_enable_local_shuffle()) { - RETURN_IF_ERROR(_plan_local_exchange(request.num_buckets, - request.bucket_seq_to_instance_idx, - request.shuffle_idx_to_instance_idx)); - } - - // 4. Initialize global states in pipelines. - for (PipelinePtr& pipeline : _pipelines) { - pipeline->children().clear(); - RETURN_IF_ERROR(pipeline->prepare(_runtime_state.get())); - } - - // 5. Build pipeline tasks and initialize local state. - RETURN_IF_ERROR(_build_pipeline_tasks(request)); - - _init_next_report_time(); - - _prepared = true; - return Status::OK(); -} - -Status PipelineXFragmentContext::_plan_local_exchange( - int num_buckets, const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx) { - for (int pip_idx = _pipelines.size() - 1; pip_idx >= 0; pip_idx--) { - _pipelines[pip_idx]->init_data_distribution(); - // Set property if child pipeline is not join operator's child. - if (!_pipelines[pip_idx]->children().empty()) { - for (auto& child : _pipelines[pip_idx]->children()) { - if (child->sink_x()->node_id() == - _pipelines[pip_idx]->operator_xs().front()->node_id()) { - RETURN_IF_ERROR(_pipelines[pip_idx]->operator_xs().front()->set_child( - child->operator_xs().back())); - _pipelines[pip_idx]->set_data_distribution(child->data_distribution()); - } - } - } - - RETURN_IF_ERROR(_plan_local_exchange( - _pipelines[pip_idx]->operator_xs().front()->ignore_data_hash_distribution() - ? _num_instances - : num_buckets, - pip_idx, _pipelines[pip_idx], bucket_seq_to_instance_idx, - shuffle_idx_to_instance_idx, - _pipelines[pip_idx]->operator_xs().front()->ignore_data_hash_distribution())); - } - return Status::OK(); -} - -Status PipelineXFragmentContext::_plan_local_exchange( - int num_buckets, int pip_idx, PipelinePtr pip, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx, - const bool ignore_data_hash_distribution) { - int idx = 1; - bool do_local_exchange = false; - do { - auto& ops = pip->operator_xs(); - do_local_exchange = false; - // Plan local exchange for each operator. - for (; idx < ops.size();) { - if (ops[idx]->required_data_distribution().need_local_exchange()) { - RETURN_IF_ERROR(_add_local_exchange( - pip_idx, idx, ops[idx]->node_id(), _runtime_state->obj_pool(), pip, - ops[idx]->required_data_distribution(), &do_local_exchange, num_buckets, - bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, - ignore_data_hash_distribution)); - } - if (do_local_exchange) { - // If local exchange is needed for current operator, we will split this pipeline to - // two pipelines by local exchange sink/source. And then we need to process remaining - // operators in this pipeline so we set idx to 2 (0 is local exchange source and 1 - // is current operator was already processed) and continue to plan local exchange. - idx = 2; - break; - } - idx++; - } - } while (do_local_exchange); - if (pip->sink_x()->required_data_distribution().need_local_exchange()) { - RETURN_IF_ERROR(_add_local_exchange( - pip_idx, idx, pip->sink_x()->node_id(), _runtime_state->obj_pool(), pip, - pip->sink_x()->required_data_distribution(), &do_local_exchange, num_buckets, - bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, - ignore_data_hash_distribution)); - } - return Status::OK(); -} - -Status PipelineXFragmentContext::_create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink, - const std::vector& output_exprs, - const TPipelineFragmentParams& params, - const RowDescriptor& row_desc, - RuntimeState* state, DescriptorTbl& desc_tbl, - PipelineId cur_pipeline_id) { - switch (thrift_sink.type) { - case TDataSinkType::DATA_STREAM_SINK: { - if (!thrift_sink.__isset.stream_sink) { - return Status::InternalError("Missing data stream sink."); - } - _sink.reset(new ExchangeSinkOperatorX(state, row_desc, next_sink_operator_id(), - thrift_sink.stream_sink, params.destinations)); - break; - } - case TDataSinkType::RESULT_SINK: { - if (!thrift_sink.__isset.result_sink) { - return Status::InternalError("Missing data buffer sink."); - } - - // TODO: figure out good buffer size based on size of output row - _sink.reset(new ResultSinkOperatorX(next_sink_operator_id(), row_desc, output_exprs, - thrift_sink.result_sink)); - break; - } - case TDataSinkType::GROUP_COMMIT_OLAP_TABLE_SINK: - case TDataSinkType::OLAP_TABLE_SINK: { - if (state->query_options().enable_memtable_on_sink_node && - !_has_inverted_index_or_partial_update(thrift_sink.olap_table_sink) && - !config::is_cloud_mode()) { - _sink.reset(new OlapTableSinkV2OperatorX(pool, next_sink_operator_id(), row_desc, - output_exprs)); - } else { - _sink.reset(new OlapTableSinkOperatorX(pool, next_sink_operator_id(), row_desc, - output_exprs)); - } - break; - } - case TDataSinkType::GROUP_COMMIT_BLOCK_SINK: { - DCHECK(thrift_sink.__isset.olap_table_sink); - _sink.reset(new GroupCommitBlockSinkOperatorX(next_sink_operator_id(), row_desc)); - break; - } - case TDataSinkType::HIVE_TABLE_SINK: { - if (!thrift_sink.__isset.hive_table_sink) { - return Status::InternalError("Missing hive table sink."); - } - _sink.reset( - new HiveTableSinkOperatorX(pool, next_sink_operator_id(), row_desc, output_exprs)); - break; - } - case TDataSinkType::JDBC_TABLE_SINK: { - if (!thrift_sink.__isset.jdbc_table_sink) { - return Status::InternalError("Missing data jdbc sink."); - } - if (config::enable_java_support) { - _sink.reset( - new JdbcTableSinkOperatorX(row_desc, next_sink_operator_id(), output_exprs)); - } else { - return Status::InternalError( - "Jdbc table sink is not enabled, you can change be config " - "enable_java_support to true and restart be."); - } - break; - } - case TDataSinkType::RESULT_FILE_SINK: { - if (!thrift_sink.__isset.result_file_sink) { - return Status::InternalError("Missing result file sink."); - } - - // TODO: figure out good buffer size based on size of output row - // Result file sink is not the top sink - if (params.__isset.destinations && !params.destinations.empty()) { - _sink.reset(new ResultFileSinkOperatorX(next_sink_operator_id(), row_desc, - thrift_sink.result_file_sink, - params.destinations, output_exprs, desc_tbl)); - } else { - _sink.reset( - new ResultFileSinkOperatorX(next_sink_operator_id(), row_desc, output_exprs)); - } - break; - } - case TDataSinkType::MULTI_CAST_DATA_STREAM_SINK: { - DCHECK(thrift_sink.__isset.multi_cast_stream_sink); - DCHECK_GT(thrift_sink.multi_cast_stream_sink.sinks.size(), 0); - // TODO: figure out good buffer size based on size of output row - auto sink_id = next_sink_operator_id(); - auto sender_size = thrift_sink.multi_cast_stream_sink.sinks.size(); - // one sink has multiple sources. - std::vector sources; - for (int i = 0; i < sender_size; ++i) { - auto source_id = next_operator_id(); - sources.push_back(source_id); - } - - _sink.reset(new MultiCastDataStreamSinkOperatorX( - sink_id, sources, thrift_sink.multi_cast_stream_sink.sinks.size(), pool, - thrift_sink.multi_cast_stream_sink, row_desc)); - for (int i = 0; i < sender_size; ++i) { - auto new_pipeline = add_pipeline(); - RowDescriptor* _row_desc = nullptr; - { - const auto& tmp_row_desc = - !thrift_sink.multi_cast_stream_sink.sinks[i].output_exprs.empty() - ? RowDescriptor(state->desc_tbl(), - {thrift_sink.multi_cast_stream_sink.sinks[i] - .output_tuple_id}, - {false}) - : _sink->row_desc(); - _row_desc = pool->add(new RowDescriptor(tmp_row_desc)); - } - auto source_id = sources[i]; - OperatorXPtr source_op; - // 1. create and set the source operator of multi_cast_data_stream_source for new pipeline - source_op.reset(new MultiCastDataStreamerSourceOperatorX( - i, pool, thrift_sink.multi_cast_stream_sink.sinks[i], row_desc, source_id)); - RETURN_IF_ERROR(new_pipeline->add_operator(source_op)); - // 2. create and set sink operator of data stream sender for new pipeline - - DataSinkOperatorXPtr sink_op; - sink_op.reset( - new ExchangeSinkOperatorX(state, *_row_desc, next_sink_operator_id(), - thrift_sink.multi_cast_stream_sink.sinks[i], - thrift_sink.multi_cast_stream_sink.destinations[i])); - - RETURN_IF_ERROR(new_pipeline->set_sink(sink_op)); - { - TDataSink* t = pool->add(new TDataSink()); - t->stream_sink = thrift_sink.multi_cast_stream_sink.sinks[i]; - RETURN_IF_ERROR(sink_op->init(*t)); - } - - // 3. set dependency dag - _dag[new_pipeline->id()].push_back(cur_pipeline_id); - } - if (sources.empty()) { - return Status::InternalError("size of sources must be greater than 0"); - } - break; - } - default: - return Status::InternalError("Unsuported sink type in pipeline: {}", thrift_sink.type); - } - return Status::OK(); -} - -Status PipelineXFragmentContext::_build_pipeline_tasks( - const doris::TPipelineFragmentParams& request) { - _total_tasks = 0; - int target_size = request.local_params.size(); - _tasks.resize(target_size); - auto& pipeline_id_to_profile = _runtime_state->pipeline_id_to_profile(); - DCHECK(pipeline_id_to_profile.empty()); - pipeline_id_to_profile.resize(_pipelines.size()); - { - size_t pip_idx = 0; - for (auto& pipeline_profile : pipeline_id_to_profile) { - pipeline_profile = - std::make_unique("Pipeline : " + std::to_string(pip_idx)); - pip_idx++; - } - } - - for (size_t i = 0; i < target_size; i++) { - const auto& local_params = request.local_params[i]; - auto fragment_instance_id = local_params.fragment_instance_id; - _fragment_instance_ids.push_back(fragment_instance_id); - std::unique_ptr runtime_filter_mgr; - auto init_runtime_state = [&](std::unique_ptr& runtime_state) { - runtime_state->set_query_mem_tracker(_query_ctx->query_mem_tracker); - - runtime_state->set_task_execution_context(shared_from_this()); - runtime_state->set_be_number(local_params.backend_num); - - if (request.__isset.backend_id) { - runtime_state->set_backend_id(request.backend_id); - } - if (request.__isset.import_label) { - runtime_state->set_import_label(request.import_label); - } - if (request.__isset.db_name) { - runtime_state->set_db_name(request.db_name); - } - if (request.__isset.load_job_id) { - runtime_state->set_load_job_id(request.load_job_id); - } - - runtime_state->set_desc_tbl(_desc_tbl); - runtime_state->set_per_fragment_instance_idx(local_params.sender_id); - runtime_state->set_num_per_fragment_instances(request.num_senders); - runtime_state->resize_op_id_to_local_state(max_operator_id()); - runtime_state->set_max_operator_id(max_operator_id()); - runtime_state->set_load_stream_per_node(request.load_stream_per_node); - runtime_state->set_total_load_streams(request.total_load_streams); - runtime_state->set_num_local_sink(request.num_local_sink); - DCHECK(runtime_filter_mgr); - runtime_state->set_pipeline_x_runtime_filter_mgr(runtime_filter_mgr.get()); - }; - - auto filterparams = std::make_unique(); - - { - filterparams->runtime_filter_wait_infinitely = - _runtime_state->runtime_filter_wait_infinitely(); - filterparams->runtime_filter_wait_time_ms = - _runtime_state->runtime_filter_wait_time_ms(); - filterparams->enable_pipeline_exec = _runtime_state->enable_pipeline_x_exec(); - filterparams->execution_timeout = _runtime_state->execution_timeout(); - - filterparams->exec_env = ExecEnv::GetInstance(); - filterparams->query_id.set_hi(_runtime_state->query_id().hi); - filterparams->query_id.set_lo(_runtime_state->query_id().lo); - - filterparams->be_exec_version = _runtime_state->be_exec_version(); - filterparams->query_ctx = _query_ctx.get(); - } - - // build local_runtime_filter_mgr for each instance - runtime_filter_mgr = std::make_unique( - request.query_id, filterparams.get(), _query_ctx->query_mem_tracker); - - filterparams->runtime_filter_mgr = runtime_filter_mgr.get(); - - _runtime_filter_states.push_back(std::move(filterparams)); - std::map pipeline_id_to_task; - auto get_local_exchange_state = [&](PipelinePtr pipeline) - -> std::map, - std::shared_ptr>> { - std::map, - std::shared_ptr>> - le_state_map; - auto source_id = pipeline->operator_xs().front()->operator_id(); - if (auto iter = _op_id_to_le_state.find(source_id); iter != _op_id_to_le_state.end()) { - le_state_map.insert({source_id, iter->second}); - } - for (auto sink_to_source_id : pipeline->sink_x()->dests_id()) { - if (auto iter = _op_id_to_le_state.find(sink_to_source_id); - iter != _op_id_to_le_state.end()) { - le_state_map.insert({sink_to_source_id, iter->second}); - } - } - return le_state_map; - }; - auto get_task_runtime_state = [&](int task_id) -> RuntimeState* { - DCHECK(_task_runtime_states[task_id]); - return _task_runtime_states[task_id].get(); - }; - for (size_t pip_idx = 0; pip_idx < _pipelines.size(); pip_idx++) { - auto& pipeline = _pipelines[pip_idx]; - if (pipeline->need_to_create_task()) { - // build task runtime state - _task_runtime_states.push_back(RuntimeState::create_unique( - this, local_params.fragment_instance_id, request.query_id, - request.fragment_id, request.query_options, _query_ctx->query_globals, - _exec_env, _query_ctx.get())); - auto& task_runtime_state = _task_runtime_states.back(); - init_runtime_state(task_runtime_state); - auto cur_task_id = _total_tasks++; - task_runtime_state->set_task_id(cur_task_id); - task_runtime_state->set_task_num(pipeline->num_tasks()); - auto task = std::make_unique( - pipeline, cur_task_id, get_task_runtime_state(cur_task_id), this, - pipeline_id_to_profile[pip_idx].get(), get_local_exchange_state(pipeline), - i); - pipeline_id_to_task.insert({pipeline->id(), task.get()}); - _tasks[i].emplace_back(std::move(task)); - } - } - - /** - * Build DAG for pipeline tasks. - * For example, we have - * - * ExchangeSink (Pipeline1) JoinBuildSink (Pipeline2) - * \ / - * JoinProbeOperator1 (Pipeline1) JoinBuildSink (Pipeline3) - * \ / - * JoinProbeOperator2 (Pipeline1) - * - * In this fragment, we have three pipelines and pipeline 1 depends on pipeline 2 and pipeline 3. - * To build this DAG, `_dag` manage dependencies between pipelines by pipeline ID and - * `pipeline_id_to_task` is used to find the task by a unique pipeline ID. - * - * Finally, we have two upstream dependencies in Pipeline1 corresponding to JoinProbeOperator1 - * and JoinProbeOperator2. - */ - - // First, set up the parent profile,task runtime state - - auto prepare_and_set_parent_profile = [&](PipelineXTask* task, size_t pip_idx) { - DCHECK(pipeline_id_to_profile[pip_idx]); - RETURN_IF_ERROR( - task->prepare(local_params, request.fragment.output_sink, _query_ctx.get())); - return Status::OK(); - }; - - for (auto& _pipeline : _pipelines) { - if (pipeline_id_to_task.contains(_pipeline->id())) { - auto* task = pipeline_id_to_task[_pipeline->id()]; - DCHECK(task != nullptr); - - // if this task has upstream dependency, then record them. - if (_dag.find(_pipeline->id()) != _dag.end()) { - auto& deps = _dag[_pipeline->id()]; - for (auto& dep : deps) { - if (pipeline_id_to_task.contains(dep)) { - auto ss = pipeline_id_to_task[dep]->get_sink_shared_state(); - if (ss) { - task->inject_shared_state(ss); - } else { - pipeline_id_to_task[dep]->inject_shared_state( - task->get_source_shared_state()); - } - } - } - } - } - } - for (size_t pip_idx = 0; pip_idx < _pipelines.size(); pip_idx++) { - if (pipeline_id_to_task.contains(_pipelines[pip_idx]->id())) { - auto* task = pipeline_id_to_task[_pipelines[pip_idx]->id()]; - RETURN_IF_ERROR(prepare_and_set_parent_profile(task, pip_idx)); - } - } - { - std::lock_guard l(_state_map_lock); - _runtime_filter_mgr_map[fragment_instance_id] = std::move(runtime_filter_mgr); - } - } - _pipeline_parent_map.clear(); - _dag.clear(); - _op_id_to_le_state.clear(); - - return Status::OK(); -} - -Status PipelineXFragmentContext::_build_pipelines(ObjectPool* pool, - const doris::TPipelineFragmentParams& request, - const DescriptorTbl& descs, OperatorXPtr* root, - PipelinePtr cur_pipe) { - if (request.fragment.plan.nodes.empty()) { - throw Exception(ErrorCode::INTERNAL_ERROR, "Invalid plan which has no plan node!"); - } - - int node_idx = 0; - - cur_pipe->_name.append(std::to_string(cur_pipe->id())); - - RETURN_IF_ERROR(_create_tree_helper(pool, request.fragment.plan.nodes, request, descs, nullptr, - &node_idx, root, cur_pipe, 0)); - - if (node_idx + 1 != request.fragment.plan.nodes.size()) { - // TODO: print thrift msg for diagnostic purposes. - return Status::InternalError( - "Plan tree only partially reconstructed. Not all thrift nodes were used."); - } - - return Status::OK(); -} - -Status PipelineXFragmentContext::_create_tree_helper(ObjectPool* pool, - const std::vector& tnodes, - const doris::TPipelineFragmentParams& request, - const DescriptorTbl& descs, - OperatorXPtr parent, int* node_idx, - OperatorXPtr* root, PipelinePtr& cur_pipe, - int child_idx) { - // propagate error case - if (*node_idx >= tnodes.size()) { - // TODO: print thrift msg - return Status::InternalError( - "Failed to reconstruct plan tree from thrift. Node id: {}, number of nodes: {}", - *node_idx, tnodes.size()); - } - const TPlanNode& tnode = tnodes[*node_idx]; - - int num_children = tnodes[*node_idx].num_children; - OperatorXPtr op = nullptr; - RETURN_IF_ERROR(_create_operator(pool, tnodes[*node_idx], request, descs, op, cur_pipe, - parent == nullptr ? -1 : parent->node_id(), child_idx)); - - // assert(parent != nullptr || (node_idx == 0 && root_expr != nullptr)); - if (parent != nullptr) { - // add to parent's child(s) - RETURN_IF_ERROR(parent->set_child(op)); - } else { - *root = op; - } - - cur_pipe->_name.push_back('-'); - cur_pipe->_name.append(std::to_string(op->id())); - cur_pipe->_name.append(op->get_name()); - - // rely on that tnodes is preorder of the plan - for (int i = 0; i < num_children; i++) { - ++*node_idx; - RETURN_IF_ERROR(_create_tree_helper(pool, tnodes, request, descs, op, node_idx, nullptr, - cur_pipe, i)); - - // we are expecting a child, but have used all nodes - // this means we have been given a bad tree and must fail - if (*node_idx >= tnodes.size()) { - // TODO: print thrift msg - return Status::InternalError( - "Failed to reconstruct plan tree from thrift. Node id: {}, number of nodes: {}", - *node_idx, tnodes.size()); - } - } - - RETURN_IF_ERROR(op->init(tnode, _runtime_state.get())); - - return Status::OK(); -} - -void PipelineXFragmentContext::_inherit_pipeline_properties( - const DataDistribution& data_distribution, PipelinePtr pipe_with_source, - PipelinePtr pipe_with_sink) { - pipe_with_sink->set_num_tasks(pipe_with_source->num_tasks()); - pipe_with_source->set_num_tasks(_num_instances); - pipe_with_source->set_data_distribution(data_distribution); -} - -Status PipelineXFragmentContext::_add_local_exchange_impl( - int idx, ObjectPool* pool, PipelinePtr cur_pipe, PipelinePtr new_pip, - DataDistribution data_distribution, bool* do_local_exchange, int num_buckets, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx, - const bool ignore_data_hash_distribution) { - auto& operator_xs = cur_pipe->operator_xs(); - const auto downstream_pipeline_id = cur_pipe->id(); - auto local_exchange_id = next_operator_id(); - // 1. Create a new pipeline with local exchange sink. - DataSinkOperatorXPtr sink; - auto sink_id = next_sink_operator_id(); - const bool is_shuffled_hash_join = operator_xs.size() > idx - ? operator_xs[idx]->is_shuffled_hash_join() - : cur_pipe->sink_x()->is_shuffled_hash_join(); - sink.reset(new LocalExchangeSinkOperatorX( - sink_id, local_exchange_id, is_shuffled_hash_join ? _total_instances : _num_instances, - data_distribution.partition_exprs, bucket_seq_to_instance_idx)); - RETURN_IF_ERROR(new_pip->set_sink(sink)); - RETURN_IF_ERROR(new_pip->sink_x()->init(data_distribution.distribution_type, num_buckets, - is_shuffled_hash_join, shuffle_idx_to_instance_idx)); - - // 2. Create and initialize LocalExchangeSharedState. - auto shared_state = LocalExchangeSharedState::create_shared(_num_instances); - switch (data_distribution.distribution_type) { - case ExchangeType::HASH_SHUFFLE: - shared_state->exchanger = ShuffleExchanger::create_unique( - std::max(cur_pipe->num_tasks(), _num_instances), - is_shuffled_hash_join ? _total_instances : _num_instances); - break; - case ExchangeType::BUCKET_HASH_SHUFFLE: - shared_state->exchanger = BucketShuffleExchanger::create_unique( - std::max(cur_pipe->num_tasks(), _num_instances), _num_instances, num_buckets, - ignore_data_hash_distribution); - break; - case ExchangeType::PASSTHROUGH: - shared_state->exchanger = - PassthroughExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); - break; - case ExchangeType::BROADCAST: - shared_state->exchanger = - BroadcastExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); - break; - case ExchangeType::PASS_TO_ONE: - shared_state->exchanger = - BroadcastExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); - break; - case ExchangeType::ADAPTIVE_PASSTHROUGH: - shared_state->exchanger = - AdaptivePassthroughExchanger::create_unique(cur_pipe->num_tasks(), _num_instances); - break; - default: - return Status::InternalError("Unsupported local exchange type : " + - std::to_string((int)data_distribution.distribution_type)); - } - auto sink_dep = std::make_shared(sink_id, local_exchange_id, - "LOCAL_EXCHANGE_SINK_DEPENDENCY", true, - _runtime_state->get_query_ctx()); - sink_dep->set_shared_state(shared_state.get()); - shared_state->sink_deps.push_back(sink_dep); - _op_id_to_le_state.insert({local_exchange_id, {shared_state, sink_dep}}); - - // 3. Set two pipelines' operator list. For example, split pipeline [Scan - AggSink] to - // pipeline1 [Scan - LocalExchangeSink] and pipeline2 [LocalExchangeSource - AggSink]. - - // 3.1 Initialize new pipeline's operator list. - std::copy(operator_xs.begin(), operator_xs.begin() + idx, - std::inserter(new_pip->operator_xs(), new_pip->operator_xs().end())); - - // 3.2 Erase unused operators in previous pipeline. - operator_xs.erase(operator_xs.begin(), operator_xs.begin() + idx); - - // 4. Initialize LocalExchangeSource and insert it into this pipeline. - OperatorXPtr source_op; - source_op.reset(new LocalExchangeSourceOperatorX(pool, local_exchange_id)); - RETURN_IF_ERROR(source_op->set_child(new_pip->operator_xs().back())); - RETURN_IF_ERROR(source_op->init(data_distribution.distribution_type)); - if (!operator_xs.empty()) { - RETURN_IF_ERROR(operator_xs.front()->set_child(source_op)); - } - operator_xs.insert(operator_xs.begin(), source_op); - - shared_state->create_source_dependencies(source_op->operator_id(), source_op->node_id(), - _query_ctx.get()); - - // 5. Set children for two pipelines separately. - std::vector> new_children; - std::vector edges_with_source; - for (auto child : cur_pipe->children()) { - bool found = false; - for (auto op : new_pip->operator_xs()) { - if (child->sink_x()->node_id() == op->node_id()) { - new_pip->set_children(child); - found = true; - }; - } - if (!found) { - new_children.push_back(child); - edges_with_source.push_back(child->id()); - } - } - new_children.push_back(new_pip); - edges_with_source.push_back(new_pip->id()); - - // 6. Set DAG for new pipelines. - if (!new_pip->children().empty()) { - std::vector edges_with_sink; - for (auto child : new_pip->children()) { - edges_with_sink.push_back(child->id()); - } - _dag.insert({new_pip->id(), edges_with_sink}); - } - cur_pipe->set_children(new_children); - _dag[downstream_pipeline_id] = edges_with_source; - RETURN_IF_ERROR(new_pip->sink_x()->set_child(new_pip->operator_xs().back())); - RETURN_IF_ERROR(cur_pipe->sink_x()->set_child(cur_pipe->operator_xs().back())); - - // 7. Inherit properties from current pipeline. - _inherit_pipeline_properties(data_distribution, cur_pipe, new_pip); - return Status::OK(); -} - -Status PipelineXFragmentContext::_add_local_exchange( - int pip_idx, int idx, int node_id, ObjectPool* pool, PipelinePtr cur_pipe, - DataDistribution data_distribution, bool* do_local_exchange, int num_buckets, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx, - const bool ignore_data_distribution) { - DCHECK(_enable_local_shuffle()); - if (_num_instances <= 1) { - return Status::OK(); - } - - if (!cur_pipe->need_to_local_exchange(data_distribution)) { - return Status::OK(); - } - *do_local_exchange = true; - - auto& operator_xs = cur_pipe->operator_xs(); - auto total_op_num = operator_xs.size(); - auto new_pip = add_pipeline(cur_pipe, pip_idx + 1); - RETURN_IF_ERROR(_add_local_exchange_impl( - idx, pool, cur_pipe, new_pip, data_distribution, do_local_exchange, num_buckets, - bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, ignore_data_distribution)); - - CHECK(total_op_num + 1 == cur_pipe->operator_xs().size() + new_pip->operator_xs().size()) - << "total_op_num: " << total_op_num - << " cur_pipe->operator_xs().size(): " << cur_pipe->operator_xs().size() - << " new_pip->operator_xs().size(): " << new_pip->operator_xs().size(); - - // Add passthrough local exchanger if necessary - if (cur_pipe->num_tasks() > 1 && new_pip->num_tasks() == 1 && - Pipeline::is_hash_exchange(data_distribution.distribution_type)) { - RETURN_IF_ERROR(_add_local_exchange_impl( - new_pip->operator_xs().size(), pool, new_pip, add_pipeline(new_pip, pip_idx + 2), - DataDistribution(ExchangeType::PASSTHROUGH), do_local_exchange, num_buckets, - bucket_seq_to_instance_idx, shuffle_idx_to_instance_idx, ignore_data_distribution)); - } - return Status::OK(); -} - -// NOLINTBEGIN(readability-function-size) -// NOLINTBEGIN(readability-function-cognitive-complexity) -Status PipelineXFragmentContext::_create_operator(ObjectPool* pool, const TPlanNode& tnode, - const doris::TPipelineFragmentParams& request, - const DescriptorTbl& descs, OperatorXPtr& op, - PipelinePtr& cur_pipe, int parent_idx, - int child_idx) { - // We directly construct the operator from Thrift because the given array is in the order of preorder traversal. - // Therefore, here we need to use a stack-like structure. - _pipeline_parent_map.pop(cur_pipe, parent_idx, child_idx); - std::stringstream error_msg; - - switch (tnode.node_type) { - case TPlanNodeType::OLAP_SCAN_NODE: { - op.reset(new OlapScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - if (request.__isset.parallel_instances) { - cur_pipe->set_num_tasks(request.parallel_instances); - op->set_ignore_data_distribution(); - } - break; - } - case doris::TPlanNodeType::JDBC_SCAN_NODE: { - if (config::enable_java_support) { - op.reset(new JDBCScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - } else { - return Status::InternalError( - "Jdbc scan node is disabled, you can change be config enable_java_support " - "to true and restart be."); - } - if (request.__isset.parallel_instances) { - cur_pipe->set_num_tasks(request.parallel_instances); - op->set_ignore_data_distribution(); - } - break; - } - case doris::TPlanNodeType::FILE_SCAN_NODE: { - op.reset(new FileScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - if (request.__isset.parallel_instances) { - cur_pipe->set_num_tasks(request.parallel_instances); - op->set_ignore_data_distribution(); - } - break; - } - case TPlanNodeType::ES_SCAN_NODE: - case TPlanNodeType::ES_HTTP_SCAN_NODE: { - op.reset(new EsScanOperatorX(pool, tnode, next_operator_id(), descs, _num_instances)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - if (request.__isset.parallel_instances) { - cur_pipe->set_num_tasks(request.parallel_instances); - op->set_ignore_data_distribution(); - } - break; - } - case TPlanNodeType::EXCHANGE_NODE: { - int num_senders = find_with_default(request.per_exch_num_senders, tnode.node_id, 0); - DCHECK_GT(num_senders, 0); - op.reset(new ExchangeSourceOperatorX(pool, tnode, next_operator_id(), descs, num_senders)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - if (request.__isset.parallel_instances) { - op->set_ignore_data_distribution(); - cur_pipe->set_num_tasks(request.parallel_instances); - } - break; - } - case TPlanNodeType::AGGREGATION_NODE: { - if (tnode.agg_node.grouping_exprs.empty() && - descs.get_tuple_descriptor(tnode.agg_node.output_tuple_id)->slots().empty()) { - return Status::InternalError("Illegal aggregate node " + std::to_string(tnode.node_id) + - ": group by and output is empty"); - } - if (tnode.agg_node.aggregate_functions.empty() && !_runtime_state->enable_agg_spill() && - request.query_options.__isset.enable_distinct_streaming_aggregation && - request.query_options.enable_distinct_streaming_aggregation && - !tnode.agg_node.grouping_exprs.empty()) { - op.reset(new DistinctStreamingAggOperatorX(pool, next_operator_id(), tnode, descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - } else if (tnode.agg_node.__isset.use_streaming_preaggregation && - tnode.agg_node.use_streaming_preaggregation && - !tnode.agg_node.grouping_exprs.empty()) { - op.reset(new StreamingAggOperatorX(pool, next_operator_id(), tnode, descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - } else { - if (_runtime_state->enable_agg_spill() && !tnode.agg_node.grouping_exprs.empty()) { - op.reset(new PartitionedAggSourceOperatorX(pool, tnode, next_operator_id(), descs)); - } else { - op.reset(new AggSourceOperatorX(pool, tnode, next_operator_id(), descs)); - } - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - cur_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(cur_pipe->id()); - - DataSinkOperatorXPtr sink; - if (_runtime_state->enable_agg_spill() && !tnode.agg_node.grouping_exprs.empty()) { - sink.reset(new PartitionedAggSinkOperatorX(pool, next_sink_operator_id(), tnode, - descs, _require_bucket_distribution)); - } else { - sink.reset(new AggSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, - _require_bucket_distribution)); - } - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(cur_pipe->set_sink(sink)); - RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); - } - _require_bucket_distribution = true; - break; - } - case TPlanNodeType::HASH_JOIN_NODE: { - const auto is_broadcast_join = tnode.hash_join_node.__isset.is_broadcast_join && - tnode.hash_join_node.is_broadcast_join; - const auto enable_join_spill = _runtime_state->enable_join_spill(); - if (enable_join_spill && !is_broadcast_join) { - auto tnode_ = tnode; - /// TODO: support rf in partitioned hash join - tnode_.runtime_filters.clear(); - const uint32_t partition_count = 32; - auto inner_probe_operator = - std::make_shared(pool, tnode_, 0, descs); - auto inner_sink_operator = std::make_shared( - pool, 0, tnode_, descs, _need_local_merge); - - RETURN_IF_ERROR(inner_probe_operator->init(tnode_, _runtime_state.get())); - RETURN_IF_ERROR(inner_sink_operator->init(tnode_, _runtime_state.get())); - - auto probe_operator = std::make_shared( - pool, tnode_, next_operator_id(), descs, partition_count); - probe_operator->set_inner_operators(inner_sink_operator, inner_probe_operator); - op = std::move(probe_operator); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - PipelinePtr build_side_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); - - auto sink_operator = std::make_shared( - pool, next_sink_operator_id(), tnode_, descs, _need_local_merge, - partition_count); - sink_operator->set_inner_operators(inner_sink_operator, inner_probe_operator); - DataSinkOperatorXPtr sink = std::move(sink_operator); - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); - RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode_, _runtime_state.get())); - - _pipeline_parent_map.push(op->node_id(), cur_pipe); - _pipeline_parent_map.push(op->node_id(), build_side_pipe); - } else { - op.reset(new HashJoinProbeOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - PipelinePtr build_side_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); - - DataSinkOperatorXPtr sink; - sink.reset(new HashJoinBuildSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, - _need_local_merge)); - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); - RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode, _runtime_state.get())); - - _pipeline_parent_map.push(op->node_id(), cur_pipe); - _pipeline_parent_map.push(op->node_id(), build_side_pipe); - } - _require_bucket_distribution = true; - break; - } - case TPlanNodeType::CROSS_JOIN_NODE: { - op.reset(new NestedLoopJoinProbeOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - PipelinePtr build_side_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); - - DataSinkOperatorXPtr sink; - sink.reset(new NestedLoopJoinBuildSinkOperatorX(pool, next_sink_operator_id(), tnode, descs, - _need_local_merge)); - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); - RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode, _runtime_state.get())); - _pipeline_parent_map.push(op->node_id(), cur_pipe); - _pipeline_parent_map.push(op->node_id(), build_side_pipe); - break; - } - case TPlanNodeType::UNION_NODE: { - int child_count = tnode.num_children; - op.reset(new UnionSourceOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - for (int i = 0; i < child_count; i++) { - PipelinePtr build_side_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(build_side_pipe->id()); - DataSinkOperatorXPtr sink; - sink.reset(new UnionSinkOperatorX(i, next_sink_operator_id(), pool, tnode, descs)); - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(build_side_pipe->set_sink(sink)); - RETURN_IF_ERROR(build_side_pipe->sink_x()->init(tnode, _runtime_state.get())); - // preset children pipelines. if any pipeline found this as its father, will use the prepared pipeline to build. - _pipeline_parent_map.push(op->node_id(), build_side_pipe); - } - break; - } - case TPlanNodeType::SORT_NODE: { - if (_runtime_state->enable_sort_spill()) { - op.reset(new SpillSortSourceOperatorX(pool, tnode, next_operator_id(), descs)); - } else { - op.reset(new SortSourceOperatorX(pool, tnode, next_operator_id(), descs)); - } - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - cur_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(cur_pipe->id()); - - DataSinkOperatorXPtr sink; - if (_runtime_state->enable_sort_spill()) { - sink.reset(new SpillSortSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); - } else { - sink.reset(new SortSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); - } - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(cur_pipe->set_sink(sink)); - RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); - break; - } - case doris::TPlanNodeType::PARTITION_SORT_NODE: { - op.reset(new PartitionSortSourceOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - cur_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(cur_pipe->id()); - - DataSinkOperatorXPtr sink; - sink.reset(new PartitionSortSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(cur_pipe->set_sink(sink)); - RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); - break; - } - case TPlanNodeType::ANALYTIC_EVAL_NODE: { - op.reset(new AnalyticSourceOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - cur_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(cur_pipe->id()); - - DataSinkOperatorXPtr sink; - sink.reset(new AnalyticSinkOperatorX(pool, next_sink_operator_id(), tnode, descs)); - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(cur_pipe->set_sink(sink)); - RETURN_IF_ERROR(cur_pipe->sink_x()->init(tnode, _runtime_state.get())); - _require_bucket_distribution = true; - break; - } - case TPlanNodeType::INTERSECT_NODE: { - RETURN_IF_ERROR(_build_operators_for_set_operation_node( - pool, tnode, descs, op, cur_pipe, parent_idx, child_idx)); - break; - } - case TPlanNodeType::EXCEPT_NODE: { - RETURN_IF_ERROR(_build_operators_for_set_operation_node( - pool, tnode, descs, op, cur_pipe, parent_idx, child_idx)); - break; - } - case TPlanNodeType::REPEAT_NODE: { - op.reset(new RepeatOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::TABLE_FUNCTION_NODE: { - op.reset(new TableFunctionOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::ASSERT_NUM_ROWS_NODE: { - op.reset(new AssertNumRowsOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::EMPTY_SET_NODE: { - op.reset(new EmptySetSourceOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::DATA_GEN_SCAN_NODE: { - op.reset(new DataGenSourceOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::SCHEMA_SCAN_NODE: { - op.reset(new SchemaScanOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::META_SCAN_NODE: { - op.reset(new MetaScanOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - case TPlanNodeType::SELECT_NODE: { - op.reset(new SelectOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - break; - } - default: - return Status::InternalError("Unsupported exec type in pipelineX: {}", - print_plan_node_type(tnode.node_type)); - } - - _require_bucket_distribution = true; - - return Status::OK(); -} -// NOLINTEND(readability-function-cognitive-complexity) -// NOLINTEND(readability-function-size) - -template -Status PipelineXFragmentContext::_build_operators_for_set_operation_node( - ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs, OperatorXPtr& op, - PipelinePtr& cur_pipe, int parent_idx, int child_idx) { - op.reset(new SetSourceOperatorX(pool, tnode, next_operator_id(), descs)); - RETURN_IF_ERROR(cur_pipe->add_operator(op)); - - const auto downstream_pipeline_id = cur_pipe->id(); - if (_dag.find(downstream_pipeline_id) == _dag.end()) { - _dag.insert({downstream_pipeline_id, {}}); - } - - for (int child_id = 0; child_id < tnode.num_children; child_id++) { - PipelinePtr probe_side_pipe = add_pipeline(cur_pipe); - _dag[downstream_pipeline_id].push_back(probe_side_pipe->id()); - - DataSinkOperatorXPtr sink; - if (child_id == 0) { - sink.reset(new SetSinkOperatorX(child_id, next_sink_operator_id(), pool, - tnode, descs)); - } else { - sink.reset(new SetProbeSinkOperatorX(child_id, next_sink_operator_id(), - pool, tnode, descs)); - } - sink->set_dests_id({op->operator_id()}); - RETURN_IF_ERROR(probe_side_pipe->set_sink(sink)); - RETURN_IF_ERROR(probe_side_pipe->sink_x()->init(tnode, _runtime_state.get())); - // prepare children pipelines. if any pipeline found this as its father, will use the prepared pipeline to build. - _pipeline_parent_map.push(op->node_id(), probe_side_pipe); - } - - return Status::OK(); -} - -Status PipelineXFragmentContext::submit() { - if (_submitted) { - return Status::InternalError("submitted"); - } - _submitted = true; - - int submit_tasks = 0; - Status st; - auto* scheduler = _query_ctx->get_pipe_exec_scheduler(); - for (auto& task : _tasks) { - for (auto& t : task) { - st = scheduler->schedule_task(t.get()); - if (!st) { - std::lock_guard l(_status_lock); - cancel(PPlanFragmentCancelReason::INTERNAL_ERROR, "submit context fail"); - _total_tasks = submit_tasks; - break; - } - submit_tasks++; - } - } - if (!st.ok()) { - std::lock_guard l(_task_mutex); - if (_closed_tasks == _total_tasks) { - _close_fragment_instance(); - } - return Status::InternalError("Submit pipeline failed. err = {}, BE: {}", st.to_string(), - BackendOptions::get_localhost()); - } else { - return st; - } -} - -void PipelineXFragmentContext::close_sink() { - for (auto& tasks : _tasks) { - auto& root_task = *tasks.begin(); - auto st = root_task->close_sink(_prepared ? Status::RuntimeError("prepare failed") - : Status::OK()); - if (!st.ok()) { - LOG_WARNING("PipelineXFragmentContext::close_sink() error").tag("msg", st.msg()); - } - } -} - -void PipelineXFragmentContext::close_if_prepare_failed(Status st) { - for (auto& task : _tasks) { - for (auto& t : task) { - DCHECK(!t->is_pending_finish()); - WARN_IF_ERROR(t->close(st), "close_if_prepare_failed failed: "); - close_a_pipeline(); - } - } - _query_ctx->cancel(st.to_string(), st, _fragment_id); -} - -void PipelineXFragmentContext::_close_fragment_instance() { - if (_is_fragment_instance_closed) { - return; - } - Defer defer_op {[&]() { _is_fragment_instance_closed = true; }}; - _runtime_profile->total_time_counter()->update(_fragment_watcher.elapsed_time()); - static_cast(send_report(true)); - if (_runtime_state->enable_profile()) { - std::stringstream ss; - // Compute the _local_time_percent before pretty_print the runtime_profile - // Before add this operation, the print out like that: - // UNION_NODE (id=0):(Active: 56.720us, non-child: 00.00%) - // After add the operation, the print out like that: - // UNION_NODE (id=0):(Active: 56.720us, non-child: 82.53%) - // We can easily know the exec node execute time without child time consumed. - _runtime_state->runtime_profile()->compute_time_in_profile(); - _runtime_state->runtime_profile()->pretty_print(&ss); - if (_runtime_state->load_channel_profile()) { - _runtime_state->load_channel_profile()->pretty_print(&ss); - } - - LOG_INFO("Query {} fragment {} profile:\n {}", print_id(this->_query_id), - this->_fragment_id, ss.str()); - } - - if (_query_ctx->enable_profile()) { - _query_ctx->add_fragment_profile_x(_fragment_id, collect_realtime_profile_x(), - collect_realtime_load_channel_profile_x()); - } - - // all submitted tasks done - _exec_env->fragment_mgr()->remove_pipeline_context( - std::dynamic_pointer_cast(shared_from_this())); -} - -Status PipelineXFragmentContext::send_report(bool done) { - Status exec_status = Status::OK(); - { - std::lock_guard l(_status_lock); - exec_status = _query_ctx->exec_status(); - } - - // If plan is done successfully, but _is_report_success is false, - // no need to send report. - if (!_is_report_success && done && exec_status.ok()) { - return Status::NeedSendAgain(""); - } - - // If both _is_report_success and _is_report_on_cancel are false, - // which means no matter query is success or failed, no report is needed. - // This may happen when the query limit reached and - // a internal cancellation being processed - if (!_is_report_success && !_is_report_on_cancel) { - return Status::NeedSendAgain(""); - } - - std::vector runtime_states; - - for (auto& task_state : _task_runtime_states) { - runtime_states.push_back(task_state.get()); - } - - ReportStatusRequest req {true, - exec_status, - runtime_states, - nullptr, - _runtime_state->load_channel_profile(), - done || !exec_status.ok(), - _query_ctx->coord_addr, - _query_id, - _fragment_id, - TUniqueId(), - _backend_num, - _runtime_state.get(), - [this](Status st) { return update_status(st); }, - [this](const PPlanFragmentCancelReason& reason, - const std::string& msg) { cancel(reason, msg); }}; - - return _report_status_cb( - req, std::dynamic_pointer_cast(shared_from_this())); -} - -std::vector> -PipelineXFragmentContext::collect_realtime_profile_x() const { - std::vector> res; - DCHECK(_query_ctx->enable_pipeline_x_exec() == true) - << fmt::format("Query {} calling a pipeline X function, but its pipeline X is disabled", - print_id(this->_query_id)); - - // we do not have mutex to protect pipeline_id_to_profile - // so we need to make sure this funciton is invoked after fragment context - // has already been prepared. - if (!this->_prepared) { - std::string msg = - "Query " + print_id(this->_query_id) + " collecting profile, but its not prepared"; - DCHECK(false) << msg; - LOG_ERROR(msg); - return res; - } - - // pipeline_id_to_profile is initialized in prepare stage - for (auto& pipeline_profile : _runtime_state->pipeline_id_to_profile()) { - auto profile_ptr = std::make_shared(); - pipeline_profile->to_thrift(profile_ptr.get()); - res.push_back(profile_ptr); - } - - return res; -} - -std::shared_ptr -PipelineXFragmentContext::collect_realtime_load_channel_profile_x() const { - // we do not have mutex to protect pipeline_id_to_profile - // so we need to make sure this funciton is invoked after fragment context - // has already been prepared. - if (!this->_prepared) { - std::string msg = - "Query " + print_id(this->_query_id) + " collecting profile, but its not prepared"; - DCHECK(false) << msg; - LOG_ERROR(msg); - return nullptr; - } - - for (auto& runtime_state : _task_runtime_states) { - if (runtime_state->runtime_profile() == nullptr) { - continue; - } - - auto tmp_load_channel_profile = std::make_shared(); - - runtime_state->runtime_profile()->to_thrift(tmp_load_channel_profile.get()); - this->_runtime_state->load_channel_profile()->update(*tmp_load_channel_profile); - } - - auto load_channel_profile = std::make_shared(); - this->_runtime_state->load_channel_profile()->to_thrift(load_channel_profile.get()); - return load_channel_profile; -} - -std::string PipelineXFragmentContext::debug_string() { - fmt::memory_buffer debug_string_buffer; - fmt::format_to(debug_string_buffer, "PipelineXFragmentContext Info:\n"); - for (size_t j = 0; j < _tasks.size(); j++) { - fmt::format_to(debug_string_buffer, "Tasks in instance {}:\n", j); - for (size_t i = 0; i < _tasks[j].size(); i++) { - fmt::format_to(debug_string_buffer, "Task {}: {}\n", i, _tasks[j][i]->debug_string()); - } - } - - return fmt::to_string(debug_string_buffer); -} -} // namespace doris::pipeline diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h deleted file mode 100644 index c87f8f4f784051..00000000000000 --- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h +++ /dev/null @@ -1,247 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/status.h" -#include "pipeline/pipeline.h" -#include "pipeline/pipeline_fragment_context.h" -#include "pipeline/pipeline_task.h" -#include "pipeline/pipeline_x/local_exchange/local_exchanger.h" -#include "pipeline/pipeline_x/pipeline_x_task.h" -#include "runtime/query_context.h" -#include "runtime/runtime_state.h" -#include "util/runtime_profile.h" -#include "util/stopwatch.hpp" - -namespace doris { -class ExecNode; -class DataSink; -struct ReportStatusRequest; -class ExecEnv; -class RuntimeFilterMergeControllerEntity; -class TDataSink; -class TPipelineFragmentParams; - -namespace pipeline { -class Dependency; - -class PipelineXFragmentContext : public PipelineFragmentContext { -public: - // Callback to report execution status of plan fragment. - // 'profile' is the cumulative profile, 'done' indicates whether the execution - // is done or still continuing. - // Note: this does not take a const RuntimeProfile&, because it might need to call - // functions like PrettyPrint() or to_thrift(), neither of which is const - // because they take locks. - PipelineXFragmentContext(const TUniqueId& query_id, const int fragment_id, - std::shared_ptr query_ctx, ExecEnv* exec_env, - const std::function& call_back, - const report_status_callback& report_status_cb); - - ~PipelineXFragmentContext() override; - - void instance_ids(std::vector& ins_ids) const override { - ins_ids.resize(_fragment_instance_ids.size()); - for (size_t i = 0; i < _fragment_instance_ids.size(); i++) { - ins_ids[i] = _fragment_instance_ids[i]; - } - } - - void instance_ids(std::vector& ins_ids) const override { - ins_ids.resize(_fragment_instance_ids.size()); - for (size_t i = 0; i < _fragment_instance_ids.size(); i++) { - ins_ids[i] = print_id(_fragment_instance_ids[i]); - } - } - - void add_merge_controller_handler( - std::shared_ptr& handler) override { - _merge_controller_handlers.emplace_back(handler); - } - - // bool is_canceled() const { return _runtime_state->is_cancelled(); } - - // Prepare global information including global states and the unique operator tree shared by all pipeline tasks. - Status prepare(const doris::TPipelineFragmentParams& request) override; - - Status submit() override; - - void close_if_prepare_failed(Status st) override; - void close_sink() override; - - void cancel(const PPlanFragmentCancelReason& reason = PPlanFragmentCancelReason::INTERNAL_ERROR, - const std::string& msg = "") override; - - Status send_report(bool) override; - - [[nodiscard]] int next_operator_id() { return _operator_id--; } - - [[nodiscard]] int max_operator_id() const { return _operator_id; } - - [[nodiscard]] int next_sink_operator_id() { return _sink_operator_id--; } - - [[nodiscard]] int max_sink_operator_id() const { return _sink_operator_id; } - - std::vector> collect_realtime_profile_x() const; - std::shared_ptr collect_realtime_load_channel_profile_x() const; - - std::string debug_string() override; - -private: - void _close_fragment_instance() override; - Status _build_pipeline_tasks(const doris::TPipelineFragmentParams& request) override; - Status _add_local_exchange(int pip_idx, int idx, int node_id, ObjectPool* pool, - PipelinePtr cur_pipe, DataDistribution data_distribution, - bool* do_local_exchange, int num_buckets, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx, - const bool ignore_data_distribution); - void _inherit_pipeline_properties(const DataDistribution& data_distribution, - PipelinePtr pipe_with_source, PipelinePtr pipe_with_sink); - Status _add_local_exchange_impl(int idx, ObjectPool* pool, PipelinePtr cur_pipe, - PipelinePtr new_pipe, DataDistribution data_distribution, - bool* do_local_exchange, int num_buckets, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx, - const bool ignore_data_distribution); - - [[nodiscard]] Status _build_pipelines(ObjectPool* pool, - const doris::TPipelineFragmentParams& request, - const DescriptorTbl& descs, OperatorXPtr* root, - PipelinePtr cur_pipe); - Status _create_tree_helper(ObjectPool* pool, const std::vector& tnodes, - const doris::TPipelineFragmentParams& request, - const DescriptorTbl& descs, OperatorXPtr parent, int* node_idx, - OperatorXPtr* root, PipelinePtr& cur_pipe, int child_idx); - - Status _create_operator(ObjectPool* pool, const TPlanNode& tnode, - const doris::TPipelineFragmentParams& request, - const DescriptorTbl& descs, OperatorXPtr& op, PipelinePtr& cur_pipe, - int parent_idx, int child_idx); - template - Status _build_operators_for_set_operation_node(ObjectPool* pool, const TPlanNode& tnode, - const DescriptorTbl& descs, OperatorXPtr& op, - PipelinePtr& cur_pipe, int parent_idx, - int child_idx); - - Status _create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink, - const std::vector& output_exprs, - const TPipelineFragmentParams& params, const RowDescriptor& row_desc, - RuntimeState* state, DescriptorTbl& desc_tbl, - PipelineId cur_pipeline_id); - Status _plan_local_exchange(int num_buckets, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx); - Status _plan_local_exchange(int num_buckets, int pip_idx, PipelinePtr pip, - const std::map& bucket_seq_to_instance_idx, - const std::map& shuffle_idx_to_instance_idx, - const bool ignore_data_distribution); - - bool _enable_local_shuffle() const { return _runtime_state->enable_local_shuffle(); } - - OperatorXPtr _root_op = nullptr; - // this is a [n * m] matrix. n is parallelism of pipeline engine and m is the number of pipelines. - std::vector>> _tasks; - - bool _need_local_merge = false; - - // It is used to manage the lifecycle of RuntimeFilterMergeController - std::vector> _merge_controller_handlers; - - // TODO: remove the _sink and _multi_cast_stream_sink_senders to set both - // of it in pipeline task not the fragment_context -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wshadow-field" -#endif - DataSinkOperatorXPtr _sink = nullptr; -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - - // `_dag` manage dependencies between pipelines by pipeline ID. the indices will be blocked by members - std::map> _dag; - - // We use preorder traversal to create an operator tree. When we meet a join node, we should - // build probe operator and build operator in separate pipelines. To do this, we should build - // ProbeSide first, and use `_pipelines_to_build` to store which pipeline the build operator - // is in, so we can build BuildSide once we complete probe side. - struct pipeline_parent_map { - std::map> _build_side_pipelines; - void push(int parent_node_id, PipelinePtr pipeline) { - if (!_build_side_pipelines.contains(parent_node_id)) { - _build_side_pipelines.insert({parent_node_id, {pipeline}}); - } else { - _build_side_pipelines[parent_node_id].push_back(pipeline); - } - } - void pop(PipelinePtr& cur_pipe, int parent_node_id, int child_idx) { - if (!_build_side_pipelines.contains(parent_node_id)) { - return; - } - DCHECK(_build_side_pipelines.contains(parent_node_id)); - auto& child_pipeline = _build_side_pipelines[parent_node_id]; - DCHECK(child_idx < child_pipeline.size()); - cur_pipe = child_pipeline[child_idx]; - } - void clear() { _build_side_pipelines.clear(); } - } _pipeline_parent_map; - - std::mutex _state_map_lock; - - int _operator_id = 0; - int _sink_operator_id = 0; - std::map, std::shared_ptr>> - _op_id_to_le_state; - - // UniqueId -> runtime mgr - std::map> _runtime_filter_mgr_map; - - //Here are two types of runtime states: - // - _runtime state is at the Fragment level. - // - _task_runtime_states is at the task level, unique to each task. - - std::vector _fragment_instance_ids; - // Local runtime states for each task - std::vector> _task_runtime_states; - - std::vector> _runtime_filter_states; - - // Total instance num running on all BEs - int _total_instances = -1; - - bool _require_bucket_distribution = false; -}; - -} // namespace pipeline -} // namespace doris diff --git a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp index 6ffc5571b4e563..e8957318a6d66b 100644 --- a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp +++ b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp @@ -29,8 +29,8 @@ #include "pipeline/exec/operator.h" #include "pipeline/exec/scan_operator.h" #include "pipeline/pipeline.h" +#include "pipeline/pipeline_fragment_context.h" #include "pipeline/task_queue.h" -#include "pipeline_x_fragment_context.h" #include "runtime/descriptors.h" #include "runtime/query_context.h" #include "runtime/thread_context.h" diff --git a/be/src/pipeline/pipeline_x/pipeline_x_task.h b/be/src/pipeline/pipeline_x/pipeline_x_task.h index 1f3dd9c3b71f5f..c1a72836fd8720 100644 --- a/be/src/pipeline/pipeline_x/pipeline_x_task.h +++ b/be/src/pipeline/pipeline_x/pipeline_x_task.h @@ -126,8 +126,6 @@ class PipelineXTask : public PipelineTask { return _op_shared_states[id].get(); } - bool is_pipelineX() const override { return true; } - void wake_up(); DataSinkOperatorXPtr sink() const { return _sink; } diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp index 8981a7e621c463..0461999d185260 100644 --- a/be/src/pipeline/task_scheduler.cpp +++ b/be/src/pipeline/task_scheduler.cpp @@ -48,156 +48,6 @@ namespace doris::pipeline { -BlockedTaskScheduler::BlockedTaskScheduler(std::string name) - : _name(std::move(name)), _started(false), _shutdown(false) {} - -Status BlockedTaskScheduler::start() { - LOG(INFO) << "BlockedTaskScheduler start"; - RETURN_IF_ERROR(Thread::create( - "BlockedTaskScheduler", _name, [this]() { this->_schedule(); }, &_thread)); - while (!this->_started.load()) { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - } - LOG(INFO) << "BlockedTaskScheduler started"; - return Status::OK(); -} - -void BlockedTaskScheduler::shutdown() { - LOG(INFO) << "Start shutdown BlockedTaskScheduler"; - if (!this->_shutdown) { - this->_shutdown = true; - if (_thread) { - _task_cond.notify_one(); - _thread->join(); - } - } -} - -Status BlockedTaskScheduler::add_blocked_task(PipelineTask* task) { - if (this->_shutdown) { - return Status::InternalError("BlockedTaskScheduler shutdown"); - } - std::unique_lock lock(_task_mutex); - if (task->is_pipelineX()) { - // put this task into current dependency's blocking queue and wait for event notification - // instead of using a separate BlockedTaskScheduler. - task->set_running(false); - return Status::OK(); - } - _blocked_tasks.push_back(task); - _task_cond.notify_one(); - task->set_running(false); - return Status::OK(); -} - -void BlockedTaskScheduler::_schedule() { - _started.store(true); - std::list local_blocked_tasks; - int empty_times = 0; - - while (!_shutdown) { - { - std::unique_lock lock(this->_task_mutex); - local_blocked_tasks.splice(local_blocked_tasks.end(), _blocked_tasks); - if (local_blocked_tasks.empty()) { - while (!_shutdown.load() && _blocked_tasks.empty()) { - _task_cond.wait_for(lock, std::chrono::milliseconds(10)); - } - - if (_shutdown.load()) { - break; - } - - DCHECK(!_blocked_tasks.empty()); - local_blocked_tasks.splice(local_blocked_tasks.end(), _blocked_tasks); - } - } - - auto origin_local_block_tasks_size = local_blocked_tasks.size(); - auto iter = local_blocked_tasks.begin(); - VecDateTimeValue now = VecDateTimeValue::local_time(); - while (iter != local_blocked_tasks.end()) { - auto* task = *iter; - auto state = task->get_state(); - task->log_detail_if_need(); - if (state == PipelineTaskState::PENDING_FINISH) { - // should cancel or should finish - if (task->is_pending_finish()) { - VLOG_DEBUG << "Task pending" << task->debug_string(); - iter++; - } else { - _make_task_run(local_blocked_tasks, iter, PipelineTaskState::PENDING_FINISH); - } - } else if (task->query_context()->is_cancelled()) { - _make_task_run(local_blocked_tasks, iter); - } else if (task->query_context()->is_timeout(now)) { - LOG(WARNING) << "Timeout, query_id=" << print_id(task->query_context()->query_id()) - << ", instance_id=" << print_id(task->instance_id()) - << ", task info: " << task->debug_string(); - - task->query_context()->cancel("", Status::Cancelled("")); - _make_task_run(local_blocked_tasks, iter); - } else if (state == PipelineTaskState::BLOCKED_FOR_DEPENDENCY) { - if (task->has_dependency()) { - iter++; - } else { - _make_task_run(local_blocked_tasks, iter); - } - } else if (state == PipelineTaskState::BLOCKED_FOR_SOURCE) { - if (task->source_can_read()) { - _make_task_run(local_blocked_tasks, iter); - } else { - iter++; - } - } else if (state == PipelineTaskState::BLOCKED_FOR_RF) { - if (task->runtime_filters_are_ready_or_timeout()) { - _make_task_run(local_blocked_tasks, iter); - } else { - iter++; - } - } else if (state == PipelineTaskState::BLOCKED_FOR_SINK) { - if (task->sink_can_write()) { - _make_task_run(local_blocked_tasks, iter); - } else { - iter++; - } - } else { - // TODO: DCHECK the state - _make_task_run(local_blocked_tasks, iter); - } - } - - if (origin_local_block_tasks_size == 0 || - local_blocked_tasks.size() == origin_local_block_tasks_size) { - empty_times += 1; - } else { - empty_times = 0; - } - - if (empty_times != 0 && (empty_times & (EMPTY_TIMES_TO_YIELD - 1)) == 0) { -#ifdef __x86_64__ - _mm_pause(); -#else - sched_yield(); -#endif - } - if (empty_times == EMPTY_TIMES_TO_YIELD * 10) { - empty_times = 0; - sched_yield(); - } - } - LOG(INFO) << "BlockedTaskScheduler schedule thread stop"; -} - -void BlockedTaskScheduler::_make_task_run(std::list& local_tasks, - std::list::iterator& task_itr, - PipelineTaskState t_state) { - auto* task = *task_itr; - task->set_state(t_state); - local_tasks.erase(task_itr++); - static_cast(task->get_task_queue()->push_back(task)); -} - TaskScheduler::~TaskScheduler() { stop(); LOG(INFO) << "Task scheduler " << _name << " shutdown"; @@ -241,13 +91,8 @@ void _close_task(PipelineTask* task, PipelineTaskState state, Status exec_status // for pending finish now. So that could call close directly. Status status = task->close(exec_status); if (!status.ok() && state != PipelineTaskState::CANCELED) { - if (task->is_pipelineX()) { //should call fragment context cancel, in it will call query context cancel - task->fragment_context()->cancel(PPlanFragmentCancelReason::INTERNAL_ERROR, - std::string(status.msg())); - } else { - task->query_context()->cancel(status.to_string(), - Status::Cancelled(status.to_string())); - } + task->fragment_context()->cancel(PPlanFragmentCancelReason::INTERNAL_ERROR, + std::string(status.msg())); state = PipelineTaskState::CANCELED; } task->set_state(state); @@ -264,7 +109,7 @@ void TaskScheduler::_do_work(size_t index) { if (!task) { continue; } - if (task->is_pipelineX() && task->is_running()) { + if (task->is_running()) { static_cast(_task_queue->push_back(task, index)); continue; } @@ -278,8 +123,6 @@ void TaskScheduler::_do_work(size_t index) { // If the state is PENDING_FINISH, then the task is come from blocked queue, its is_pending_finish // has to return false. The task is finished and need to close now. if (state == PipelineTaskState::PENDING_FINISH) { - DCHECK(task->is_pipelineX() || !task->is_pending_finish()) - << "must not pending close " << task->debug_string(); Status exec_status = fragment_ctx->get_query_ctx()->exec_status(); _close_task(task, canceled ? PipelineTaskState::CANCELED : PipelineTaskState::FINISHED, exec_status); @@ -301,13 +144,8 @@ void TaskScheduler::_do_work(size_t index) { continue; } - if (task->is_pipelineX()) { - task->set_state(PipelineTaskState::RUNNABLE); - } + task->set_state(PipelineTaskState::RUNNABLE); - DCHECK(task->is_pipelineX() || task->get_state() == PipelineTaskState::RUNNABLE) - << "state:" << get_state_name(task->get_state()) - << " task: " << task->debug_string(); // task exec bool eos = false; auto status = Status::OK(); @@ -354,11 +192,9 @@ void TaskScheduler::_do_work(size_t index) { continue; } else if (!status.ok()) { task->set_eos_time(); - LOG(WARNING) << fmt::format( - "Pipeline task failed. query_id: {} reason: {}", - PrintInstanceStandardInfo(task->query_context()->query_id(), - task->fragment_context()->get_fragment_instance_id()), - status.to_string()); + LOG(WARNING) << fmt::format("Pipeline task failed. query_id: {} reason: {}", + print_id(task->query_context()->query_id()), + status.to_string()); // Print detail informations below when you debugging here. // // LOG(WARNING)<< "task:\n"<debug_string(); @@ -375,35 +211,21 @@ void TaskScheduler::_do_work(size_t index) { task->set_eos_time(); // TODO: pipeline parallel need to wait the last task finish to call finalize // and find_p_dependency - VLOG_DEBUG << fmt::format( - "Try close task: {}, fragment_ctx->is_canceled(): {}", - PrintInstanceStandardInfo(task->query_context()->query_id(), - task->fragment_context()->get_fragment_instance_id()), - fragment_ctx->is_canceled()); - if (task->is_pipelineX()) { - // is pending finish will add the task to dependency's blocking queue, and then the task will be - // added to running queue when dependency is ready. - if (task->is_pending_finish()) { - // Only meet eos, should set task to PENDING_FINISH state - task->set_state(PipelineTaskState::PENDING_FINISH); - task->set_running(false); - } else { - // Close the task directly? - Status exec_status = fragment_ctx->get_query_ctx()->exec_status(); - _close_task( - task, - canceled ? PipelineTaskState::CANCELED : PipelineTaskState::FINISHED, - exec_status); - } - } else { + VLOG_DEBUG << fmt::format("Try close task: {}, fragment_ctx->is_canceled(): {}", + print_id(task->query_context()->query_id()), + fragment_ctx->is_canceled()); + // is pending finish will add the task to dependency's blocking queue, and then the task will be + // added to running queue when dependency is ready. + if (task->is_pending_finish()) { // Only meet eos, should set task to PENDING_FINISH state - // pipeline is ok, because it will check is pending finish, and if it is ready, it will be invoked. task->set_state(PipelineTaskState::PENDING_FINISH); task->set_running(false); - // After the task is added to the block queue, it maybe run by another thread - // and the task maybe released in the other thread. And will core at - // task set running. - static_cast(_blocked_task_scheduler->add_blocked_task(task)); + } else { + // Close the task directly? + Status exec_status = fragment_ctx->get_query_ctx()->exec_status(); + _close_task(task, + canceled ? PipelineTaskState::CANCELED : PipelineTaskState::FINISHED, + exec_status); } continue; } @@ -414,7 +236,7 @@ void TaskScheduler::_do_work(size_t index) { case PipelineTaskState::BLOCKED_FOR_SINK: case PipelineTaskState::BLOCKED_FOR_RF: case PipelineTaskState::BLOCKED_FOR_DEPENDENCY: - static_cast(_blocked_task_scheduler->add_blocked_task(task)); + task->set_running(false); break; case PipelineTaskState::RUNNABLE: task->set_running(false); diff --git a/be/src/pipeline/task_scheduler.h b/be/src/pipeline/task_scheduler.h index 5bbf85fad452fd..8e513748203e21 100644 --- a/be/src/pipeline/task_scheduler.h +++ b/be/src/pipeline/task_scheduler.h @@ -44,41 +44,11 @@ class TaskQueue; namespace doris::pipeline { -class BlockedTaskScheduler { -public: - explicit BlockedTaskScheduler(std::string name); - - ~BlockedTaskScheduler() = default; - - Status start(); - void shutdown(); - Status add_blocked_task(PipelineTask* task); - -private: - std::mutex _task_mutex; - std::string _name; - std::condition_variable _task_cond; - std::list _blocked_tasks; - - scoped_refptr _thread; - std::atomic _started; - std::atomic _shutdown; - - static constexpr auto EMPTY_TIMES_TO_YIELD = 64; - - void _schedule(); - void _make_task_run(std::list& local_tasks, - std::list::iterator& task_itr, - PipelineTaskState state = PipelineTaskState::RUNNABLE); -}; - class TaskScheduler { public: - TaskScheduler(ExecEnv* exec_env, std::shared_ptr b_scheduler, - std::shared_ptr task_queue, std::string name, + TaskScheduler(ExecEnv* exec_env, std::shared_ptr task_queue, std::string name, CgroupCpuCtl* cgroup_cpu_ctl) : _task_queue(std::move(task_queue)), - _blocked_task_scheduler(std::move(b_scheduler)), _shutdown(false), _name(name), _cgroup_cpu_ctl(cgroup_cpu_ctl) {} @@ -97,7 +67,6 @@ class TaskScheduler { std::unique_ptr _fix_thread_pool; std::shared_ptr _task_queue; std::vector>> _markers; - std::shared_ptr _blocked_task_scheduler; std::atomic _shutdown; std::string _name; CgroupCpuCtl* _cgroup_cpu_ctl = nullptr; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index eb3e6f8a89c81b..1cf0aea8349d94 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -45,7 +45,6 @@ class DeltaWriterV2Pool; } // namespace vectorized namespace pipeline { class TaskScheduler; -class BlockedTaskScheduler; struct RuntimeFilterTimerQueue; } // namespace pipeline class WorkloadGroupMgr; @@ -305,10 +304,6 @@ class ExecEnv { } std::shared_ptr get_dummy_lru_cache() { return _dummy_lru_cache; } - std::shared_ptr get_global_block_scheduler() { - return _global_block_scheduler; - } - pipeline::RuntimeFilterTimerQueue* runtime_filter_timer_queue() { return _runtime_filter_timer_queue; } @@ -453,11 +448,6 @@ class ExecEnv { TabletHotspot* _tablet_hotspot; CloudWarmUpManager* _cloud_warm_up_manager; - // used for query with group cpu hard limit - std::shared_ptr _global_block_scheduler; - // used for query without workload group - std::shared_ptr _without_group_block_scheduler; - pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 5a7e39cf158c41..ae36757b10a37c 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -364,15 +364,10 @@ Status ExecEnv::init_pipeline_task_scheduler() { LOG_INFO("pipeline executors_size set ").tag("size", executors_size); // TODO pipeline workload group combie two blocked schedulers. auto t_queue = std::make_shared(executors_size); - _without_group_block_scheduler = - std::make_shared("PipeNoGSchePool"); - _without_group_task_scheduler = new pipeline::TaskScheduler( - this, _without_group_block_scheduler, t_queue, "PipeNoGSchePool", nullptr); + _without_group_task_scheduler = + new pipeline::TaskScheduler(this, t_queue, "PipeNoGSchePool", nullptr); RETURN_IF_ERROR(_without_group_task_scheduler->start()); - RETURN_IF_ERROR(_without_group_block_scheduler->start()); - _global_block_scheduler = std::make_shared("PipeGBlockSche"); - RETURN_IF_ERROR(_global_block_scheduler->start()); _runtime_filter_timer_queue = new doris::pipeline::RuntimeFilterTimerQueue(); _runtime_filter_timer_queue->run(); return Status::OK(); @@ -627,10 +622,8 @@ void ExecEnv::destroy() { // stop workload scheduler SAFE_STOP(_workload_sched_mgr); // stop pipline step 1, non-cgroup execution - SAFE_SHUTDOWN(_without_group_block_scheduler.get()); SAFE_STOP(_without_group_task_scheduler); // stop pipline step 2, cgroup execution - SAFE_SHUTDOWN(_global_block_scheduler.get()); SAFE_STOP(_workload_group_manager); SAFE_STOP(_external_scan_context_mgr); diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index c4605f1de54590..7dbf1571e50737 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -42,7 +42,6 @@ #include #include "common/status.h" -#include "pipeline/pipeline_x/pipeline_x_fragment_context.h" // IWYU pragma: no_include #include // IWYU pragma: keep #include @@ -845,7 +844,7 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params, _setup_shared_hashtable_for_broadcast_join(params, query_ctx.get()); int64_t duration_ns = 0; std::shared_ptr context = - std::make_shared( + std::make_shared( query_ctx->query_id(), params.fragment_id, query_ctx, _exec_env, cb, std::bind(std::mem_fn(&FragmentMgr::trigger_pipeline_context_report), this, std::placeholders::_1, std::placeholders::_2)); @@ -892,7 +891,7 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params, g_fragment_last_active_time.set_value(now); std::lock_guard lock(_lock); std::vector ins_ids; - reinterpret_cast(context.get())->instance_ids(ins_ids); + context->instance_ids(ins_ids); // TODO: simplify this mapping for (const auto& ins_id : ins_ids) { _pipeline_map.insert({ins_id, context}); @@ -1029,8 +1028,7 @@ void FragmentMgr::cancel_worker() { for (auto& pipeline_itr : _pipeline_map) { if (pipeline_itr.second->is_timeout(now)) { std::vector ins_ids; - reinterpret_cast(pipeline_itr.second.get()) - ->instance_ids(ins_ids); + pipeline_itr.second->instance_ids(ins_ids); for (auto& ins_id : ins_ids) { to_cancel.push_back(ins_id); } diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h index 25b555f4fe8d60..5b70be6d8a5c41 100644 --- a/be/src/runtime/fragment_mgr.h +++ b/be/src/runtime/fragment_mgr.h @@ -51,7 +51,6 @@ extern bvar::Status g_fragment_last_active_time; namespace pipeline { class PipelineFragmentContext; -class PipelineXFragmentContext; } // namespace pipeline class QueryContext; class ExecEnv; @@ -104,7 +103,7 @@ class FragmentMgr : public RestMonitorIface { void cancel_instance(const TUniqueId& instance_id, const PPlanFragmentCancelReason& reason, const std::string& msg = ""); // Cancel fragment (only pipelineX). - // {query id fragment} -> PipelineXFragmentContext + // {query id fragment} -> PipelineFragmentContext void cancel_fragment(const TUniqueId& query_id, int32_t fragment_id, const PPlanFragmentCancelReason& reason, const std::string& msg = ""); diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index 081d8ca1f590ed..c521c855515952 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -33,7 +33,6 @@ #include "olap/olap_common.h" #include "pipeline/pipeline_fragment_context.h" #include "pipeline/pipeline_x/dependency.h" -#include "pipeline/pipeline_x/pipeline_x_fragment_context.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/runtime_query_statistics_mgr.h" @@ -430,20 +429,16 @@ QueryContext::_collect_realtime_query_profile_x() const { for (auto& [fragment_id, fragment_ctx_wptr] : _fragment_id_to_pipeline_ctx) { if (auto fragment_ctx = fragment_ctx_wptr.lock()) { - // In theory, cast result can not be nullptr since we have checked the pipeline X engine above - std::shared_ptr fragment_ctx_x = - std::dynamic_pointer_cast(fragment_ctx); - - if (fragment_ctx_x == nullptr) { + if (fragment_ctx == nullptr) { std::string msg = - fmt::format("PipelineXFragmentContext is nullptr, query {} fragment_id: {}", + fmt::format("PipelineFragmentContext is nullptr, query {} fragment_id: {}", print_id(_query_id), fragment_id); LOG_ERROR(msg); DCHECK(false) << msg; continue; } - auto profile = fragment_ctx_x->collect_realtime_profile_x(); + auto profile = fragment_ctx->collect_realtime_profile_x(); if (profile.empty()) { std::string err_msg = fmt::format( diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 2713ee441dd0df..4df90fd798af24 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -141,7 +141,7 @@ RuntimeState::RuntimeState(const TUniqueId& instance_id, const TUniqueId& query_ query_id, RuntimeFilterParamsContext::create(this), _query_mem_tracker)); } -RuntimeState::RuntimeState(pipeline::PipelineXFragmentContext*, const TUniqueId& instance_id, +RuntimeState::RuntimeState(pipeline::PipelineFragmentContext*, const TUniqueId& instance_id, const TUniqueId& query_id, int32_t fragment_id, const TQueryOptions& query_options, const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx) diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index b266b76778f672..025d9517c28ef4 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -49,7 +49,7 @@ class IRuntimeFilter; namespace pipeline { class PipelineXLocalStateBase; class PipelineXSinkLocalStateBase; -class PipelineXFragmentContext; +class PipelineFragmentContext; class PipelineXTask; } // namespace pipeline @@ -76,7 +76,7 @@ class RuntimeState { ExecEnv* exec_env, QueryContext* ctx); // for only use in pipelineX - RuntimeState(pipeline::PipelineXFragmentContext*, const TUniqueId& instance_id, + RuntimeState(pipeline::PipelineFragmentContext*, const TUniqueId& instance_id, const TUniqueId& query_id, int32 fragment_id, const TQueryOptions& query_options, const TQueryGlobals& query_globals, ExecEnv* exec_env, QueryContext* ctx); @@ -662,7 +662,7 @@ class RuntimeState { // runtime filter std::unique_ptr _runtime_filter_mgr; - // owned by PipelineXFragmentContext + // owned by PipelineFragmentContext RuntimeFilterMgr* _pipeline_x_runtime_filter_mgr = nullptr; // Data stream receivers created by a plan fragment are gathered here to make sure diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index c82346f040ec82..05e38b973c6057 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -361,9 +361,8 @@ void WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e } auto task_queue = std::make_shared(executors_size); std::unique_ptr pipeline_task_scheduler = - std::make_unique( - exec_env, exec_env->get_global_block_scheduler(), std::move(task_queue), - "Pipe_" + tg_name, cg_cpu_ctl_ptr); + std::make_unique(exec_env, std::move(task_queue), + "Pipe_" + tg_name, cg_cpu_ctl_ptr); Status ret = pipeline_task_scheduler->start(); if (ret.ok()) { _task_sched = std::move(pipeline_task_scheduler); From 677158931ba9c0ba647291c6b7c4f45fd98175df Mon Sep 17 00:00:00 2001 From: xzj7019 <131111794+xzj7019@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:27:51 +0800 Subject: [PATCH 055/163] [opt](Nereids) bucket shuffle downgrade expansion (#34088) Expand bucket shuffle downgrade condition, which originally requiring a single partition after pruning, basic table and bucket number < para number. Currently, we expect this option can be used for disabling bucket shuffle more efficiently, without above restrictions. Co-authored-by: zhongjian.xzj --- .../ChildrenPropertiesRegulator.java | 39 +-- .../bs_downgrade_shape/query13.out | 40 +++ .../bs_downgrade_shape/query19.out | 41 +++ .../bs_downgrade_shape/query44.out | 75 ++++++ .../bs_downgrade_shape/query45.out | 38 +++ .../bs_downgrade_shape/query54.out | 84 ++++++ .../bs_downgrade_shape/query56.out | 95 +++++++ .../bs_downgrade_shape/query6.out | 54 ++++ .../bs_downgrade_shape/query61.out | 83 ++++++ .../bs_downgrade_shape/query68.out | 47 ++++ .../bs_downgrade_shape/query8.out | 47 ++++ .../bs_downgrade_shape/query91.out | 46 ++++ .../bs_downgrade_shape/query95.out | 55 ++++ .../bs_downgrade_shape/query13.groovy | 136 ++++++++++ .../bs_downgrade_shape/query19.groovy | 82 ++++++ .../bs_downgrade_shape/query44.groovy | 102 +++++++ .../bs_downgrade_shape/query45.groovy | 72 +++++ .../bs_downgrade_shape/query54.groovy | 144 ++++++++++ .../bs_downgrade_shape/query56.groovy | 170 ++++++++++++ .../bs_downgrade_shape/query6.groovy | 84 ++++++ .../bs_downgrade_shape/query61.groovy | 120 +++++++++ .../bs_downgrade_shape/query68.groovy | 116 ++++++++ .../bs_downgrade_shape/query8.groovy | 248 ++++++++++++++++++ .../bs_downgrade_shape/query91.groovy | 94 +++++++ .../bs_downgrade_shape/query95.groovy | 96 +++++++ 25 files changed, 2173 insertions(+), 35 deletions(-) create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out create mode 100644 regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.out create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy create mode 100644 regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index 366730f7dc521e..8cae2c8775cafd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -30,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinction; import org.apache.doris.nereids.trees.plans.AggMode; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.SortPhase; @@ -40,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; -import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; @@ -209,38 +207,12 @@ public Boolean visitPhysicalFilter(PhysicalFilter filter, Void c return true; } - private boolean isBucketShuffleDownGrade(Plan oneSidePlan, DistributionSpecHash otherSideSpec) { - // improper to do bucket shuffle join: - // oneSide: - // 1. base table - // 2. single partition after pruning - // 3. tablets' number is small enough (< paraInstanceNum) - // otherSide: ShuffleType.EXECUTION_BUCKETED + private boolean isBucketShuffleDownGrade(DistributionSpecHash srcSideSpec) { boolean isBucketShuffleDownGrade = ConnectContext.get().getSessionVariable().isEnableBucketShuffleDownGrade(); if (!isBucketShuffleDownGrade) { return false; - } else if (otherSideSpec.getShuffleType() != ShuffleType.EXECUTION_BUCKETED) { - return false; } else { - int paraNum = Math.max(1, ConnectContext.get().getSessionVariable().getParallelExecInstanceNum()); - if (((GroupPlan) oneSidePlan).getGroup().getPhysicalExpressions().isEmpty()) { - return false; - } else { - Plan plan = ((GroupPlan) oneSidePlan).getGroup().getPhysicalExpressions().get(0).getPlan(); - while ((plan instanceof PhysicalProject || plan instanceof PhysicalFilter) - && !((GroupPlan) plan.child(0)).getGroup().getPhysicalExpressions().isEmpty()) { - plan = ((GroupPlan) plan.child(0)).getGroup().getPhysicalExpressions().get(0).getPlan(); - } - if (plan != null && plan instanceof PhysicalOlapScan - && ((PhysicalOlapScan) plan).getSelectedPartitionIds().size() <= 1 - && ((PhysicalOlapScan) plan).getTable() != null - && ((PhysicalOlapScan) plan).getTable().getDefaultDistributionInfo() != null - && ((PhysicalOlapScan) plan).getTable().getDefaultDistributionInfo().getBucketNum() < paraNum) { - return true; - } else { - return false; - } - } + return srcSideSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED; } } @@ -262,9 +234,6 @@ public Boolean visitPhysicalHashJoin(PhysicalHashJoin[s_store_sk] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------PhysicalOlapScan[store] apply RFs: RF4 +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((household_demographics.hd_dep_count = 1) AND ((((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) OR ((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) AND (household_demographics.hd_dep_count = 3)))) build RFs:RF3 ss_cdemo_sk->[cd_demo_sk] +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalProject +----------------------filter(((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'College')) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Primary'))) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = '2 yr Degree')))) +------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3 +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] +------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) +----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) +----------------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------PhysicalProject +------------------------------filter((date_dim.d_year = 2001)) +--------------------------------PhysicalOlapScan[date_dim] +------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------PhysicalProject +----------------------------filter(hd_dep_count IN (1, 3)) +------------------------------PhysicalOlapScan[household_demographics] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.out new file mode 100644 index 00000000000000..6b7d023e3be42f --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_19 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------PhysicalProject +----------hashAgg[GLOBAL] +------------PhysicalDistribute[DistributionSpecHash] +--------------hashAgg[LOCAL] +----------------PhysicalProject +------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 s_store_sk->[ss_store_sk] +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->[ca_address_sk] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[customer_address] apply RFs: RF3 +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalProject +----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 ss_customer_sk->[c_customer_sk] +------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[customer] apply RFs: RF2 +------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[ss_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF4 +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------PhysicalProject +------------------------------------------filter((item.i_manager_id = 14)) +--------------------------------------------PhysicalOlapScan[item] +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.out new file mode 100644 index 00000000000000..ea4ea67293e48b --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_44 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------PhysicalProject +----------hashJoin[INNER_JOIN] hashCondition=((asceding.rnk = descending.rnk)) otherCondition=() +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN] hashCondition=((i1.i_item_sk = asceding.item_sk)) otherCondition=() build RFs:RF1 item_sk->[i_item_sk] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] apply RFs: RF1 +------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------PhysicalProject +----------------------filter((rnk < 11)) +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[MERGE_SORT] +----------------------------PhysicalDistribute[DistributionSpecGather] +------------------------------PhysicalQuickSort[LOCAL_SORT] +--------------------------------PhysicalPartitionTopN +----------------------------------PhysicalProject +------------------------------------NestedLoopJoin[INNER_JOIN](cast(rank_col as DOUBLE) > cast((0.9 * rank_col) as DOUBLE)) +--------------------------------------PhysicalProject +----------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------------------hashAgg[LOCAL] +----------------------------------------------PhysicalProject +------------------------------------------------filter((ss1.ss_store_sk = 4)) +--------------------------------------------------PhysicalOlapScan[store_sales] +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------PhysicalProject +------------------------------------------PhysicalAssertNumRows +--------------------------------------------PhysicalDistribute[DistributionSpecGather] +----------------------------------------------PhysicalProject +------------------------------------------------hashAgg[GLOBAL] +--------------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------------hashAgg[LOCAL] +------------------------------------------------------PhysicalProject +--------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +----------------------------------------------------------PhysicalOlapScan[store_sales] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->[i_item_sk] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] apply RFs: RF0 +------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------PhysicalProject +----------------------filter((rnk < 11)) +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[MERGE_SORT] +----------------------------PhysicalDistribute[DistributionSpecGather] +------------------------------PhysicalQuickSort[LOCAL_SORT] +--------------------------------PhysicalPartitionTopN +----------------------------------PhysicalProject +------------------------------------NestedLoopJoin[INNER_JOIN](cast(rank_col as DOUBLE) > cast((0.9 * rank_col) as DOUBLE)) +--------------------------------------PhysicalProject +----------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------------------hashAgg[LOCAL] +----------------------------------------------PhysicalProject +------------------------------------------------filter((ss1.ss_store_sk = 4)) +--------------------------------------------------PhysicalOlapScan[store_sales] +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------PhysicalProject +------------------------------------------PhysicalAssertNumRows +--------------------------------------------PhysicalDistribute[DistributionSpecGather] +----------------------------------------------PhysicalProject +------------------------------------------------hashAgg[GLOBAL] +--------------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------------hashAgg[LOCAL] +------------------------------------------------------PhysicalProject +--------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +----------------------------------------------------------PhysicalOlapScan[store_sales] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out new file mode 100644 index 00000000000000..091bda5f39c326 --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.out @@ -0,0 +1,38 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_45 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1)) +------------------hashJoin[INNER_JOIN] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->[ca_address_sk] +--------------------PhysicalProject +----------------------PhysicalOlapScan[customer_address] apply RFs: RF3 +--------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 ws_bill_customer_sk->[c_customer_sk] +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[customer] apply RFs: RF2 +--------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk] +------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[ws_item_sk] +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------PhysicalProject +------------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[item] +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------PhysicalProject +------------------------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------------------------PhysicalOlapScan[item] +------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_qoy = 1) and (date_dim.d_year = 2000)) +------------------------------------PhysicalOlapScan[date_dim] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.out new file mode 100644 index 00000000000000..be67d64e1d3f4e --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.out @@ -0,0 +1,84 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_54 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------PhysicalProject +----------hashAgg[GLOBAL] +------------PhysicalDistribute[DistributionSpecHash] +--------------hashAgg[LOCAL] +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= (d_month_seq + 3)) +----------------------------PhysicalProject +------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) >= (d_month_seq + 1)) +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[ss_sold_date_sk] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN] hashCondition=((my_customers.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ss_customer_sk] +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 +----------------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_address.ca_county = store.s_county) and (customer_address.ca_state = store.s_state)) otherCondition=() build RFs:RF4 s_county->[ca_county];RF5 s_state->[ca_state] +----------------------------------------------hashJoin[INNER_JOIN] hashCondition=((my_customers.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->[ca_address_sk] +------------------------------------------------PhysicalProject +--------------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3 RF4 RF5 +------------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------------------PhysicalProject +----------------------------------------------------hashAgg[GLOBAL] +------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------------------------------hashAgg[LOCAL] +----------------------------------------------------------PhysicalProject +------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer.c_customer_sk = cs_or_ws_sales.customer_sk)) otherCondition=() build RFs:RF2 customer_sk->[c_customer_sk] +--------------------------------------------------------------PhysicalProject +----------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF2 +--------------------------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------------------------------PhysicalProject +------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((cs_or_ws_sales.sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk,ws_sold_date_sk] +--------------------------------------------------------------------PhysicalProject +----------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((cs_or_ws_sales.item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[cs_item_sk,ws_item_sk] +------------------------------------------------------------------------PhysicalUnion +--------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------------------------------------PhysicalProject +------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 +--------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------------------------------------------PhysicalProject +------------------------------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 +------------------------------------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------------------------------------------PhysicalProject +----------------------------------------------------------------------------filter((item.i_category = 'Music') and (item.i_class = 'country')) +------------------------------------------------------------------------------PhysicalOlapScan[item] +--------------------------------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------------------------------------PhysicalProject +------------------------------------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +--------------------------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------------------PhysicalProject +--------------------------------------------------PhysicalOlapScan[store] +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------PhysicalAssertNumRows +------------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------------hashAgg[GLOBAL] +----------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------hashAgg[LOCAL] +--------------------------------------------PhysicalProject +----------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------PhysicalAssertNumRows +--------------------------------PhysicalDistribute[DistributionSpecGather] +----------------------------------hashAgg[GLOBAL] +------------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------------hashAgg[LOCAL] +----------------------------------------PhysicalProject +------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 1999)) +--------------------------------------------PhysicalOlapScan[date_dim] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.out new file mode 100644 index 00000000000000..b74545fe8068f4 --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.out @@ -0,0 +1,95 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_56 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3 +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000)) +------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=() build RFs:RF0 i_item_id->[i_item_id] +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[item] apply RFs: RF0 +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_gmt_offset = -6.00)) +----------------------------------PhysicalOlapScan[customer_address] +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk] +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7 +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000)) +------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=() build RFs:RF4 i_item_id->[i_item_id] +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[item] apply RFs: RF4 +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_gmt_offset = -6.00)) +----------------------------------PhysicalOlapScan[customer_address] +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ws_bill_addr_sk->[ca_address_sk] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------filter((customer_address.ca_gmt_offset = -6.00)) +----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF11 +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[ws_sold_date_sk] +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF9 RF10 +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2000)) +------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=() build RFs:RF8 i_item_id->[i_item_id] +------------------------------------PhysicalProject +--------------------------------------PhysicalOlapScan[item] apply RFs: RF8 +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------------PhysicalOlapScan[item] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.out new file mode 100644 index 00000000000000..b2169d25149efb --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_6 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------PhysicalProject +----------filter((cnt >= 10)) +------------hashAgg[GLOBAL] +--------------PhysicalDistribute[DistributionSpecHash] +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN] hashCondition=((a.ca_address_sk = c.c_current_addr_sk)) otherCondition=() build RFs:RF5 c_current_addr_sk->[ca_address_sk] +----------------------PhysicalDistribute[DistributionSpecHash] +------------------------PhysicalProject +--------------------------PhysicalOlapScan[customer_address] apply RFs: RF5 +----------------------PhysicalDistribute[DistributionSpecHash] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_customer_sk = s.ss_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->[c_customer_sk] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[customer] apply RFs: RF4 +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN] hashCondition=((s.ss_item_sk = i.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk] +----------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN] hashCondition=((s.ss_sold_date_sk = d.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk] +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 +----------------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------------hashJoin[INNER_JOIN] hashCondition=((d.d_month_seq = date_dim.d_month_seq)) otherCondition=() build RFs:RF1 d_month_seq->[d_month_seq] +--------------------------------------------PhysicalProject +----------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF1 +--------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------------PhysicalAssertNumRows +------------------------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------------------------hashAgg[GLOBAL] +----------------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------------hashAgg[LOCAL] +--------------------------------------------------------PhysicalProject +----------------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) +------------------------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(cast(i_current_price as DECIMALV3(9, 4)))))) build RFs:RF0 i_category->[i_category] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[item] apply RFs: RF0 +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------hashAgg[GLOBAL] +------------------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------------------hashAgg[LOCAL] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[item] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out new file mode 100644 index 00000000000000..982462b51cff28 --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.out @@ -0,0 +1,83 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_61 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalTopN[LOCAL_SORT] +------PhysicalProject +--------NestedLoopJoin[CROSS_JOIN] +----------hashAgg[GLOBAL] +------------PhysicalDistribute[DistributionSpecGather] +--------------hashAgg[LOCAL] +----------------PhysicalProject +------------------hashJoin[INNER_JOIN] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF10 c_current_addr_sk->[ca_address_sk] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------PhysicalProject +------------------------filter((customer_address.ca_gmt_offset = -7.00)) +--------------------------PhysicalOlapScan[customer_address] apply RFs: RF10 +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF9 ss_customer_sk->[c_customer_sk] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer] apply RFs: RF9 +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF8 s_store_sk->[ss_store_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF7 p_promo_sk->[ss_promo_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[ss_item_sk] +--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk] +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF5 RF6 RF7 RF8 +----------------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------------PhysicalProject +--------------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) +----------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------PhysicalProject +------------------------------------------filter((item.i_category = 'Home')) +--------------------------------------------PhysicalOlapScan[item] +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter((((promotion.p_channel_dmail = 'Y') OR (promotion.p_channel_email = 'Y')) OR (promotion.p_channel_tv = 'Y'))) +------------------------------------------PhysicalOlapScan[promotion] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------PhysicalProject +------------------------------------filter((store.s_gmt_offset = -7.00)) +--------------------------------------PhysicalOlapScan[store] +----------PhysicalDistribute[DistributionSpecReplicated] +------------hashAgg[GLOBAL] +--------------PhysicalDistribute[DistributionSpecGather] +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3 RF4 +----------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------PhysicalProject +--------------------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) +----------------------------------------PhysicalOlapScan[date_dim] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------PhysicalProject +------------------------------------filter((item.i_category = 'Home')) +--------------------------------------PhysicalOlapScan[item] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashJoin[INNER_JOIN] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[customer] apply RFs: RF0 +------------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------------PhysicalProject +----------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +------------------------------------PhysicalOlapScan[customer_address] +----------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------PhysicalProject +--------------------------filter((store.s_gmt_offset = -7.00)) +----------------------------PhysicalOlapScan[store] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out new file mode 100644 index 00000000000000..dc8b5303dfd5d5 --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.out @@ -0,0 +1,47 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_68 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------PhysicalProject +----------hashJoin[INNER_JOIN] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->[ca_address_sk] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------PhysicalOlapScan[customer_address] apply RFs: RF5 +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->[c_customer_sk] +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalProject +----------------------PhysicalOlapScan[customer] apply RFs: RF4 +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalProject +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ss_addr_sk->[ca_address_sk] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3 +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] +--------------------------------------PhysicalProject +----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +------------------------------------------PhysicalProject +--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +------------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------------PhysicalProject +----------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------------PhysicalOlapScan[date_dim] +--------------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------------PhysicalProject +------------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------------PhysicalOlapScan[store] +----------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------PhysicalProject +--------------------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_vehicle_count = 4))) +----------------------------------------PhysicalOlapScan[household_demographics] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.out new file mode 100644 index 00000000000000..aa750d8681807f --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.out @@ -0,0 +1,47 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_8 -- +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalTopN[LOCAL_SORT] +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN] hashCondition=((expr_substring(s_zip, 1, 2) = expr_substring(ca_zip, 1, 2))) otherCondition=() +------------------PhysicalProject +--------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 +--------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------PhysicalProject +------------------------------filter((date_dim.d_qoy = 2) and (date_dim.d_year = 1998)) +--------------------------------PhysicalOlapScan[date_dim] +----------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------PhysicalProject +--------------------------PhysicalOlapScan[store] +------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------PhysicalProject +----------------------PhysicalIntersect +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalProject +----------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +------------------------------PhysicalOlapScan[customer_address] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalProject +----------------------------filter((cnt > 10)) +------------------------------hashAgg[GLOBAL] +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------hashAgg[LOCAL] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 c_current_addr_sk->[ca_address_sk] +----------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------PhysicalProject +--------------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 +----------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------PhysicalProject +--------------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) +----------------------------------------------PhysicalOlapScan[customer] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out new file mode 100644 index 00000000000000..42598f25208596 --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.out @@ -0,0 +1,46 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_91 -- +PhysicalResultSink +--PhysicalQuickSort[MERGE_SORT] +----PhysicalDistribute[DistributionSpecGather] +------PhysicalQuickSort[LOCAL_SORT] +--------PhysicalProject +----------hashAgg[GLOBAL] +------------PhysicalDistribute[DistributionSpecHash] +--------------hashAgg[LOCAL] +----------------PhysicalProject +------------------hashJoin[INNER_JOIN] hashCondition=((catalog_returns.cr_call_center_sk = call_center.cc_call_center_sk)) otherCondition=() build RFs:RF5 cc_call_center_sk->[cr_call_center_sk] +--------------------PhysicalProject +----------------------hashJoin[INNER_JOIN] hashCondition=((catalog_returns.cr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cr_returned_date_sk] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_returns.cr_returning_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cr_returning_customer_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF3 RF4 RF5 +----------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF2 c_current_addr_sk->[ca_address_sk] +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------PhysicalProject +------------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +--------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF2 +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------hashJoin[INNER_JOIN] hashCondition=((household_demographics.hd_demo_sk = customer.c_current_hdemo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[c_current_hdemo_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = customer.c_current_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[c_current_cdemo_sk] +--------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1 +--------------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------------PhysicalProject +------------------------------------------filter((((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = 'Unknown')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'Advanced Degree')))) +--------------------------------------------PhysicalOlapScan[customer_demographics] +------------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------------PhysicalProject +----------------------------------------filter((hd_buy_potential like 'Unknown%')) +------------------------------------------PhysicalOlapScan[household_demographics] +------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------PhysicalProject +----------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) +------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------PhysicalProject +------------------------PhysicalOlapScan[call_center] + diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.out new file mode 100644 index 00000000000000..476a65baed6102 --- /dev/null +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ds_shape_95 -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalProject +------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->[ws_order_number] +--------PhysicalDistribute[DistributionSpecHash] +----------PhysicalProject +------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF7 +--------PhysicalDistribute[DistributionSpecHash] +----------PhysicalProject +------------PhysicalOlapScan[web_sales] apply RFs: RF7 +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalTopN[LOCAL_SORT] +--------hashAgg[DISTINCT_GLOBAL] +----------PhysicalDistribute[DistributionSpecGather] +------------hashAgg[DISTINCT_LOCAL] +--------------hashAgg[GLOBAL] +----------------hashAgg[LOCAL] +------------------PhysicalProject +--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF6 ws_order_number->[wr_order_number,ws_order_number] +----------------------PhysicalDistribute[DistributionSpecHash] +------------------------PhysicalProject +--------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number] +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF5 RF6 +----------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 +----------------------PhysicalProject +------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF7 ws_order_number->[ws_order_number,ws_order_number] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk] +------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk] +--------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk] +----------------------------------PhysicalProject +------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3 +----------------------------------PhysicalDistribute[DistributionSpecReplicated] +------------------------------------PhysicalProject +--------------------------------------filter((customer_address.ca_state = 'VA')) +----------------------------------------PhysicalOlapScan[customer_address] +--------------------------------PhysicalDistribute[DistributionSpecReplicated] +----------------------------------PhysicalProject +------------------------------------filter((date_dim.d_date <= '2001-05-31') and (date_dim.d_date >= '2001-04-01')) +--------------------------------------PhysicalOlapScan[date_dim] +------------------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------------------PhysicalProject +----------------------------------filter((web_site.web_company_name = 'pri')) +------------------------------------PhysicalOlapScan[web_site] + diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy new file mode 100644 index 00000000000000..5718fe49c2774a --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query13.groovy @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query13") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select avg(ss_quantity) + ,avg(ss_ext_sales_price) + ,avg(ss_ext_wholesale_cost) + ,sum(ss_ext_wholesale_cost) + from store_sales + ,store + ,customer_demographics + ,household_demographics + ,customer_address + ,date_dim + where s_store_sk = ss_store_sk + and ss_sold_date_sk = d_date_sk and d_year = 2001 + and((ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'M' + and cd_education_status = 'College' + and ss_sales_price between 100.00 and 150.00 + and hd_dep_count = 3 + )or + (ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'D' + and cd_education_status = 'Primary' + and ss_sales_price between 50.00 and 100.00 + and hd_dep_count = 1 + ) or + (ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'W' + and cd_education_status = '2 yr Degree' + and ss_sales_price between 150.00 and 200.00 + and hd_dep_count = 1 + )) + and((ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('IL', 'TN', 'TX') + and ss_net_profit between 100 and 200 + ) or + (ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('WY', 'OH', 'ID') + and ss_net_profit between 150 and 300 + ) or + (ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('MS', 'SC', 'IA') + and ss_net_profit between 50 and 250 + )) +""" + qt_ds_shape_13 ''' + explain shape plan + select avg(ss_quantity) + ,avg(ss_ext_sales_price) + ,avg(ss_ext_wholesale_cost) + ,sum(ss_ext_wholesale_cost) + from store_sales + ,store + ,customer_demographics + ,household_demographics + ,customer_address + ,date_dim + where s_store_sk = ss_store_sk + and ss_sold_date_sk = d_date_sk and d_year = 2001 + and((ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'M' + and cd_education_status = 'College' + and ss_sales_price between 100.00 and 150.00 + and hd_dep_count = 3 + )or + (ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'D' + and cd_education_status = 'Primary' + and ss_sales_price between 50.00 and 100.00 + and hd_dep_count = 1 + ) or + (ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'W' + and cd_education_status = '2 yr Degree' + and ss_sales_price between 150.00 and 200.00 + and hd_dep_count = 1 + )) + and((ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('IL', 'TN', 'TX') + and ss_net_profit between 100 and 200 + ) or + (ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('WY', 'OH', 'ID') + and ss_net_profit between 150 and 300 + ) or + (ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('MS', 'SC', 'IA') + and ss_net_profit between 50 and 250 + )) + + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy new file mode 100644 index 00000000000000..322c84b782b6b4 --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query19.groovy @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query19") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, + sum(ss_ext_sales_price) ext_price + from date_dim, store_sales, item,customer,customer_address,store + where d_date_sk = ss_sold_date_sk + and ss_item_sk = i_item_sk + and i_manager_id=14 + and d_moy=11 + and d_year=2002 + and ss_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and substr(ca_zip,1,5) <> substr(s_zip,1,5) + and ss_store_sk = s_store_sk + group by i_brand + ,i_brand_id + ,i_manufact_id + ,i_manufact + order by ext_price desc + ,i_brand + ,i_brand_id + ,i_manufact_id + ,i_manufact +limit 100 """ + qt_ds_shape_19 ''' + explain shape plan + select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, + sum(ss_ext_sales_price) ext_price + from date_dim, store_sales, item,customer,customer_address,store + where d_date_sk = ss_sold_date_sk + and ss_item_sk = i_item_sk + and i_manager_id=14 + and d_moy=11 + and d_year=2002 + and ss_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and substr(ca_zip,1,5) <> substr(s_zip,1,5) + and ss_store_sk = s_store_sk + group by i_brand + ,i_brand_id + ,i_manufact_id + ,i_manufact + order by ext_price desc + ,i_brand + ,i_brand_id + ,i_manufact_id + ,i_manufact +limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy new file mode 100644 index 00000000000000..309c650df16f85 --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query44.groovy @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query44") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing +from(select * + from (select item_sk,rank() over (order by rank_col asc) rnk + from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + from store_sales ss1 + where ss_store_sk = 4 + group by ss_item_sk + having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + from store_sales + where ss_store_sk = 4 + and ss_hdemo_sk is null + group by ss_store_sk))V1)V11 + where rnk < 11) asceding, + (select * + from (select item_sk,rank() over (order by rank_col desc) rnk + from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + from store_sales ss1 + where ss_store_sk = 4 + group by ss_item_sk + having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + from store_sales + where ss_store_sk = 4 + and ss_hdemo_sk is null + group by ss_store_sk))V2)V21 + where rnk < 11) descending, +item i1, +item i2 +where asceding.rnk = descending.rnk + and i1.i_item_sk=asceding.item_sk + and i2.i_item_sk=descending.item_sk +order by asceding.rnk +limit 100""" + qt_ds_shape_44 ''' + explain shape plan + select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing +from(select * + from (select item_sk,rank() over (order by rank_col asc) rnk + from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + from store_sales ss1 + where ss_store_sk = 4 + group by ss_item_sk + having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + from store_sales + where ss_store_sk = 4 + and ss_hdemo_sk is null + group by ss_store_sk))V1)V11 + where rnk < 11) asceding, + (select * + from (select item_sk,rank() over (order by rank_col desc) rnk + from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + from store_sales ss1 + where ss_store_sk = 4 + group by ss_item_sk + having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + from store_sales + where ss_store_sk = 4 + and ss_hdemo_sk is null + group by ss_store_sk))V2)V21 + where rnk < 11) descending, +item i1, +item i2 +where asceding.rnk = descending.rnk + and i1.i_item_sk=asceding.item_sk + and i2.i_item_sk=descending.item_sk +order by asceding.rnk +limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy new file mode 100644 index 00000000000000..90b08d167e9a14 --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query45.groovy @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query45") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select ca_zip, ca_city, sum(ws_sales_price) + from web_sales, customer, customer_address, date_dim, item + where ws_bill_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and ws_item_sk = i_item_sk + and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') + or + i_item_id in (select i_item_id + from item + where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) + ) + ) + and ws_sold_date_sk = d_date_sk + and d_qoy = 1 and d_year = 2000 + group by ca_zip, ca_city + order by ca_zip, ca_city + limit 100""" + qt_ds_shape_45 ''' + explain shape plan + select ca_zip, ca_city, sum(ws_sales_price) + from web_sales, customer, customer_address, date_dim, item + where ws_bill_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and ws_item_sk = i_item_sk + and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') + or + i_item_id in (select i_item_id + from item + where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) + ) + ) + and ws_sold_date_sk = d_date_sk + and d_qoy = 1 and d_year = 2000 + group by ca_zip, ca_city + order by ca_zip, ca_city + limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy new file mode 100644 index 00000000000000..67cc77943d67b4 --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query54.groovy @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query54") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """with my_customers as ( + select distinct c_customer_sk + , c_current_addr_sk + from + ( select cs_sold_date_sk sold_date_sk, + cs_bill_customer_sk customer_sk, + cs_item_sk item_sk + from catalog_sales + union all + select ws_sold_date_sk sold_date_sk, + ws_bill_customer_sk customer_sk, + ws_item_sk item_sk + from web_sales + ) cs_or_ws_sales, + item, + date_dim, + customer + where sold_date_sk = d_date_sk + and item_sk = i_item_sk + and i_category = 'Music' + and i_class = 'country' + and c_customer_sk = cs_or_ws_sales.customer_sk + and d_moy = 1 + and d_year = 1999 + ) + , my_revenue as ( + select c_customer_sk, + sum(ss_ext_sales_price) as revenue + from my_customers, + store_sales, + customer_address, + store, + date_dim + where c_current_addr_sk = ca_address_sk + and ca_county = s_county + and ca_state = s_state + and ss_sold_date_sk = d_date_sk + and c_customer_sk = ss_customer_sk + and d_month_seq between (select distinct d_month_seq+1 + from date_dim where d_year = 1999 and d_moy = 1) + and (select distinct d_month_seq+3 + from date_dim where d_year = 1999 and d_moy = 1) + group by c_customer_sk + ) + , segments as + (select cast((revenue/50) as int) as segment + from my_revenue + ) + select segment, count(*) as num_customers, segment*50 as segment_base + from segments + group by segment + order by segment, num_customers + limit 100""" + qt_ds_shape_54 ''' + explain shape plan + with my_customers as ( + select distinct c_customer_sk + , c_current_addr_sk + from + ( select cs_sold_date_sk sold_date_sk, + cs_bill_customer_sk customer_sk, + cs_item_sk item_sk + from catalog_sales + union all + select ws_sold_date_sk sold_date_sk, + ws_bill_customer_sk customer_sk, + ws_item_sk item_sk + from web_sales + ) cs_or_ws_sales, + item, + date_dim, + customer + where sold_date_sk = d_date_sk + and item_sk = i_item_sk + and i_category = 'Music' + and i_class = 'country' + and c_customer_sk = cs_or_ws_sales.customer_sk + and d_moy = 1 + and d_year = 1999 + ) + , my_revenue as ( + select c_customer_sk, + sum(ss_ext_sales_price) as revenue + from my_customers, + store_sales, + customer_address, + store, + date_dim + where c_current_addr_sk = ca_address_sk + and ca_county = s_county + and ca_state = s_state + and ss_sold_date_sk = d_date_sk + and c_customer_sk = ss_customer_sk + and d_month_seq between (select distinct d_month_seq+1 + from date_dim where d_year = 1999 and d_moy = 1) + and (select distinct d_month_seq+3 + from date_dim where d_year = 1999 and d_moy = 1) + group by c_customer_sk + ) + , segments as + (select cast((revenue/50) as int) as segment + from my_revenue + ) + select segment, count(*) as num_customers, segment*50 as segment_base + from segments + group by segment + order by segment, num_customers + limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy new file mode 100644 index 00000000000000..8eb5ff81c8b9d8 --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query56.groovy @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query56") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """with ss as ( + select i_item_id,sum(ss_ext_sales_price) total_sales + from + store_sales, + date_dim, + customer_address, + item + where i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and ss_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id), + cs as ( + select i_item_id,sum(cs_ext_sales_price) total_sales + from + catalog_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and cs_item_sk = i_item_sk + and cs_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and cs_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id), + ws as ( + select i_item_id,sum(ws_ext_sales_price) total_sales + from + web_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and ws_item_sk = i_item_sk + and ws_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and ws_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id) + select i_item_id ,sum(total_sales) total_sales + from (select * from ss + union all + select * from cs + union all + select * from ws) tmp1 + group by i_item_id + order by total_sales, + i_item_id + limit 100""" + qt_ds_shape_56 ''' + explain shape plan + with ss as ( + select i_item_id,sum(ss_ext_sales_price) total_sales + from + store_sales, + date_dim, + customer_address, + item + where i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and ss_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id), + cs as ( + select i_item_id,sum(cs_ext_sales_price) total_sales + from + catalog_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and cs_item_sk = i_item_sk + and cs_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and cs_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id), + ws as ( + select i_item_id,sum(ws_ext_sales_price) total_sales + from + web_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and ws_item_sk = i_item_sk + and ws_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and ws_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id) + select i_item_id ,sum(total_sales) total_sales + from (select * from ss + union all + select * from cs + union all + select * from ws) tmp1 + group by i_item_id + order by total_sales, + i_item_id + limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy new file mode 100644 index 00000000000000..8a4e9fc33be442 --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query6.groovy @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query6") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select a.ca_state state, count(*) cnt + from customer_address a + ,customer c + ,store_sales s + ,date_dim d + ,item i + where a.ca_address_sk = c.c_current_addr_sk + and c.c_customer_sk = s.ss_customer_sk + and s.ss_sold_date_sk = d.d_date_sk + and s.ss_item_sk = i.i_item_sk + and d.d_month_seq = + (select distinct (d_month_seq) + from date_dim + where d_year = 2002 + and d_moy = 3 ) + and i.i_current_price > 1.2 * + (select avg(j.i_current_price) + from item j + where j.i_category = i.i_category) + group by a.ca_state + having count(*) >= 10 + order by cnt, a.ca_state + limit 100""" + qt_ds_shape_6 ''' + explain shape plan + select a.ca_state state, count(*) cnt + from customer_address a + ,customer c + ,store_sales s + ,date_dim d + ,item i + where a.ca_address_sk = c.c_current_addr_sk + and c.c_customer_sk = s.ss_customer_sk + and s.ss_sold_date_sk = d.d_date_sk + and s.ss_item_sk = i.i_item_sk + and d.d_month_seq = + (select distinct (d_month_seq) + from date_dim + where d_year = 2002 + and d_moy = 3 ) + and i.i_current_price > 1.2 * + (select avg(j.i_current_price) + from item j + where j.i_category = i.i_category) + group by a.ca_state + having count(*) >= 10 + order by cnt, a.ca_state + limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy new file mode 100644 index 00000000000000..afc92b3a6d688b --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query61.groovy @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query61") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 +from + (select sum(ss_ext_sales_price) promotions + from store_sales + ,store + ,promotion + ,date_dim + ,customer + ,customer_address + ,item + where ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and ss_promo_sk = p_promo_sk + and ss_customer_sk= c_customer_sk + and ca_address_sk = c_current_addr_sk + and ss_item_sk = i_item_sk + and ca_gmt_offset = -7 + and i_category = 'Home' + and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') + and s_gmt_offset = -7 + and d_year = 2000 + and d_moy = 12) promotional_sales, + (select sum(ss_ext_sales_price) total + from store_sales + ,store + ,date_dim + ,customer + ,customer_address + ,item + where ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and ss_customer_sk= c_customer_sk + and ca_address_sk = c_current_addr_sk + and ss_item_sk = i_item_sk + and ca_gmt_offset = -7 + and i_category = 'Home' + and s_gmt_offset = -7 + and d_year = 2000 + and d_moy = 12) all_sales +order by promotions, total +limit 100""" + qt_ds_shape_61 ''' + explain shape plan + select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 +from + (select sum(ss_ext_sales_price) promotions + from store_sales + ,store + ,promotion + ,date_dim + ,customer + ,customer_address + ,item + where ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and ss_promo_sk = p_promo_sk + and ss_customer_sk= c_customer_sk + and ca_address_sk = c_current_addr_sk + and ss_item_sk = i_item_sk + and ca_gmt_offset = -7 + and i_category = 'Home' + and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') + and s_gmt_offset = -7 + and d_year = 2000 + and d_moy = 12) promotional_sales, + (select sum(ss_ext_sales_price) total + from store_sales + ,store + ,date_dim + ,customer + ,customer_address + ,item + where ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and ss_customer_sk= c_customer_sk + and ca_address_sk = c_current_addr_sk + and ss_item_sk = i_item_sk + and ca_gmt_offset = -7 + and i_category = 'Home' + and s_gmt_offset = -7 + and d_year = 2000 + and d_moy = 12) all_sales +order by promotions, total +limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy new file mode 100644 index 00000000000000..a9d1967d203c9a --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query68.groovy @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query68") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select c_last_name + ,c_first_name + ,ca_city + ,bought_city + ,ss_ticket_number + ,extended_price + ,extended_tax + ,list_price + from (select ss_ticket_number + ,ss_customer_sk + ,ca_city bought_city + ,sum(ss_ext_sales_price) extended_price + ,sum(ss_ext_list_price) list_price + ,sum(ss_ext_tax) extended_tax + from store_sales + ,date_dim + ,store + ,household_demographics + ,customer_address + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and store_sales.ss_addr_sk = customer_address.ca_address_sk + and date_dim.d_dom between 1 and 2 + and (household_demographics.hd_dep_count = 3 or + household_demographics.hd_vehicle_count= 4) + and date_dim.d_year in (1998,1998+1,1998+2) + and store.s_city in ('Fairview','Midway') + group by ss_ticket_number + ,ss_customer_sk + ,ss_addr_sk,ca_city) dn + ,customer + ,customer_address current_addr + where ss_customer_sk = c_customer_sk + and customer.c_current_addr_sk = current_addr.ca_address_sk + and current_addr.ca_city <> bought_city + order by c_last_name + ,ss_ticket_number + limit 100""" + qt_ds_shape_68 ''' + explain shape plan + select c_last_name + ,c_first_name + ,ca_city + ,bought_city + ,ss_ticket_number + ,extended_price + ,extended_tax + ,list_price + from (select ss_ticket_number + ,ss_customer_sk + ,ca_city bought_city + ,sum(ss_ext_sales_price) extended_price + ,sum(ss_ext_list_price) list_price + ,sum(ss_ext_tax) extended_tax + from store_sales + ,date_dim + ,store + ,household_demographics + ,customer_address + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and store_sales.ss_addr_sk = customer_address.ca_address_sk + and date_dim.d_dom between 1 and 2 + and (household_demographics.hd_dep_count = 3 or + household_demographics.hd_vehicle_count= 4) + and date_dim.d_year in (1998,1998+1,1998+2) + and store.s_city in ('Fairview','Midway') + group by ss_ticket_number + ,ss_customer_sk + ,ss_addr_sk,ca_city) dn + ,customer + ,customer_address current_addr + where ss_customer_sk = c_customer_sk + and customer.c_current_addr_sk = current_addr.ca_address_sk + and current_addr.ca_city <> bought_city + order by c_last_name + ,ss_ticket_number + limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy new file mode 100644 index 00000000000000..98a6e694c196ff --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query8.groovy @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query8") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select s_store_name + ,sum(ss_net_profit) + from store_sales + ,date_dim + ,store, + (select ca_zip + from ( + SELECT substr(ca_zip,1,5) ca_zip + FROM customer_address + WHERE substr(ca_zip,1,5) IN ( + '47602','16704','35863','28577','83910','36201', + '58412','48162','28055','41419','80332', + '38607','77817','24891','16226','18410', + '21231','59345','13918','51089','20317', + '17167','54585','67881','78366','47770', + '18360','51717','73108','14440','21800', + '89338','45859','65501','34948','25973', + '73219','25333','17291','10374','18829', + '60736','82620','41351','52094','19326', + '25214','54207','40936','21814','79077', + '25178','75742','77454','30621','89193', + '27369','41232','48567','83041','71948', + '37119','68341','14073','16891','62878', + '49130','19833','24286','27700','40979', + '50412','81504','94835','84844','71954', + '39503','57649','18434','24987','12350', + '86379','27413','44529','98569','16515', + '27287','24255','21094','16005','56436', + '91110','68293','56455','54558','10298', + '83647','32754','27052','51766','19444', + '13869','45645','94791','57631','20712', + '37788','41807','46507','21727','71836', + '81070','50632','88086','63991','20244', + '31655','51782','29818','63792','68605', + '94898','36430','57025','20601','82080', + '33869','22728','35834','29086','92645', + '98584','98072','11652','78093','57553', + '43830','71144','53565','18700','90209', + '71256','38353','54364','28571','96560', + '57839','56355','50679','45266','84680', + '34306','34972','48530','30106','15371', + '92380','84247','92292','68852','13338', + '34594','82602','70073','98069','85066', + '47289','11686','98862','26217','47529', + '63294','51793','35926','24227','14196', + '24594','32489','99060','49472','43432', + '49211','14312','88137','47369','56877', + '20534','81755','15794','12318','21060', + '73134','41255','63073','81003','73873', + '66057','51184','51195','45676','92696', + '70450','90669','98338','25264','38919', + '59226','58581','60298','17895','19489', + '52301','80846','95464','68770','51634', + '19988','18367','18421','11618','67975', + '25494','41352','95430','15734','62585', + '97173','33773','10425','75675','53535', + '17879','41967','12197','67998','79658', + '59130','72592','14851','43933','68101', + '50636','25717','71286','24660','58058', + '72991','95042','15543','33122','69280', + '11912','59386','27642','65177','17672', + '33467','64592','36335','54010','18767', + '63193','42361','49254','33113','33159', + '36479','59080','11855','81963','31016', + '49140','29392','41836','32958','53163', + '13844','73146','23952','65148','93498', + '14530','46131','58454','13376','13378', + '83986','12320','17193','59852','46081', + '98533','52389','13086','68843','31013', + '13261','60560','13443','45533','83583', + '11489','58218','19753','22911','25115', + '86709','27156','32669','13123','51933', + '39214','41331','66943','14155','69998', + '49101','70070','35076','14242','73021', + '59494','15782','29752','37914','74686', + '83086','34473','15751','81084','49230', + '91894','60624','17819','28810','63180', + '56224','39459','55233','75752','43639', + '55349','86057','62361','50788','31830', + '58062','18218','85761','60083','45484', + '21204','90229','70041','41162','35390', + '16364','39500','68908','26689','52868', + '81335','40146','11340','61527','61794', + '71997','30415','59004','29450','58117', + '69952','33562','83833','27385','61860', + '96435','48333','23065','32961','84919', + '61997','99132','22815','56600','68730', + '48017','95694','32919','88217','27116', + '28239','58032','18884','16791','21343', + '97462','18569','75660','15475') + intersect + select ca_zip + from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt + FROM customer_address, customer + WHERE ca_address_sk = c_current_addr_sk and + c_preferred_cust_flag='Y' + group by ca_zip + having count(*) > 10)A1)A2) V1 + where ss_store_sk = s_store_sk + and ss_sold_date_sk = d_date_sk + and d_qoy = 2 and d_year = 1998 + and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) + group by s_store_name + order by s_store_name + limit 100""" + qt_ds_shape_8 ''' + explain shape plan + select s_store_name + ,sum(ss_net_profit) + from store_sales + ,date_dim + ,store, + (select ca_zip + from ( + SELECT substr(ca_zip,1,5) ca_zip + FROM customer_address + WHERE substr(ca_zip,1,5) IN ( + '47602','16704','35863','28577','83910','36201', + '58412','48162','28055','41419','80332', + '38607','77817','24891','16226','18410', + '21231','59345','13918','51089','20317', + '17167','54585','67881','78366','47770', + '18360','51717','73108','14440','21800', + '89338','45859','65501','34948','25973', + '73219','25333','17291','10374','18829', + '60736','82620','41351','52094','19326', + '25214','54207','40936','21814','79077', + '25178','75742','77454','30621','89193', + '27369','41232','48567','83041','71948', + '37119','68341','14073','16891','62878', + '49130','19833','24286','27700','40979', + '50412','81504','94835','84844','71954', + '39503','57649','18434','24987','12350', + '86379','27413','44529','98569','16515', + '27287','24255','21094','16005','56436', + '91110','68293','56455','54558','10298', + '83647','32754','27052','51766','19444', + '13869','45645','94791','57631','20712', + '37788','41807','46507','21727','71836', + '81070','50632','88086','63991','20244', + '31655','51782','29818','63792','68605', + '94898','36430','57025','20601','82080', + '33869','22728','35834','29086','92645', + '98584','98072','11652','78093','57553', + '43830','71144','53565','18700','90209', + '71256','38353','54364','28571','96560', + '57839','56355','50679','45266','84680', + '34306','34972','48530','30106','15371', + '92380','84247','92292','68852','13338', + '34594','82602','70073','98069','85066', + '47289','11686','98862','26217','47529', + '63294','51793','35926','24227','14196', + '24594','32489','99060','49472','43432', + '49211','14312','88137','47369','56877', + '20534','81755','15794','12318','21060', + '73134','41255','63073','81003','73873', + '66057','51184','51195','45676','92696', + '70450','90669','98338','25264','38919', + '59226','58581','60298','17895','19489', + '52301','80846','95464','68770','51634', + '19988','18367','18421','11618','67975', + '25494','41352','95430','15734','62585', + '97173','33773','10425','75675','53535', + '17879','41967','12197','67998','79658', + '59130','72592','14851','43933','68101', + '50636','25717','71286','24660','58058', + '72991','95042','15543','33122','69280', + '11912','59386','27642','65177','17672', + '33467','64592','36335','54010','18767', + '63193','42361','49254','33113','33159', + '36479','59080','11855','81963','31016', + '49140','29392','41836','32958','53163', + '13844','73146','23952','65148','93498', + '14530','46131','58454','13376','13378', + '83986','12320','17193','59852','46081', + '98533','52389','13086','68843','31013', + '13261','60560','13443','45533','83583', + '11489','58218','19753','22911','25115', + '86709','27156','32669','13123','51933', + '39214','41331','66943','14155','69998', + '49101','70070','35076','14242','73021', + '59494','15782','29752','37914','74686', + '83086','34473','15751','81084','49230', + '91894','60624','17819','28810','63180', + '56224','39459','55233','75752','43639', + '55349','86057','62361','50788','31830', + '58062','18218','85761','60083','45484', + '21204','90229','70041','41162','35390', + '16364','39500','68908','26689','52868', + '81335','40146','11340','61527','61794', + '71997','30415','59004','29450','58117', + '69952','33562','83833','27385','61860', + '96435','48333','23065','32961','84919', + '61997','99132','22815','56600','68730', + '48017','95694','32919','88217','27116', + '28239','58032','18884','16791','21343', + '97462','18569','75660','15475') + intersect + select ca_zip + from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt + FROM customer_address, customer + WHERE ca_address_sk = c_current_addr_sk and + c_preferred_cust_flag='Y' + group by ca_zip + having count(*) > 10)A1)A2) V1 + where ss_store_sk = s_store_sk + and ss_sold_date_sk = d_date_sk + and d_qoy = 2 and d_year = 1998 + and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) + group by s_store_name + order by s_store_name + limit 100 + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy new file mode 100644 index 00000000000000..9fd084f69bebbc --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query91.groovy @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query91") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """select + cc_call_center_id Call_Center, + cc_name Call_Center_Name, + cc_manager Manager, + sum(cr_net_loss) Returns_Loss +from + call_center, + catalog_returns, + date_dim, + customer, + customer_address, + customer_demographics, + household_demographics +where + cr_call_center_sk = cc_call_center_sk +and cr_returned_date_sk = d_date_sk +and cr_returning_customer_sk= c_customer_sk +and cd_demo_sk = c_current_cdemo_sk +and hd_demo_sk = c_current_hdemo_sk +and ca_address_sk = c_current_addr_sk +and d_year = 2000 +and d_moy = 12 +and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') + or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +and hd_buy_potential like 'Unknown%' +and ca_gmt_offset = -7 +group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status +order by sum(cr_net_loss) desc""" + qt_ds_shape_91 ''' + explain shape plan + select + cc_call_center_id Call_Center, + cc_name Call_Center_Name, + cc_manager Manager, + sum(cr_net_loss) Returns_Loss +from + call_center, + catalog_returns, + date_dim, + customer, + customer_address, + customer_demographics, + household_demographics +where + cr_call_center_sk = cc_call_center_sk +and cr_returned_date_sk = d_date_sk +and cr_returning_customer_sk= c_customer_sk +and cd_demo_sk = c_current_cdemo_sk +and hd_demo_sk = c_current_hdemo_sk +and ca_address_sk = c_current_addr_sk +and d_year = 2000 +and d_moy = 12 +and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') + or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +and hd_buy_potential like 'Unknown%' +and ca_gmt_offset = -7 +group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status +order by sum(cr_net_loss) desc + ''' +} diff --git a/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy new file mode 100644 index 00000000000000..2837fea540febf --- /dev/null +++ b/regression-test/suites/nereids_tpcds_shape_sf1000_p0/bs_downgrade_shape/query95.groovy @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +suite("query95") { + String db = context.config.getDbNameByFile(new File(context.file.parent)) + sql "use ${db}" + sql 'set enable_nereids_planner=true' + sql 'set enable_fallback_to_original_planner=false' + sql 'set exec_mem_limit=21G' + sql 'set be_number_for_test=3' + sql 'set parallel_fragment_exec_instance_num=8; ' + sql 'set parallel_pipeline_task_num=8; ' + sql 'set forbid_unknown_col_stats=true' + sql 'set enable_nereids_timeout = false' + sql 'set enable_runtime_filter_prune=false' + sql 'set runtime_filter_type=8' + sql 'set dump_nereids_memo=false' + sql 'set enable_bucket_shuffle_downgrade=true' + def ds = """with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from web_sales ws1,web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + select + count(distinct ws_order_number) as "order count" + ,sum(ws_ext_ship_cost) as "total shipping cost" + ,sum(ws_net_profit) as "total net profit" +from + web_sales ws1 + ,date_dim + ,customer_address + ,web_site +where + d_date between '2001-4-01' and + (cast('2001-4-01' as date) + interval 60 day) +and ws1.ws_ship_date_sk = d_date_sk +and ws1.ws_ship_addr_sk = ca_address_sk +and ca_state = 'VA' +and ws1.ws_web_site_sk = web_site_sk +and web_company_name = 'pri' +and ws1.ws_order_number in (select ws_order_number + from ws_wh) +and ws1.ws_order_number in (select wr_order_number + from web_returns,ws_wh + where wr_order_number = ws_wh.ws_order_number) +order by count(distinct ws_order_number) +limit 100""" + qt_ds_shape_95 ''' + explain shape plan + with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from web_sales ws1,web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + select + count(distinct ws_order_number) as "order count" + ,sum(ws_ext_ship_cost) as "total shipping cost" + ,sum(ws_net_profit) as "total net profit" +from + web_sales ws1 + ,date_dim + ,customer_address + ,web_site +where + d_date between '2001-4-01' and + (cast('2001-4-01' as date) + interval 60 day) +and ws1.ws_ship_date_sk = d_date_sk +and ws1.ws_ship_addr_sk = ca_address_sk +and ca_state = 'VA' +and ws1.ws_web_site_sk = web_site_sk +and web_company_name = 'pri' +and ws1.ws_order_number in (select ws_order_number + from ws_wh) +and ws1.ws_order_number in (select wr_order_number + from web_returns,ws_wh + where wr_order_number = ws_wh.ws_order_number) +order by count(distinct ws_order_number) +limit 100 + ''' +} From ee4196d9d23c252fc35a287c04adf9d705e7637f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 26 Apr 2024 18:31:11 +0800 Subject: [PATCH 056/163] [Improvement](agg) Improve count distinct distribute keys (#33167) --- .../aggregate_function_simple_factory.cpp | 2 + .../aggregate_function_uniq.h | 2 +- ...aggregate_function_uniq_distribute_key.cpp | 73 +++++ .../aggregate_function_uniq_distribute_key.h | 253 ++++++++++++++++++ 4 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp create mode 100644 be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.h diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp index 00597b212befd0..d95d0ce6ccb90d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp @@ -40,6 +40,7 @@ void register_aggregate_function_count(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_count_by_enum(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_HLL_union_agg(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_uniq(AggregateFunctionSimpleFactory& factory); +void register_aggregate_function_uniq_distribute_key(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_bit(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_bitmap(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_quantile_state(AggregateFunctionSimpleFactory& factory); @@ -80,6 +81,7 @@ AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() { register_aggregate_function_count(instance); register_aggregate_function_count_by_enum(instance); register_aggregate_function_uniq(instance); + register_aggregate_function_uniq_distribute_key(instance); register_aggregate_function_bit(instance); register_aggregate_function_bitmap(instance); register_aggregate_function_group_array_intersect(instance); diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.h b/be/src/vec/aggregate_functions/aggregate_function_uniq.h index 2e8855134ebd31..58abd3842c21b2 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_uniq.h +++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.h @@ -75,7 +75,7 @@ struct AggregateFunctionUniqExactData { Set set; - static String get_name() { return "uniqExact"; } + static String get_name() { return "multi_distinct"; } }; namespace detail { diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp b/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp new file mode 100644 index 00000000000000..3bf979483b527c --- /dev/null +++ b/be/src/vec/aggregate_functions/aggregate_function_uniq_distribute_key.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/aggregate_functions/aggregate_function_uniq_distribute_key.h" + +#include + +#include "vec/aggregate_functions/aggregate_function_simple_factory.h" +#include "vec/aggregate_functions/factory_helpers.h" +#include "vec/aggregate_functions/helpers.h" + +namespace doris::vectorized { + +template