Merge remote-tracking branch 'origin/k8s-operator-doc' into k8s-opera…

…tor-doc

.github/workflows/auto_trigger_teamcity.yml

@@ -62,6 +62,7 @@ jobs:
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ " p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
             trigger_pipelines="Doris_DorisRegression_P0Regression  ${trigger_pipelines}"
+            if [[ "${comment_message}" = *"10" ]]; then repeat_times="10"; fi
           fi
           if [[ "${comment_message}" =~ "run" && "${comment_message}" =~ " pipelinex_p0" && ! "${comment_message}" =~ "Thanks for your contribution" ]]; then
             trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX  ${trigger_pipelines}"
@@ -133,7 +134,12 @@ jobs:
             if [ "_""${same_build_sign}" == "_false" ];then
               sleep 10s
               echo "there is no running build or queue build, so trigger a new !"
-              execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
+              echo "repeat_times: ${repeat_times}"
+              if [[ -n "${repeat_times}" ]]; then
+                execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${repeat_times}"
+              else
+                execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}"
+              fi
               echo "${execute_command}"
               eval "${execute_command}"
               echo "-----------------------------------------------------------------"

.github/workflows/codestyle.yaml → .github/workflows/checkstyle.yaml

@@ -27,11 +27,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           persist-credentials: false
           submodules: recursive
-          fetch-depth: 0
 
       - name: Paths Filter
         uses: ./.github/actions/paths-filter
@@ -49,7 +48,8 @@ jobs:
         with:
           maven-version: 3.8.4
 
-      - name: Run java spotless & checkstyle
+      - name: Run java checkstyle
         if: steps.filter.outputs.fe_changes == 'true'
         run:
-          cd fe && mvn clean spotless:check checkstyle:check
+          cd fe && mvn clean checkstyle:check
+

.github/workflows/clang-format.yml

@@ -31,14 +31,21 @@ jobs:
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths filter
         uses: ./.github/actions/paths-filter
@@ -49,6 +56,15 @@ jobs:
               - 'be/src/**'
               - 'be/test/**'
 
+      - name: Checkout clang-format-lint-action
+        run: |
+          rm -rf ./.github/actions/clang-format-lint-action
+          git clone https://github.com/DoozyX/clang-format-lint-action .github/actions/clang-format-lint-action
+
+          pushd .github/actions/clang-format-lint-action &>/dev/null
+          git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7
+          popd &>/dev/null
+
       - name: "Format it!"
         if: ${{ steps.filter.outputs.be_changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action

.github/workflows/code-checks.yml

@@ -27,21 +27,22 @@ jobs:
       - name: Checkout ${{ github.ref }} ( ${{ github.sha }} )
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
-      - name: Patch
+      - name: Checkout action-sh-checker
         run: |
-          pushd .github/actions/action-sh-checker >/dev/null
+          rm -rf ./.github/actions/action-sh-checker
+          git clone https://github.com/luizm/action-sh-checker .github/actions/action-sh-checker
+
+          pushd .github/actions/action-sh-checker &>/dev/null
+          git checkout 76ab0b22e1f194e4a582edc7969df6485c4e9246
           sed -i 's/\[ "$GITHUB_EVENT_NAME" == "pull_request" \]/\[\[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "pull_request_target" \]\]/' entrypoint.sh
-          popd >/dev/null
+          popd &>/dev/null
 
       - name: Run ShellCheck
         uses: ./.github/actions/action-sh-checker
@@ -63,7 +64,15 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
 
       - name: Paths Filter
         uses: ./.github/actions/paths-filter
@@ -117,14 +126,22 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Download
         uses: actions/download-artifact@v3
         with:
           name: compile_commands
           path: ./be/build_Release
 
+      - name: Checkout clang-tidy review
+        run: |
+          rm -rf ./.github/actions/clang-tidy-review
+          git clone https://github.com/ZedThree/clang-tidy-review .github/actions/clang-tidy-review
+
+          pushd .github/actions/clang-tidy-review &>/dev/null
+          git checkout 2c55ef8cfc9acb3715d433e58aea086dcec9b206
+          popd &>/dev/null
+
       - name: Run clang-tidy review
         uses: ./.github/actions/clang-tidy-review
         id: review

.github/workflows/license-eyes.yml

@@ -30,15 +30,12 @@ jobs:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
         if: ${{ github.event_name == 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Check License
         uses: apache/[email protected]

.gitmodules

@@ -4,9 +4,6 @@
 [submodule ".github/actions/get-workflow-origin"]
 	path = .github/actions/get-workflow-origin
 	url = https://github.com/potiuk/get-workflow-origin.git
-[submodule ".github/actions/clang-format-lint-action"]
-	path = .github/actions/clang-format-lint-action
-	url = https://github.com/DoozyX/clang-format-lint-action.git
 [submodule ".github/actions/setup-maven"]
 	path = .github/actions/setup-maven
 	url = https://github.com/stCarolas/setup-maven.git
@@ -19,12 +16,6 @@
 [submodule ".github/actions/ccache-action"]
 	path = .github/actions/ccache-action
 	url = https://github.com/hendrikmuhs/ccache-action
-[submodule ".github/actions/action-sh-checker"]
-	path = .github/actions/action-sh-checker
-	url = https://github.com/luizm/action-sh-checker
-[submodule ".github/actions/clang-tidy-review"]
-	path = .github/actions/clang-tidy-review
-	url = https://github.com/ZedThree/clang-tidy-review.git
 [submodule "be/src/apache-orc"]
 	path = be/src/apache-orc
 	url = https://github.com/apache/doris-thirdparty.git

be/src/agent/cgroup_cpu_ctl.cpp

@@ -54,6 +54,19 @@ void CgroupCpuCtl::update_cpu_hard_limit(int cpu_hard_limit) {
     }
 }
 
+void CgroupCpuCtl::update_cpu_soft_limit(int cpu_shares) {
+    if (!_init_succ) {
+        return;
+    }
+    std::lock_guard<std::shared_mutex> w_lock(_lock_mutex);
+    if (_cpu_shares != cpu_shares) {
+        Status ret = modify_cg_cpu_soft_limit_no_lock(cpu_shares);
+        if (ret.ok()) {
+            _cpu_shares = cpu_shares;
+        }
+    }
+}
+
 Status CgroupCpuCtl::write_cg_sys_file(std::string file_path, int value, std::string msg,
                                        bool is_append) {
     int fd = open(file_path.c_str(), is_append ? O_RDWR | O_APPEND : O_RDWR);
@@ -97,9 +110,11 @@ Status CgroupV1CpuCtl::init() {
         }
     }
 
-    // quota path
+    // quota file
     _cgroup_v1_cpu_tg_quota_file = _cgroup_v1_cpu_tg_path + "/cpu.cfs_quota_us";
-    // task path
+    // cpu.shares file
+    _cgroup_v1_cpu_tg_shares_file = _cgroup_v1_cpu_tg_path + "/cpu.shares";
+    // task file
     _cgroup_v1_cpu_tg_task_file = _cgroup_v1_cpu_tg_path + "/tasks";
     LOG(INFO) << "cgroup v1 cpu path init success"
               << ", query tg path=" << _cgroup_v1_cpu_tg_path
@@ -110,6 +125,11 @@ Status CgroupV1CpuCtl::init() {
     return Status::OK();
 }
 
+Status CgroupV1CpuCtl::modify_cg_cpu_soft_limit_no_lock(int cpu_shares) {
+    std::string msg = "modify cpu shares to " + std::to_string(cpu_shares);
+    return CgroupCpuCtl::write_cg_sys_file(_cgroup_v1_cpu_tg_shares_file, cpu_shares, msg, false);
+}
+
 Status CgroupV1CpuCtl::modify_cg_cpu_hard_limit_no_lock(int cpu_hard_limit) {
     int val = _cpu_cfs_period_us * _cpu_core_num * cpu_hard_limit / 100;
     std::string msg = "modify cpu quota value to " + std::to_string(val);

be/src/agent/cgroup_cpu_ctl.h

@@ -28,29 +28,40 @@
 
 namespace doris {
 
+// cgroup cpu.cfs_quota_us default value, it means disable cpu hard limit
+const static int CPU_HARD_LIMIT_DEFAULT_VALUE = -1;
+
+// cgroup cpu.shares default value
+const static uint64_t CPU_SOFT_LIMIT_DEFAULT_VALUE = 1024;
+
 class CgroupCpuCtl {
 public:
     virtual ~CgroupCpuCtl() = default;
     CgroupCpuCtl(uint64_t tg_id) { _tg_id = tg_id; }
 
     virtual Status init();
 
-    virtual Status modify_cg_cpu_hard_limit_no_lock(int cpu_hard_limit) = 0;
-
     virtual Status add_thread_to_cgroup() = 0;
 
     void update_cpu_hard_limit(int cpu_hard_limit);
 
+    void update_cpu_soft_limit(int cpu_shares);
+
 protected:
     Status write_cg_sys_file(std::string file_path, int value, std::string msg, bool is_append);
 
+    virtual Status modify_cg_cpu_hard_limit_no_lock(int cpu_hard_limit) = 0;
+
+    virtual Status modify_cg_cpu_soft_limit_no_lock(int cpu_shares) = 0;
+
     std::string _doris_cgroup_cpu_path;
     uint64_t _cpu_core_num = CpuInfo::num_cores();
     uint64_t _cpu_cfs_period_us = 100000;
     uint64_t _cpu_hard_limit = 0;
     std::shared_mutex _lock_mutex;
     bool _init_succ = false;
     uint64_t _tg_id; // workload group id
+    uint64_t _cpu_shares = 0;
 };
 
 /*
@@ -73,20 +84,25 @@ class CgroupCpuCtl {
     6 workload group quota file:
         /sys/fs/cgroup/cpu/{doris_home}/query/{workload group id}/cpu.cfs_quota_us
     
-     7 workload group tasks file:
+    7 workload group tasks file:
         /sys/fs/cgroup/cpu/{doris_home}/query/{workload group id}/tasks
+    
+    8 workload group cpu.shares file:
+    /sys/fs/cgroup/cpu/{doris_home}/query/{workload group id}/cpu.shares
 */
 class CgroupV1CpuCtl : public CgroupCpuCtl {
 public:
     CgroupV1CpuCtl(uint64_t tg_id) : CgroupCpuCtl(tg_id) {}
     Status init() override;
     Status modify_cg_cpu_hard_limit_no_lock(int cpu_hard_limit) override;
+    Status modify_cg_cpu_soft_limit_no_lock(int cpu_shares) override;
     Status add_thread_to_cgroup() override;
 
 private:
     std::string _cgroup_v1_cpu_query_path;
     std::string _cgroup_v1_cpu_tg_path; // workload group path
     std::string _cgroup_v1_cpu_tg_quota_file;
+    std::string _cgroup_v1_cpu_tg_shares_file;
     std::string _cgroup_v1_cpu_tg_task_file;
 };
 

be/src/common/config.cpp

@@ -457,7 +457,7 @@ DEFINE_Int32(webserver_num_workers, "48");
 // Period to update rate counters and sampling counters in ms.
 DEFINE_mInt32(periodic_counter_update_period_ms, "500");
 
-DEFINE_Bool(enable_single_replica_load, "false");
+DEFINE_Bool(enable_single_replica_load, "true");
 // Number of download workers for single replica load
 DEFINE_Int32(single_replica_load_download_num_workers, "64");
 
@@ -742,8 +742,6 @@ DEFINE_mDouble(tablet_version_graph_orphan_vertex_ratio, "0.1");
 
 // share delta writers when memtable_on_sink_node = true
 DEFINE_Bool(share_delta_writers, "true");
-// number of brpc stream per load
-DEFINE_Int32(num_streams_per_load, "5");
 // timeout for open load stream rpc in ms
 DEFINE_Int64(open_load_stream_timeout_ms, "500");
 
@@ -969,6 +967,9 @@ DEFINE_Bool(enable_workload_group_for_scan, "false");
 // Will remove after fully test.
 DEFINE_Bool(enable_index_apply_preds_except_leafnode_of_andnode, "true");
 
+DEFINE_mBool(enable_flatten_nested_for_variant, "false");
+DEFINE_mDouble(ratio_of_defaults_as_sparse_column, "0.95");
+
 // block file cache
 DEFINE_Bool(enable_file_cache, "false");
 // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
@@ -1013,8 +1014,6 @@ DEFINE_Int32(max_depth_in_bkd_tree, "32");
 DEFINE_Bool(inverted_index_compaction_enable, "false");
 // use num_broadcast_buffer blocks as buffer to do broadcast
 DEFINE_Int32(num_broadcast_buffer, "32");
-// semi-structure configs
-DEFINE_Bool(enable_parse_multi_dimession_array, "false");
 
 // max depth of expression tree allowed.
 DEFINE_Int32(max_depth_of_expr_tree, "600");
@@ -1112,14 +1111,18 @@ DEFINE_Bool(enable_flush_file_cache_async, "true");
 
 // cgroup
 DEFINE_String(doris_cgroup_cpu_path, "");
+DEFINE_Bool(enable_cgroup_cpu_soft_limit, "false");
 
 DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
 
 // Dir of default timezone files
 DEFINE_String(default_tzfiles_path, "${DORIS_HOME}/zoneinfo");
 
-// Max size(bytes) of group commit queues, used for mem back pressure.
-DEFINE_Int32(group_commit_max_queue_size, "65536");
+// Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
+DEFINE_Int32(group_commit_max_queue_size, "67108864");
+
+// Max size(bytes) of wal disk using, used for disk space back pressure, default 64M.
+DEFINE_Int32(wal_max_disk_size, "67108864");
 
 // Ingest binlog work pool size, -1 is disable, 0 is hardware concurrency
 DEFINE_Int32(ingest_binlog_work_pool_size, "-1");