Merge branch 'master' into nereids_show_role

.asf.yaml

@@ -152,13 +152,13 @@ github:
     - LemonLiTree
     - Yukang-Lian
     - TangSiyang2001
-    - Lchangliang
     - freemandealer
     - shuke987
     - wm1581066
     - KassieZ
     - yujun777
     - doris-robot
+    - LiBinfeng-01
 
 notifications:
     pullrequests_status:  [email protected]

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,63 @@
-## Proposed changes
+### What problem does this PR solve?
+<!--
+You need to clearly describe your PR in this part:
 
+1. What problem was fixed (it's best to include specific error reporting information). How it was fixed.
+2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be.
+3. What features were added. Why this function was added.
+4. Which codes were refactored and why this part of the code was refactored.
+5. Which functions were optimized and what is the difference before and after the optimization.
+
+The description of the PR needs to enable reviewers to quickly and clearly understand the logic of the code modification.
+-->
+
+<!--
+If there are related issues, please fill in the issue number.
+- If you want the issue to be closed after the PR is merged, please use "close #12345". Otherwise, use "ref #12345"
+-->
 Issue Number: close #xxx
 
-<!--Describe your changes.-->
+<!--
+If this PR is followup a preivous PR, for example, fix the bug that introduced by a related PR,
+link the PR here
+-->
+Related PR: #xxx
+
+Problem Summary:
+
+### Check List (For Committer)
+
+- Test <!-- At least one of them must be included. -->
+
+    - [ ] Regression test
+    - [ ] Unit Test
+    - [ ] Manual test (add detailed scripts or steps below)
+    - [ ] No need to test or manual test. Explain why:
+        - [ ] This is a refactor/code format and no logic has been changed.
+        - [ ] Previous test can cover this change.
+        - [ ] No colde files have been changed.
+        - [ ] Other reason <!-- Add your reason?  -->
+
+- Behavior changed:
+
+    - [ ] No.
+    - [ ] Yes. <!-- Explain the behavior change -->
+
+- Does this need documentation?
+
+    - [ ] No.
+    - [ ] Yes. <!-- Add document PR link here. eg: https://github.com/apache/doris-website/pull/1214 -->
+
+- Release note
+
+    <!-- bugfix, feat, behavior changed need a release note -->
+    <!-- Add one line release note for this PR. -->
+    None
+
+### Check List (For Reviewer who merge this PR)
+
+- [ ] Confirm the release note
+- [ ] Confirm test cases
+- [ ] Confirm document
+- [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
 

.github/workflows/auto-cherry-pick.yml

@@ -30,7 +30,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') && github.event.pull_request.merged == true }}
+    if: ${{ (contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/2.1.x')) && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -45,18 +45,27 @@ jobs:
           pip install PyGithub
       - name: Check SHA
         run: |
-            expected_sha="80b7c6087f2a3e4f4c7f035a52e8e7b05ce00f27aa5c1bd52179df685c912447f94a96145fd3204a3958d8ed9777de5a5183b120e99e0e95bbca0366d69b0ac0"
+            expected_sha="4e4c0d7689b765c7f0677d75d23222555afa9286af46cf77ced66fa247a298d9f8a8c86830d0ce55f70e5f09532b54fbafee040c0343833077cbc7e214d486d2"
             calculated_sha=$(sha512sum tools/auto-pick-script.py | awk '{ print $1 }')
             if [ "$calculated_sha" != "$expected_sha" ]; then
               echo "SHA mismatch! Expected: $expected_sha, but got: $calculated_sha"
               exit 1
             else
               echo "SHA matches: $calculated_sha"
             fi
-      - name: Auto cherry-pick
+      - name: Auto cherry-pick to branch-3.0
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO_NAME: ${{ github.repository }}
           CONFLICT_LABEL: cherry-pick-conflict-in-3.0
         run: |
           python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.0  
+      - name: Auto cherry-pick to branch-2.1
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_NAME: ${{ github.repository }}
+          CONFLICT_LABEL: cherry-pick-conflict-in-2.1.x
+        run: |
+            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-2.1         

.github/workflows/build-extension.yml

@@ -20,7 +20,9 @@ name: Build Extensions
 
 on:
   pull_request:
-
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
 concurrency:
   group: ${{ github.ref }} (Build Extensions)
   cancel-in-progress: true
@@ -29,6 +31,12 @@ jobs:
   changes:
     name: Detect Changes
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     outputs:
       broker_changes: ${{ steps.filter.outputs.broker_changes }}
       docs_changes: ${{ steps.filter.outputs.docs_changes }}

.github/workflows/build-thirdparty.yml

@@ -19,6 +19,9 @@ name: Build Third Party Libraries
 
 on:
   pull_request:
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]  
 
 concurrency:
   group: ${{ github.ref }} (Build Third Party Libraries)
@@ -28,6 +31,12 @@ jobs:
   changes:
     name: Detect Changes
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]') 
     outputs:
       thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
     steps:

.github/workflows/checkstyle.yaml

@@ -20,11 +20,20 @@ name: FE Code Style Checker
 
 on:
   pull_request:
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
 
 jobs:
   java-checkstyle:
     name: "CheckStyle"
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/clang-format.yml

@@ -19,12 +19,22 @@
 ---
 name: Code Formatter
 
-on: [push, pull_request_target]
-
+on:
+  pull_request:
+  pull_request_target:
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
 jobs:
   clang-format:
     name: "Clang Formatter"
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') || (github.event_name == 'pull_request_target') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}

.github/workflows/license-eyes.yml

@@ -22,10 +22,21 @@ on:
   push:
     branches:
       - master
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
+
 jobs:
   license-check:
     name: "License Check"
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request_target') ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}

be/src/agent/agent_server.cpp

@@ -33,6 +33,7 @@
 #include "agent/utils.h"
 #include "agent/workload_group_listener.h"
 #include "agent/workload_sched_policy_listener.h"
+#include "cloud/config.h"
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
@@ -193,7 +194,7 @@ void AgentServer::start_workers(StorageEngine& engine, ExecEnv* exec_env) {
             "REPORT_DISK_STATE", _master_info, config::report_disk_state_interval_seconds, [&engine, &master_info = _master_info] { report_disk_callback(engine, master_info); }));
 
     _report_workers.push_back(std::make_unique<ReportWorker>(
-            "REPORT_OLAP_TABLE", _master_info, config::report_tablet_interval_seconds,[&engine, &master_info = _master_info] { report_tablet_callback(engine, master_info); }));
+            "REPORT_OLAP_TABLET", _master_info, config::report_tablet_interval_seconds,[&engine, &master_info = _master_info] { report_tablet_callback(engine, master_info); }));
     // clang-format on
 }
 
@@ -211,13 +212,25 @@ void AgentServer::cloud_start_workers(CloudStorageEngine& engine, ExecEnv* exec_
             "CALC_DBM_TASK", config::calc_delete_bitmap_worker_count,
             [&engine](auto&& task) { return calc_delete_bitmap_callback(engine, task); });
 
+    // cloud, drop tablet just clean clear_cache, so just one thread do it
+    _workers[TTaskType::DROP] = std::make_unique<TaskWorkerPool>(
+            "DROP_TABLE", 1, [&engine](auto&& task) { return drop_tablet_callback(engine, task); });
+
     _report_workers.push_back(std::make_unique<ReportWorker>(
             "REPORT_TASK", _master_info, config::report_task_interval_seconds,
             [&master_info = _master_info] { report_task_callback(master_info); }));
 
     _report_workers.push_back(std::make_unique<ReportWorker>(
             "REPORT_DISK_STATE", _master_info, config::report_disk_state_interval_seconds,
             [&engine, &master_info = _master_info] { report_disk_callback(engine, master_info); }));
+
+    if (config::enable_cloud_tablet_report) {
+        _report_workers.push_back(std::make_unique<ReportWorker>(
+                "REPORT_OLAP_TABLET", _master_info, config::report_tablet_interval_seconds,
+                [&engine, &master_info = _master_info] {
+                    report_tablet_callback(engine, master_info);
+                }));
+    }
 }
 
 // TODO(lingbin): each task in the batch may have it own status or FE must check and

be/src/agent/heartbeat_server.cpp

@@ -26,6 +26,7 @@
 #include <ostream>
 #include <string>
 
+#include "cloud/cloud_tablet_mgr.h"
 #include "cloud/config.h"
 #include "common/config.h"
 #include "common/status.h"
@@ -275,6 +276,11 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) {
         LOG(INFO) << "set config cloud_unique_id " << master_info.cloud_unique_id << " " << st;
     }
 
+    if (master_info.__isset.tablet_report_inactive_duration_ms) {
+        doris::g_tablet_report_inactive_duration_ms =
+                master_info.tablet_report_inactive_duration_ms;
+    }
+
     if (need_report) {
         LOG(INFO) << "Master FE is changed or restarted. report tablet and disk info immediately";
         _engine.notify_listeners();

be/src/agent/task_worker_pool.cpp

@@ -48,6 +48,8 @@
 #include "cloud/cloud_delete_task.h"
 #include "cloud/cloud_engine_calc_delete_bitmap_task.h"
 #include "cloud/cloud_schema_change_job.h"
+#include "cloud/cloud_tablet_mgr.h"
+#include "cloud/config.h"
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
@@ -116,6 +118,10 @@ bool register_task_info(const TTaskType::type task_type, int64_t signature) {
         // no need to report task of these types
         return true;
     }
+    if (task_type == TTaskType::type::DROP && config::is_cloud_mode()) {
+        // cloud no need to report drop task status
+        return true;
+    }
 
     if (signature == -1) { // No need to report task with unintialized signature
         return true;
@@ -1134,6 +1140,46 @@ void report_tablet_callback(StorageEngine& engine, const TMasterInfo& master_inf
     }
 }
 
+void report_tablet_callback(CloudStorageEngine& engine, const TMasterInfo& master_info) {
+    // Random sleep 1~5 seconds before doing report.
+    // In order to avoid the problem that the FE receives many report requests at the same time
+    // and can not be processed.
+    if (config::report_random_wait) {
+        random_sleep(5);
+    }
+
+    TReportRequest request;
+    request.__set_backend(BackendOptions::get_local_backend());
+    request.__isset.tablets = true;
+
+    increase_report_version();
+    uint64_t report_version;
+    uint64_t total_num_tablets = 0;
+    for (int i = 0; i < 5; i++) {
+        request.tablets.clear();
+        report_version = s_report_version;
+        engine.tablet_mgr().build_all_report_tablets_info(&request.tablets, &total_num_tablets);
+        if (report_version == s_report_version) {
+            break;
+        }
+    }
+
+    if (report_version < s_report_version) {
+        LOG(WARNING) << "report version " << report_version << " change to " << s_report_version;
+        DorisMetrics::instance()->report_all_tablets_requests_skip->increment(1);
+        return;
+    }
+
+    request.__set_report_version(report_version);
+    request.__set_num_tablets(total_num_tablets);
+
+    bool succ = handle_report(request, master_info, "tablet");
+    report_tablet_total << 1;
+    if (!succ) [[unlikely]] {
+        report_tablet_failed << 1;
+    }
+}
+
 void upload_callback(StorageEngine& engine, ExecEnv* env, const TAgentTaskRequest& req) {
     const auto& upload_request = req.upload_req;
 
@@ -1610,6 +1656,21 @@ void drop_tablet_callback(StorageEngine& engine, const TAgentTaskRequest& req) {
     remove_task_info(req.task_type, req.signature);
 }
 
+void drop_tablet_callback(CloudStorageEngine& engine, const TAgentTaskRequest& req) {
+    const auto& drop_tablet_req = req.drop_tablet_req;
+    DBUG_EXECUTE_IF("WorkPoolCloudDropTablet.drop_tablet_callback.failed", {
+        LOG_WARNING("WorkPoolCloudDropTablet.drop_tablet_callback.failed")
+                .tag("tablet_id", drop_tablet_req.tablet_id);
+        return;
+    });
+    // 1. erase lru from tablet mgr
+    // TODO(dx) clean tablet file cache
+    // get tablet's info(such as cachekey, tablet id, rsid)
+    engine.tablet_mgr().erase_tablet(drop_tablet_req.tablet_id);
+    // 2. gen clean file cache task
+    return;
+}
+
 void push_callback(StorageEngine& engine, const TAgentTaskRequest& req) {
     const auto& push_req = req.push_req;