[Feature] Add a http interface to acquire the tablets distribution between different disks

be/src/http/CMakeLists.txt

@@ -38,6 +38,7 @@ add_library(Webserver STATIC
   action/mini_load.cpp
   action/health_action.cpp
   action/tablets_info_action.cpp
+  action/tablets_distribution_action.cpp
   action/checksum_action.cpp
   action/snapshot_action.cpp
   action/reload_tablet_action.cpp

be/src/http/action/tablets_distribution_action.cpp

@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "http/action/tablets_distribution_action.h"
+
+#include <string>
+
+#include "common/status.h"
+#include "gutil/strings/substitute.h"
+#include "http/http_channel.h"
+#include "http/http_request.h"
+#include "http/http_headers.h"
+#include "http/http_status.h"
+#include "service/backend_options.h"
+#include "olap/storage_engine.h"
+#include "olap/tablet_manager.h"
+#include "util/json_util.h"
+
+namespace doris {
+
+const static std::string HEADER_JSON = "application/json";
+
+TabletsDistributionAction::TabletsDistributionAction() {
+    _host = BackendOptions::get_localhost();
+}
+
+void TabletsDistributionAction::handle(HttpRequest *req) {
+    req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
+
+    std::string req_group_method = req->param("group_by");
+    if (req_group_method == "partition") {
+        std::string req_partition_id = req->param("partition_id");
+        uint64_t partition_id = 0;
+        if (req_partition_id != "") {
+            try {
+                partition_id = std::stoull(req_partition_id);
+            } catch (const std::exception& e) {
+                LOG(WARNING) << "invalid argument. partition_id:" << req_partition_id;
+                Status status = Status::InternalError(strings::Substitute("invalid argument: partition_id"));
+                std::string status_result = to_json(status);
+                HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result);
+                return;
+            }
+        }
+        HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution_group_by_partition(partition_id).ToString());
+        return;
+    }
+    LOG(WARNING) << "invalid argument. group_by:" << req_group_method;
+    Status status = Status::InternalError(strings::Substitute("invalid argument: group_by"));
+    std::string status_result = to_json(status);
+    HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result);
+}
+
+EasyJson TabletsDistributionAction::get_tablets_distribution_group_by_partition(uint64_t partition_id) {
+    std::map<int64_t, std::map<DataDir*, int64_t>> tablets_num_on_disk;
+    std::map<int64_t, std::map<DataDir*, std::vector<TabletSize>>> tablets_info_on_disk;
+    TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager();
+    tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_info_on_disk);
+
+    EasyJson tablets_distribution_ej;
+    tablets_distribution_ej["msg"] = "OK";
+    tablets_distribution_ej["code"] = 0;
+    EasyJson data = tablets_distribution_ej.Set("data", EasyJson::kObject);
+    data["host"] = _host;
+    EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray);
+    int64_t tablet_total_number = 0;
+    std::map<int64_t, std::map<DataDir*, int64_t>>::iterator partition_iter = tablets_num_on_disk.begin();
+    for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) {
+        if (partition_id != 0 && partition_id != partition_iter->first) {
+            continue;
+        }
+        EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject);
+        partition["partition_id"] = partition_iter->first;
+        EasyJson disks = partition.Set("disks", EasyJson::kArray);
+        std::map<DataDir*, int64_t>::iterator disk_iter = (partition_iter->second).begin();
+        for (; disk_iter != (partition_iter->second).end(); disk_iter++) {
+            EasyJson disk = disks.PushBack(EasyJson::kObject);
+            disk["disk_path"] = disk_iter->first->path();
+            disk["tablets_num"] = disk_iter->second;
+            tablet_total_number += disk_iter->second;
+            if (partition_id != 0) {
+                EasyJson tablets = disk.Set("tablets", EasyJson::kArray);
+                for (int64_t i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) {
+                    EasyJson tablet = tablets.PushBack(EasyJson::kObject);
+                    tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id;
+                    tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash;
+                    tablet["tablet_size"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size;
+                }
+            }
+        }
+    }
+    tablets_distribution_ej["count"] = tablet_total_number;
+    return tablets_distribution_ej;
+}
+
+} // namespace doris
+

be/src/http/action/tablets_distribution_action.h

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "http/http_handler.h"
+#include "util/easy_json.h"
+#include <string>
+
+namespace doris {
+
+// Get BE tablets distribution info from http API.
+class TabletsDistributionAction : public HttpHandler {
+public:
+    TabletsDistributionAction();
+    void handle(HttpRequest *req) override;
+    EasyJson get_tablets_distribution_group_by_partition(uint64_t partition_id);
+    std::string host() { return _host; }
+
+private:
+    std::string _host;
+};
+} // namespace doris
+

be/src/olap/olap_common.h

@@ -89,6 +89,18 @@ struct TabletInfo {
     UniqueId tablet_uid;
 };
 
+struct TabletSize {
+    TabletSize(TTabletId in_tablet_id, TSchemaHash in_schema_hash, size_t in_tablet_size) :
+            tablet_id(in_tablet_id),
+            schema_hash(in_schema_hash),
+            tablet_size(in_tablet_size) {}
+
+
+    TTabletId tablet_id;
+    TSchemaHash schema_hash;
+    size_t tablet_size;
+};
+
 enum RangeCondition {
     GT = 0, // greater than
     GE = 1, // greater or equal

be/src/olap/tablet_manager.cpp

@@ -1475,4 +1475,31 @@ TabletManager::tablets_shard& TabletManager::_get_tablets_shard(TTabletId tablet
     return _tablets_shards[tabletId & _tablets_shards_mask];
 }
 
+void TabletManager::get_tablets_distribution_on_different_disks(
+                    std::map<int64_t, std::map<DataDir*, int64_t>> &tablets_num_on_disk,
+                    std::map<int64_t, std::map<DataDir*, std::vector<TabletSize>>> &tablets_info_on_disk) {
+    std::vector<DataDir*> data_dirs = StorageEngine::instance()->get_stores();
+    ReadLock rlock(&_partition_tablet_map_lock);
+    std::map<int64_t, std::set<TabletInfo>>::iterator partition_iter = _partition_tablet_map.begin();
+    for (; partition_iter != _partition_tablet_map.end(); partition_iter++) {
+        std::map<DataDir*, int64_t> tablets_num;
+        std::map<DataDir*, std::vector<TabletSize>> tablets_info;
+        for(int i = 0; i < data_dirs.size(); i++) {
+            tablets_num[data_dirs[i]] = 0;
+        }
+        int64_t partition_id = partition_iter->first;
+        std::set<TabletInfo>::iterator tablet_info_iter = (partition_iter->second).begin();
+        for(; tablet_info_iter != (partition_iter->second).end(); tablet_info_iter++) {
+            TabletSharedPtr tablet = get_tablet(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash);
+            DataDir* data_dir = tablet->data_dir();
+            size_t tablet_footprint = tablet->tablet_footprint();
+            tablets_num[data_dir]++;
+            TabletSize tablet_size(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash, tablet_footprint);
+            tablets_info[data_dir].push_back(tablet_size);
+        }
+        tablets_num_on_disk[partition_id] = tablets_num;
+        tablets_info_on_disk[partition_id] = tablets_info;
+    }
+}
+
 } // end namespace doris

be/src/olap/tablet_manager.h

@@ -136,6 +136,10 @@ class TabletManager {
     void register_clone_tablet(int64_t tablet_id);
     void unregister_clone_tablet(int64_t tablet_id);
 
+    void get_tablets_distribution_on_different_disks(
+                    std::map<int64_t, std::map<DataDir*, int64_t>> &tablets_num_on_disk,
+                    std::map<int64_t, std::map<DataDir*, std::vector<TabletSize>>> &tablets_info_on_disk);
+
 private:
     // Add a tablet pointer to StorageEngine
     // If force, drop the existing tablet add this new one

be/src/service/http_service.cpp

@@ -28,6 +28,7 @@
 #include "http/action/restore_tablet_action.h"
 #include "http/action/snapshot_action.h"
 #include "http/action/stream_load.h"
+#include "http/action/tablets_distribution_action.h"
 #include "http/action/tablets_info_action.h"
 #include "http/action/update_config_action.h"
 #include "http/default_path_handlers.h"
@@ -88,6 +89,10 @@ Status HttpService::start() {
     TabletsInfoAction* tablets_info_action = new TabletsInfoAction();
     _ev_http_server->register_handler(HttpMethod::GET, "/tablets_json", tablets_info_action);
 
+    // Register Tablets Distribution action
+    TabletsDistributionAction* tablets_distribution_action = new TabletsDistributionAction();
+    _ev_http_server->register_handler(HttpMethod::GET, "/api/tablets_distribution", tablets_distribution_action);
+
     // register pprof actions
     PprofActions::setup(_env, _ev_http_server.get());
 

docs/en/administrator-guide/http-actions/tablets_distribution.md

@@ -0,0 +1,124 @@
+---
+{
+    "title": "GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS",
+    "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS
+
+Get the distribution of tablets under each partition between different disks on BE node
+
+```
+curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition
+```
+
+The return is the number distribution of tablets under each partition between different disks on BE node, which only include tablet number.
+
+```
+{
+    msg: "OK",
+    code: 0,
+    data: {
+        host: "***",
+        tablets_distribution: [
+            {
+                partition_id:***,
+                disks:[
+                    {
+                        disk_path:"***",
+                        tablets_num:***,
+                    },
+                    {
+                        disk_path:"***",
+                        tablets_num:***,
+                    },
+
+                    ...
+
+                ]
+            },
+            {
+                partition_id:***,
+                disks:[
+                    {
+                        disk_path:"***",
+                        tablets_num:***,
+                    },
+                    {
+                        disk_path:"***",
+                        tablets_num:***,
+                    },
+
+                    ...
+
+                ]
+            },
+
+            ...
+
+        ]
+    },
+    count: ***
+}
+```
+
+```
+curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition&partition_id=xxx
+```
+
+The return is the number distribution of tablets under the particular partition between different disks on BE node, which include tablet number, tablet id, schema hash and tablet size.
+
+```
+{
+    msg: "OK",
+    code: 0,
+    data: {
+        host: "***",
+        tablets_distribution: [
+            {
+                partition_id:***,
+                disks:[
+                    {
+                        disk_path:"***",
+                        tablets_num:***,
+                        tablets:[
+                            {
+                                tablet_id:***,
+                                schema_hash:***,
+                                tablet_size:***
+                            },
+
+                            ...
+
+                        ]
+                    },
+
+                    ...
+
+                ]
+            }
+        ]
+    },
+    count: ***
+}
+```