From 32cd99273f7c9dd6bcd80d659ab8ad238008911d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AD=8F=E7=A5=9A?= Date: Tue, 8 Sep 2020 10:36:22 +0800 Subject: [PATCH 1/8] get tablets distribution between different disks on BE --- be/src/http/CMakeLists.txt | 1 + .../action/tablets_distribution_action.cpp | 113 ++++++++++++++++++ .../http/action/tablets_distribution_action.h | 38 ++++++ be/src/olap/olap_common.h | 12 ++ be/src/olap/tablet_manager.cpp | 31 +++++ be/src/olap/tablet_manager.h | 5 + be/src/service/http_service.cpp | 5 + 7 files changed, 205 insertions(+) create mode 100644 be/src/http/action/tablets_distribution_action.cpp create mode 100644 be/src/http/action/tablets_distribution_action.h diff --git a/be/src/http/CMakeLists.txt b/be/src/http/CMakeLists.txt index 57eb6dcd80aac6..176955bf88016a 100644 --- a/be/src/http/CMakeLists.txt +++ b/be/src/http/CMakeLists.txt @@ -38,6 +38,7 @@ add_library(Webserver STATIC action/mini_load.cpp action/health_action.cpp action/tablets_info_action.cpp + action/tablets_distribution_action.cpp action/checksum_action.cpp action/snapshot_action.cpp action/reload_tablet_action.cpp diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp new file mode 100644 index 00000000000000..043551d6a8a97d --- /dev/null +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/action/tablets_distribution_action.h" + +#include + +#include "http/http_channel.h" +#include "http/http_request.h" +#include "http/http_headers.h" +#include "http/http_status.h" +#include "service/backend_options.h" +#include "olap/storage_engine.h" +#include "olap/tablet_manager.h" + +namespace doris { + +const static std::string HEADER_JSON = "application/json"; + +TabletsDistributionAction::TabletsDistributionAction() { + _host = BackendOptions::get_localhost(); +} + +void TabletsDistributionAction::handle(HttpRequest *req) { + req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); + HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution().ToString()); +} + +EasyJson TabletsDistributionAction::get_tablets_distribution() { + std::map> tablets_num_on_disk; + std::map> tablets_size_on_disk; + std::map>> tablets_info_on_disk; + TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager(); + tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_size_on_disk, tablets_info_on_disk); + + EasyJson tablets_distribution_ej; + tablets_distribution_ej["msg"] = "OK"; + tablets_distribution_ej["code"] = 0; + EasyJson data = tablets_distribution_ej.Set("data", EasyJson::kObject); + data["host"] = _host; + EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray); + std::map>::iterator partition_iter = tablets_num_on_disk.begin(); + for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { + EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); + partition["partition_id"] = partition_iter->first; + EasyJson disks = partition.Set("disks", EasyJson::kArray); + std::map::iterator disk_iter = (partition_iter->second).begin(); + int disk_id = 0; + for (; disk_iter != (partition_iter->second).end(); disk_iter++) { + EasyJson disk = disks.PushBack(EasyJson::kObject); + disk["disk_id"] = ++disk_id; + disk["tablets_num"] = disk_iter->second; + disk["tablets_total_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; + EasyJson tablets = disk.Set("tablets", EasyJson::kArray); + for (int i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { + EasyJson tablet = tablets.PushBack(EasyJson::kObject); + tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id; + tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash; + tablet["tablet_footprint"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size; + } + } + } + tablets_distribution_ej["count"] = 0; + return tablets_distribution_ej; +} +/* +EasyJson TabletsDistributionAction::get_tablets_distribution() { + std::map> tablets_num_on_disk; + std::map> tablets_size_on_disk; + std::map>> tablets_info_on_disk; + TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager(); + tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_size_on_disk, tablets_info_on_disk); + + EasyJson tablets_distribution_ej; + tablets_distribution_ej["msg"] = "OK"; + tablets_distribution_ej["code"] = 0; + EasyJson data = tablets_distribution_ej.Set("data", EasyJson::kObject); + data["host"] = _host; + EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray); + std::map>::iterator partition_iter = tablets_num_on_disk.begin(); + for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { + EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); + partition["partition_id"] = partition_iter->first; + EasyJson disks = partition.Set("disks", EasyJson::kArray); + std::map::iterator disk_iter = (partition_iter->second).begin(); + int disk_id = 0; + for (; disk_iter != (partition_iter->second).end(); disk_iter++) { + EasyJson disk = disks.PushBack(EasyJson::kObject); + disk["disk_id"] = ++disk_id; + disk["tablets_num"] = disk_iter->second; + disk["tablets_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; + } + } + tablets_distribution_ej["count"] = 0; + return tablets_distribution_ej; +} +*/ +} // namespace doris + diff --git a/be/src/http/action/tablets_distribution_action.h b/be/src/http/action/tablets_distribution_action.h new file mode 100644 index 00000000000000..8e83973c5b1d69 --- /dev/null +++ b/be/src/http/action/tablets_distribution_action.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "http/http_handler.h" +#include "util/easy_json.h" +#include + +namespace doris { + +// Get BE tablets distribution info from http API. +class TabletsDistributionAction : public HttpHandler { +public: + TabletsDistributionAction(); + void handle(HttpRequest *req) override; + EasyJson get_tablets_distribution(); + std::string host() { return _host; } + +private: + std::string _host; +}; +} // namespace doris + diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 64f17fe3b10a77..1fd83d6e70a752 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -89,6 +89,18 @@ struct TabletInfo { UniqueId tablet_uid; }; +struct TabletSize { + TabletSize(TTabletId in_tablet_id, TSchemaHash in_schema_hash, size_t in_tablet_size) : + tablet_id(in_tablet_id), + schema_hash(in_schema_hash), + tablet_size(in_tablet_size) {} + + + TTabletId tablet_id; + TSchemaHash schema_hash; + size_t tablet_size; +}; + enum RangeCondition { GT = 0, // greater than GE = 1, // greater or equal diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 1fd47c68329a03..80a68ab83a12e3 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1475,4 +1475,35 @@ TabletManager::tablets_shard& TabletManager::_get_tablets_shard(TTabletId tablet return _tablets_shards[tabletId & _tablets_shards_mask]; } +void TabletManager::get_tablets_distribution_on_different_disks( + std::map> &tablets_num_on_disk, + std::map> &tablets_size_on_disk, + std::map>> &tablets_info_on_disk) { + std::vector data_dirs = StorageEngine::instance()->get_stores(); + std::map>::iterator partition_iter = _partition_tablet_map.begin(); + for (; partition_iter != _partition_tablet_map.end(); partition_iter++) { + std::map tablets_num; + std::map tablets_size; + std::map> tablets_info; + for(int i = 0; i < data_dirs.size(); i++) { + tablets_num[data_dirs[i]] = 0; + tablets_size[data_dirs[i]] = 0; + } + int64_t partition_id = partition_iter->first; + std::set::iterator tablet_info_iter = (partition_iter->second).begin(); + for(; tablet_info_iter != (partition_iter->second).end(); tablet_info_iter++) { + TabletSharedPtr tablet = _get_tablet_unlocked(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash); + DataDir* data_dir = tablet->data_dir(); + size_t tablet_footprint = tablet->tablet_footprint(); + tablets_num[data_dir]++; + tablets_size[data_dir] = tablets_size[data_dir] + tablet_footprint; + TabletSize tablet_size(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash, tablet_footprint); + tablets_info[data_dir].push_back(tablet_size); + } + tablets_num_on_disk[partition_id] = tablets_num; + tablets_size_on_disk[partition_id] = tablets_size; + tablets_info_on_disk[partition_id] = tablets_info; + } +} + } // end namespace doris diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 84ad8af18aa7e9..963d7307009359 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -136,6 +136,11 @@ class TabletManager { void register_clone_tablet(int64_t tablet_id); void unregister_clone_tablet(int64_t tablet_id); + void get_tablets_distribution_on_different_disks( + std::map> &tablets_num_on_disk, + std::map> &tablets_size_on_disk, + std::map>> &tablets_info_on_disk); + private: // Add a tablet pointer to StorageEngine // If force, drop the existing tablet add this new one diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 931310295e1653..702821892ce11c 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -28,6 +28,7 @@ #include "http/action/restore_tablet_action.h" #include "http/action/snapshot_action.h" #include "http/action/stream_load.h" +#include "http/action/tablets_distribution_action.h" #include "http/action/tablets_info_action.h" #include "http/action/update_config_action.h" #include "http/default_path_handlers.h" @@ -88,6 +89,10 @@ Status HttpService::start() { TabletsInfoAction* tablets_info_action = new TabletsInfoAction(); _ev_http_server->register_handler(HttpMethod::GET, "/tablets_json", tablets_info_action); + // Register Tablets Distribution action + TabletsDistributionAction* tablets_distribution_action = new TabletsDistributionAction(); + _ev_http_server->register_handler(HttpMethod::GET, "/tablets_distribution", tablets_distribution_action); + // register pprof actions PprofActions::setup(_env, _ev_http_server.get()); From 187b11a72f0eaf4a8b171f9b3a11fc108571fb3f Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Thu, 10 Sep 2020 20:31:32 +0800 Subject: [PATCH 2/8] get tablets distribution between different disks on BE --- be/src/http/action/tablets_distribution_action.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp index 043551d6a8a97d..5ed7af29d69ec0 100644 --- a/be/src/http/action/tablets_distribution_action.cpp +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -59,10 +59,9 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { partition["partition_id"] = partition_iter->first; EasyJson disks = partition.Set("disks", EasyJson::kArray); std::map::iterator disk_iter = (partition_iter->second).begin(); - int disk_id = 0; for (; disk_iter != (partition_iter->second).end(); disk_iter++) { EasyJson disk = disks.PushBack(EasyJson::kObject); - disk["disk_id"] = ++disk_id; + disk["disk_path_hash"] = disk_iter->first->path_hash(); disk["tablets_num"] = disk_iter->second; disk["tablets_total_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; EasyJson tablets = disk.Set("tablets", EasyJson::kArray); From cacc331765acfc525b0b71e70432776b06c1eca0 Mon Sep 17 00:00:00 2001 From: weizuo Date: Wed, 16 Dec 2020 14:21:52 +0800 Subject: [PATCH 3/8] get tablets distribution between different disks on BE --- be/src/http/action/tablets_distribution_action.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp index 5ed7af29d69ec0..9d19bf322ed0af 100644 --- a/be/src/http/action/tablets_distribution_action.cpp +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -61,7 +61,7 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { std::map::iterator disk_iter = (partition_iter->second).begin(); for (; disk_iter != (partition_iter->second).end(); disk_iter++) { EasyJson disk = disks.PushBack(EasyJson::kObject); - disk["disk_path_hash"] = disk_iter->first->path_hash(); + disk["disk_path"] = disk_iter->first->path(); disk["tablets_num"] = disk_iter->second; disk["tablets_total_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; EasyJson tablets = disk.Set("tablets", EasyJson::kArray); @@ -69,7 +69,7 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { EasyJson tablet = tablets.PushBack(EasyJson::kObject); tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id; tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash; - tablet["tablet_footprint"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size; + tablet["tablet_size"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size; } } } From 5319a735bd19f6eacbfe0a7f7d086a36eef060b6 Mon Sep 17 00:00:00 2001 From: weizuo Date: Wed, 16 Dec 2020 14:42:54 +0800 Subject: [PATCH 4/8] get tablets distribution between different disks on BE --- .../action/tablets_distribution_action.cpp | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp index 9d19bf322ed0af..a14174866a56e7 100644 --- a/be/src/http/action/tablets_distribution_action.cpp +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -53,6 +53,7 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { EasyJson data = tablets_distribution_ej.Set("data", EasyJson::kObject); data["host"] = _host; EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray); + int64_t tablet_total_number = 0; std::map>::iterator partition_iter = tablets_num_on_disk.begin(); for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); @@ -63,6 +64,7 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { EasyJson disk = disks.PushBack(EasyJson::kObject); disk["disk_path"] = disk_iter->first->path(); disk["tablets_num"] = disk_iter->second; + tablet_total_number += disk_iter->second; disk["tablets_total_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; EasyJson tablets = disk.Set("tablets", EasyJson::kArray); for (int i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { @@ -73,40 +75,9 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { } } } - tablets_distribution_ej["count"] = 0; + tablets_distribution_ej["count"] = tablet_total_number; return tablets_distribution_ej; } -/* -EasyJson TabletsDistributionAction::get_tablets_distribution() { - std::map> tablets_num_on_disk; - std::map> tablets_size_on_disk; - std::map>> tablets_info_on_disk; - TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager(); - tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_size_on_disk, tablets_info_on_disk); - EasyJson tablets_distribution_ej; - tablets_distribution_ej["msg"] = "OK"; - tablets_distribution_ej["code"] = 0; - EasyJson data = tablets_distribution_ej.Set("data", EasyJson::kObject); - data["host"] = _host; - EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray); - std::map>::iterator partition_iter = tablets_num_on_disk.begin(); - for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { - EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); - partition["partition_id"] = partition_iter->first; - EasyJson disks = partition.Set("disks", EasyJson::kArray); - std::map::iterator disk_iter = (partition_iter->second).begin(); - int disk_id = 0; - for (; disk_iter != (partition_iter->second).end(); disk_iter++) { - EasyJson disk = disks.PushBack(EasyJson::kObject); - disk["disk_id"] = ++disk_id; - disk["tablets_num"] = disk_iter->second; - disk["tablets_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; - } - } - tablets_distribution_ej["count"] = 0; - return tablets_distribution_ej; -} -*/ } // namespace doris From 643901e8c46e489535bdb687c40b8c74983afe57 Mon Sep 17 00:00:00 2001 From: weizuo Date: Wed, 16 Dec 2020 16:37:43 +0800 Subject: [PATCH 5/8] get tablets distribution between different disks on BE --- .../action/tablets_distribution_action.cpp | 38 +++++++++++++++---- .../http/action/tablets_distribution_action.h | 2 +- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp index a14174866a56e7..273c619fde7eb9 100644 --- a/be/src/http/action/tablets_distribution_action.cpp +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -19,6 +19,8 @@ #include +#include "common/status.h" +#include "gutil/strings/substitute.h" #include "http/http_channel.h" #include "http/http_request.h" #include "http/http_headers.h" @@ -26,6 +28,7 @@ #include "service/backend_options.h" #include "olap/storage_engine.h" #include "olap/tablet_manager.h" +#include "util/json_util.h" namespace doris { @@ -37,10 +40,24 @@ TabletsDistributionAction::TabletsDistributionAction() { void TabletsDistributionAction::handle(HttpRequest *req) { req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); - HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution().ToString()); + + std::string req_partition_id = req->param("partition_id"); + uint64_t partition_id = 0; + if (req_partition_id != "") { + try { + partition_id = std::stoull(req_partition_id); + } catch (const std::exception& e) { + LOG(WARNING) << "invalid argument. partition_id:" << req_partition_id; + Status status = Status::InternalError(strings::Substitute("invalid argument: partition_id")); + std::string status_result = to_json(status); + HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); + return; + } + } + HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution(partition_id).ToString()); } -EasyJson TabletsDistributionAction::get_tablets_distribution() { +EasyJson TabletsDistributionAction::get_tablets_distribution(uint64_t partition_id) { std::map> tablets_num_on_disk; std::map> tablets_size_on_disk; std::map>> tablets_info_on_disk; @@ -56,6 +73,9 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { int64_t tablet_total_number = 0; std::map>::iterator partition_iter = tablets_num_on_disk.begin(); for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { + if (partition_id != 0 && partition_id != partition_iter->first) { + continue; + } EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); partition["partition_id"] = partition_iter->first; EasyJson disks = partition.Set("disks", EasyJson::kArray); @@ -66,12 +86,14 @@ EasyJson TabletsDistributionAction::get_tablets_distribution() { disk["tablets_num"] = disk_iter->second; tablet_total_number += disk_iter->second; disk["tablets_total_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; - EasyJson tablets = disk.Set("tablets", EasyJson::kArray); - for (int i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { - EasyJson tablet = tablets.PushBack(EasyJson::kObject); - tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id; - tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash; - tablet["tablet_size"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size; + if (partition_id != 0) { + EasyJson tablets = disk.Set("tablets", EasyJson::kArray); + for (int i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { + EasyJson tablet = tablets.PushBack(EasyJson::kObject); + tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id; + tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash; + tablet["tablet_size"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size; + } } } } diff --git a/be/src/http/action/tablets_distribution_action.h b/be/src/http/action/tablets_distribution_action.h index 8e83973c5b1d69..d6841a06ca9eff 100644 --- a/be/src/http/action/tablets_distribution_action.h +++ b/be/src/http/action/tablets_distribution_action.h @@ -28,7 +28,7 @@ class TabletsDistributionAction : public HttpHandler { public: TabletsDistributionAction(); void handle(HttpRequest *req) override; - EasyJson get_tablets_distribution(); + EasyJson get_tablets_distribution(uint64_t partition_id); std::string host() { return _host; } private: From ee8b95a095d761d28e525e0e3bf4f6dca34fc577 Mon Sep 17 00:00:00 2001 From: weizuo Date: Wed, 16 Dec 2020 17:37:57 +0800 Subject: [PATCH 6/8] get tablets distribution between different disks on BE --- be/src/http/action/tablets_distribution_action.cpp | 12 +++++------- be/src/olap/tablet_manager.cpp | 12 ++++-------- be/src/olap/tablet_manager.h | 3 +-- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp index 273c619fde7eb9..fe2e8dc90a691d 100644 --- a/be/src/http/action/tablets_distribution_action.cpp +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -58,11 +58,10 @@ void TabletsDistributionAction::handle(HttpRequest *req) { } EasyJson TabletsDistributionAction::get_tablets_distribution(uint64_t partition_id) { - std::map> tablets_num_on_disk; - std::map> tablets_size_on_disk; + std::map> tablets_num_on_disk; std::map>> tablets_info_on_disk; TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager(); - tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_size_on_disk, tablets_info_on_disk); + tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_info_on_disk); EasyJson tablets_distribution_ej; tablets_distribution_ej["msg"] = "OK"; @@ -71,7 +70,7 @@ EasyJson TabletsDistributionAction::get_tablets_distribution(uint64_t partition_ data["host"] = _host; EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray); int64_t tablet_total_number = 0; - std::map>::iterator partition_iter = tablets_num_on_disk.begin(); + std::map>::iterator partition_iter = tablets_num_on_disk.begin(); for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { if (partition_id != 0 && partition_id != partition_iter->first) { continue; @@ -79,16 +78,15 @@ EasyJson TabletsDistributionAction::get_tablets_distribution(uint64_t partition_ EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); partition["partition_id"] = partition_iter->first; EasyJson disks = partition.Set("disks", EasyJson::kArray); - std::map::iterator disk_iter = (partition_iter->second).begin(); + std::map::iterator disk_iter = (partition_iter->second).begin(); for (; disk_iter != (partition_iter->second).end(); disk_iter++) { EasyJson disk = disks.PushBack(EasyJson::kObject); disk["disk_path"] = disk_iter->first->path(); disk["tablets_num"] = disk_iter->second; tablet_total_number += disk_iter->second; - disk["tablets_total_size"] = tablets_size_on_disk[partition_iter->first][disk_iter->first]; if (partition_id != 0) { EasyJson tablets = disk.Set("tablets", EasyJson::kArray); - for (int i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { + for (int64_t i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { EasyJson tablet = tablets.PushBack(EasyJson::kObject); tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id; tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 80a68ab83a12e3..42b748ea775d61 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1476,32 +1476,28 @@ TabletManager::tablets_shard& TabletManager::_get_tablets_shard(TTabletId tablet } void TabletManager::get_tablets_distribution_on_different_disks( - std::map> &tablets_num_on_disk, - std::map> &tablets_size_on_disk, + std::map> &tablets_num_on_disk, std::map>> &tablets_info_on_disk) { std::vector data_dirs = StorageEngine::instance()->get_stores(); + ReadLock rlock(&_partition_tablet_map_lock); std::map>::iterator partition_iter = _partition_tablet_map.begin(); for (; partition_iter != _partition_tablet_map.end(); partition_iter++) { - std::map tablets_num; - std::map tablets_size; + std::map tablets_num; std::map> tablets_info; for(int i = 0; i < data_dirs.size(); i++) { tablets_num[data_dirs[i]] = 0; - tablets_size[data_dirs[i]] = 0; } int64_t partition_id = partition_iter->first; std::set::iterator tablet_info_iter = (partition_iter->second).begin(); for(; tablet_info_iter != (partition_iter->second).end(); tablet_info_iter++) { - TabletSharedPtr tablet = _get_tablet_unlocked(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash); + TabletSharedPtr tablet = get_tablet(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash); DataDir* data_dir = tablet->data_dir(); size_t tablet_footprint = tablet->tablet_footprint(); tablets_num[data_dir]++; - tablets_size[data_dir] = tablets_size[data_dir] + tablet_footprint; TabletSize tablet_size(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash, tablet_footprint); tablets_info[data_dir].push_back(tablet_size); } tablets_num_on_disk[partition_id] = tablets_num; - tablets_size_on_disk[partition_id] = tablets_size; tablets_info_on_disk[partition_id] = tablets_info; } } diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 963d7307009359..be9ab62299ba60 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -137,8 +137,7 @@ class TabletManager { void unregister_clone_tablet(int64_t tablet_id); void get_tablets_distribution_on_different_disks( - std::map> &tablets_num_on_disk, - std::map> &tablets_size_on_disk, + std::map> &tablets_num_on_disk, std::map>> &tablets_info_on_disk); private: From 89ba840677e8f8c7508e3648a6f88a214ace800e Mon Sep 17 00:00:00 2001 From: weizuo Date: Wed, 16 Dec 2020 18:11:07 +0800 Subject: [PATCH 7/8] get tablets distribution between different disks on BE --- .../http-actions/tablets_distribution.md | 124 ++++++++++++++++++ .../http-actions/tablets_distribution.md | 124 ++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 docs/en/administrator-guide/http-actions/tablets_distribution.md create mode 100644 docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md diff --git a/docs/en/administrator-guide/http-actions/tablets_distribution.md b/docs/en/administrator-guide/http-actions/tablets_distribution.md new file mode 100644 index 00000000000000..79fdf76e038f5a --- /dev/null +++ b/docs/en/administrator-guide/http-actions/tablets_distribution.md @@ -0,0 +1,124 @@ +--- +{ + "title": "GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS", + "language": "en" +} +--- + + + +# GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS + +Get the distribution of tablets under each partition between different disks on BE node + +``` +curl -X GET http://be_host:webserver_port/tablets_distribution +``` + +The return is the number distribution of tablets under each partition between different disks on BE node, which only include tablet number. + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + + ... + + ] + }, + count: *** +} +``` + +``` +curl -X GET http://be_host:webserver_port/tablets_distribution？partition_id=xxx +``` + +The return is the number distribution of tablets under the particular partition between different disks on BE node, which include tablet number, tablet id, schema hash and tablet size. + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + tablets:[ + { + tablet_id:***, + schema_hash:***, + tablet_size:*** + }, + + ... + + ] + }, + + ... + + ] + } + ] + }, + count: *** +} +``` diff --git a/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md b/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md new file mode 100644 index 00000000000000..5a5a0be829ed65 --- /dev/null +++ b/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md @@ -0,0 +1,124 @@ +--- +{ + "title": "GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS", + "language": "zh-CN" +} +--- + + + +# GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS + +获取BE节点上每一个partition下的tablet在不同磁盘上的分布情况 + +``` +curl -X GET http://be_host:webserver_port/tablets_distribution +``` + +返回值就是BE节点上每一个partition下的tablet在各个磁盘上的数量分布，只包含tablet数量。 + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + + ... + + ] + }, + count: *** +} +``` + +``` +curl -X GET http://be_host:webserver_port/tablets_distribution？partition_id=xxx +``` + +返回值就是BE节点上指定id的partition下的tablet在各个磁盘上的分布，包含tablet数量以及每一个tablet的id、schema hash和tablet size信息。 + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + tablets:[ + { + tablet_id:***, + schema_hash:***, + tablet_size:*** + }, + + ... + + ] + }, + + ... + + ] + } + ] + }, + count: *** +} +``` From 1d320a9070ff7a8d680926b3038d23172e741b04 Mon Sep 17 00:00:00 2001 From: weizuo Date: Wed, 13 Jan 2021 21:15:41 +0800 Subject: [PATCH 8/8] get tablets distribution between different disks on BE --- .../action/tablets_distribution_action.cpp | 34 ++++++++++++------- .../http/action/tablets_distribution_action.h | 2 +- be/src/service/http_service.cpp | 2 +- .../http-actions/tablets_distribution.md | 4 +-- .../http-actions/tablets_distribution.md | 4 +-- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp index fe2e8dc90a691d..2c978e3d1a2db8 100644 --- a/be/src/http/action/tablets_distribution_action.cpp +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -41,23 +41,31 @@ TabletsDistributionAction::TabletsDistributionAction() { void TabletsDistributionAction::handle(HttpRequest *req) { req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); - std::string req_partition_id = req->param("partition_id"); - uint64_t partition_id = 0; - if (req_partition_id != "") { - try { - partition_id = std::stoull(req_partition_id); - } catch (const std::exception& e) { - LOG(WARNING) << "invalid argument. partition_id:" << req_partition_id; - Status status = Status::InternalError(strings::Substitute("invalid argument: partition_id")); - std::string status_result = to_json(status); - HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); - return; + std::string req_group_method = req->param("group_by"); + if (req_group_method == "partition") { + std::string req_partition_id = req->param("partition_id"); + uint64_t partition_id = 0; + if (req_partition_id != "") { + try { + partition_id = std::stoull(req_partition_id); + } catch (const std::exception& e) { + LOG(WARNING) << "invalid argument. partition_id:" << req_partition_id; + Status status = Status::InternalError(strings::Substitute("invalid argument: partition_id")); + std::string status_result = to_json(status); + HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); + return; + } } + HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution_group_by_partition(partition_id).ToString()); + return; } - HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution(partition_id).ToString()); + LOG(WARNING) << "invalid argument. group_by:" << req_group_method; + Status status = Status::InternalError(strings::Substitute("invalid argument: group_by")); + std::string status_result = to_json(status); + HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); } -EasyJson TabletsDistributionAction::get_tablets_distribution(uint64_t partition_id) { +EasyJson TabletsDistributionAction::get_tablets_distribution_group_by_partition(uint64_t partition_id) { std::map> tablets_num_on_disk; std::map>> tablets_info_on_disk; TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager(); diff --git a/be/src/http/action/tablets_distribution_action.h b/be/src/http/action/tablets_distribution_action.h index d6841a06ca9eff..42582c07d5931a 100644 --- a/be/src/http/action/tablets_distribution_action.h +++ b/be/src/http/action/tablets_distribution_action.h @@ -28,7 +28,7 @@ class TabletsDistributionAction : public HttpHandler { public: TabletsDistributionAction(); void handle(HttpRequest *req) override; - EasyJson get_tablets_distribution(uint64_t partition_id); + EasyJson get_tablets_distribution_group_by_partition(uint64_t partition_id); std::string host() { return _host; } private: diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 702821892ce11c..a2e430246f4488 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -91,7 +91,7 @@ Status HttpService::start() { // Register Tablets Distribution action TabletsDistributionAction* tablets_distribution_action = new TabletsDistributionAction(); - _ev_http_server->register_handler(HttpMethod::GET, "/tablets_distribution", tablets_distribution_action); + _ev_http_server->register_handler(HttpMethod::GET, "/api/tablets_distribution", tablets_distribution_action); // register pprof actions PprofActions::setup(_env, _ev_http_server.get()); diff --git a/docs/en/administrator-guide/http-actions/tablets_distribution.md b/docs/en/administrator-guide/http-actions/tablets_distribution.md index 79fdf76e038f5a..df0b1c9282aebb 100644 --- a/docs/en/administrator-guide/http-actions/tablets_distribution.md +++ b/docs/en/administrator-guide/http-actions/tablets_distribution.md @@ -29,7 +29,7 @@ under the License. Get the distribution of tablets under each partition between different disks on BE node ``` -curl -X GET http://be_host:webserver_port/tablets_distribution +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition ``` The return is the number distribution of tablets under each partition between different disks on BE node, which only include tablet number. @@ -83,7 +83,7 @@ The return is the number distribution of tablets under each partition between di ``` ``` -curl -X GET http://be_host:webserver_port/tablets_distribution？partition_id=xxx +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition&partition_id=xxx ``` The return is the number distribution of tablets under the particular partition between different disks on BE node, which include tablet number, tablet id, schema hash and tablet size. diff --git a/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md b/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md index 5a5a0be829ed65..44c410aa6d3a9b 100644 --- a/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md +++ b/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md @@ -29,7 +29,7 @@ under the License. 获取BE节点上每一个partition下的tablet在不同磁盘上的分布情况 ``` -curl -X GET http://be_host:webserver_port/tablets_distribution +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition ``` 返回值就是BE节点上每一个partition下的tablet在各个磁盘上的数量分布，只包含tablet数量。 @@ -83,7 +83,7 @@ curl -X GET http://be_host:webserver_port/tablets_distribution ``` ``` -curl -X GET http://be_host:webserver_port/tablets_distribution？partition_id=xxx +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition&partition_id=xxx ``` 返回值就是BE节点上指定id的partition下的tablet在各个磁盘上的分布，包含tablet数量以及每一个tablet的id、schema hash和tablet size信息。