diff --git a/be/src/http/CMakeLists.txt b/be/src/http/CMakeLists.txt index 57eb6dcd80aac6..176955bf88016a 100644 --- a/be/src/http/CMakeLists.txt +++ b/be/src/http/CMakeLists.txt @@ -38,6 +38,7 @@ add_library(Webserver STATIC action/mini_load.cpp action/health_action.cpp action/tablets_info_action.cpp + action/tablets_distribution_action.cpp action/checksum_action.cpp action/snapshot_action.cpp action/reload_tablet_action.cpp diff --git a/be/src/http/action/tablets_distribution_action.cpp b/be/src/http/action/tablets_distribution_action.cpp new file mode 100644 index 00000000000000..2c978e3d1a2db8 --- /dev/null +++ b/be/src/http/action/tablets_distribution_action.cpp @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/action/tablets_distribution_action.h" + +#include + +#include "common/status.h" +#include "gutil/strings/substitute.h" +#include "http/http_channel.h" +#include "http/http_request.h" +#include "http/http_headers.h" +#include "http/http_status.h" +#include "service/backend_options.h" +#include "olap/storage_engine.h" +#include "olap/tablet_manager.h" +#include "util/json_util.h" + +namespace doris { + +const static std::string HEADER_JSON = "application/json"; + +TabletsDistributionAction::TabletsDistributionAction() { + _host = BackendOptions::get_localhost(); +} + +void TabletsDistributionAction::handle(HttpRequest *req) { + req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); + + std::string req_group_method = req->param("group_by"); + if (req_group_method == "partition") { + std::string req_partition_id = req->param("partition_id"); + uint64_t partition_id = 0; + if (req_partition_id != "") { + try { + partition_id = std::stoull(req_partition_id); + } catch (const std::exception& e) { + LOG(WARNING) << "invalid argument. partition_id:" << req_partition_id; + Status status = Status::InternalError(strings::Substitute("invalid argument: partition_id")); + std::string status_result = to_json(status); + HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); + return; + } + } + HttpChannel::send_reply(req, HttpStatus::OK, get_tablets_distribution_group_by_partition(partition_id).ToString()); + return; + } + LOG(WARNING) << "invalid argument. group_by:" << req_group_method; + Status status = Status::InternalError(strings::Substitute("invalid argument: group_by")); + std::string status_result = to_json(status); + HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); +} + +EasyJson TabletsDistributionAction::get_tablets_distribution_group_by_partition(uint64_t partition_id) { + std::map> tablets_num_on_disk; + std::map>> tablets_info_on_disk; + TabletManager* tablet_manager = StorageEngine::instance()->tablet_manager(); + tablet_manager->get_tablets_distribution_on_different_disks(tablets_num_on_disk, tablets_info_on_disk); + + EasyJson tablets_distribution_ej; + tablets_distribution_ej["msg"] = "OK"; + tablets_distribution_ej["code"] = 0; + EasyJson data = tablets_distribution_ej.Set("data", EasyJson::kObject); + data["host"] = _host; + EasyJson tablets_distribution = data.Set("tablets_distribution", EasyJson::kArray); + int64_t tablet_total_number = 0; + std::map>::iterator partition_iter = tablets_num_on_disk.begin(); + for (; partition_iter != tablets_num_on_disk.end(); partition_iter++) { + if (partition_id != 0 && partition_id != partition_iter->first) { + continue; + } + EasyJson partition = tablets_distribution.PushBack(EasyJson::kObject); + partition["partition_id"] = partition_iter->first; + EasyJson disks = partition.Set("disks", EasyJson::kArray); + std::map::iterator disk_iter = (partition_iter->second).begin(); + for (; disk_iter != (partition_iter->second).end(); disk_iter++) { + EasyJson disk = disks.PushBack(EasyJson::kObject); + disk["disk_path"] = disk_iter->first->path(); + disk["tablets_num"] = disk_iter->second; + tablet_total_number += disk_iter->second; + if (partition_id != 0) { + EasyJson tablets = disk.Set("tablets", EasyJson::kArray); + for (int64_t i = 0; i < tablets_info_on_disk[partition_iter->first][disk_iter->first].size(); i++) { + EasyJson tablet = tablets.PushBack(EasyJson::kObject); + tablet["tablet_id"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_id; + tablet["schema_hash"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].schema_hash; + tablet["tablet_size"] = tablets_info_on_disk[partition_iter->first][disk_iter->first][i].tablet_size; + } + } + } + } + tablets_distribution_ej["count"] = tablet_total_number; + return tablets_distribution_ej; +} + +} // namespace doris + diff --git a/be/src/http/action/tablets_distribution_action.h b/be/src/http/action/tablets_distribution_action.h new file mode 100644 index 00000000000000..42582c07d5931a --- /dev/null +++ b/be/src/http/action/tablets_distribution_action.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "http/http_handler.h" +#include "util/easy_json.h" +#include + +namespace doris { + +// Get BE tablets distribution info from http API. +class TabletsDistributionAction : public HttpHandler { +public: + TabletsDistributionAction(); + void handle(HttpRequest *req) override; + EasyJson get_tablets_distribution_group_by_partition(uint64_t partition_id); + std::string host() { return _host; } + +private: + std::string _host; +}; +} // namespace doris + diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 64f17fe3b10a77..1fd83d6e70a752 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -89,6 +89,18 @@ struct TabletInfo { UniqueId tablet_uid; }; +struct TabletSize { + TabletSize(TTabletId in_tablet_id, TSchemaHash in_schema_hash, size_t in_tablet_size) : + tablet_id(in_tablet_id), + schema_hash(in_schema_hash), + tablet_size(in_tablet_size) {} + + + TTabletId tablet_id; + TSchemaHash schema_hash; + size_t tablet_size; +}; + enum RangeCondition { GT = 0, // greater than GE = 1, // greater or equal diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 1fd47c68329a03..42b748ea775d61 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1475,4 +1475,31 @@ TabletManager::tablets_shard& TabletManager::_get_tablets_shard(TTabletId tablet return _tablets_shards[tabletId & _tablets_shards_mask]; } +void TabletManager::get_tablets_distribution_on_different_disks( + std::map> &tablets_num_on_disk, + std::map>> &tablets_info_on_disk) { + std::vector data_dirs = StorageEngine::instance()->get_stores(); + ReadLock rlock(&_partition_tablet_map_lock); + std::map>::iterator partition_iter = _partition_tablet_map.begin(); + for (; partition_iter != _partition_tablet_map.end(); partition_iter++) { + std::map tablets_num; + std::map> tablets_info; + for(int i = 0; i < data_dirs.size(); i++) { + tablets_num[data_dirs[i]] = 0; + } + int64_t partition_id = partition_iter->first; + std::set::iterator tablet_info_iter = (partition_iter->second).begin(); + for(; tablet_info_iter != (partition_iter->second).end(); tablet_info_iter++) { + TabletSharedPtr tablet = get_tablet(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash); + DataDir* data_dir = tablet->data_dir(); + size_t tablet_footprint = tablet->tablet_footprint(); + tablets_num[data_dir]++; + TabletSize tablet_size(tablet_info_iter->tablet_id, tablet_info_iter->schema_hash, tablet_footprint); + tablets_info[data_dir].push_back(tablet_size); + } + tablets_num_on_disk[partition_id] = tablets_num; + tablets_info_on_disk[partition_id] = tablets_info; + } +} + } // end namespace doris diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 84ad8af18aa7e9..be9ab62299ba60 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -136,6 +136,10 @@ class TabletManager { void register_clone_tablet(int64_t tablet_id); void unregister_clone_tablet(int64_t tablet_id); + void get_tablets_distribution_on_different_disks( + std::map> &tablets_num_on_disk, + std::map>> &tablets_info_on_disk); + private: // Add a tablet pointer to StorageEngine // If force, drop the existing tablet add this new one diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 931310295e1653..a2e430246f4488 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -28,6 +28,7 @@ #include "http/action/restore_tablet_action.h" #include "http/action/snapshot_action.h" #include "http/action/stream_load.h" +#include "http/action/tablets_distribution_action.h" #include "http/action/tablets_info_action.h" #include "http/action/update_config_action.h" #include "http/default_path_handlers.h" @@ -88,6 +89,10 @@ Status HttpService::start() { TabletsInfoAction* tablets_info_action = new TabletsInfoAction(); _ev_http_server->register_handler(HttpMethod::GET, "/tablets_json", tablets_info_action); + // Register Tablets Distribution action + TabletsDistributionAction* tablets_distribution_action = new TabletsDistributionAction(); + _ev_http_server->register_handler(HttpMethod::GET, "/api/tablets_distribution", tablets_distribution_action); + // register pprof actions PprofActions::setup(_env, _ev_http_server.get()); diff --git a/docs/en/administrator-guide/http-actions/tablets_distribution.md b/docs/en/administrator-guide/http-actions/tablets_distribution.md new file mode 100644 index 00000000000000..df0b1c9282aebb --- /dev/null +++ b/docs/en/administrator-guide/http-actions/tablets_distribution.md @@ -0,0 +1,124 @@ +--- +{ + "title": "GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS", + "language": "en" +} +--- + + + +# GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS + +Get the distribution of tablets under each partition between different disks on BE node + +``` +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition +``` + +The return is the number distribution of tablets under each partition between different disks on BE node, which only include tablet number. + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + + ... + + ] + }, + count: *** +} +``` + +``` +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition&partition_id=xxx +``` + +The return is the number distribution of tablets under the particular partition between different disks on BE node, which include tablet number, tablet id, schema hash and tablet size. + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + tablets:[ + { + tablet_id:***, + schema_hash:***, + tablet_size:*** + }, + + ... + + ] + }, + + ... + + ] + } + ] + }, + count: *** +} +``` diff --git a/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md b/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md new file mode 100644 index 00000000000000..44c410aa6d3a9b --- /dev/null +++ b/docs/zh-CN/administrator-guide/http-actions/tablets_distribution.md @@ -0,0 +1,124 @@ +--- +{ + "title": "GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS", + "language": "zh-CN" +} +--- + + + +# GET TABLETS DISTRIBUTION BETWEEN DIFFERENT DISKS + +获取BE节点上每一个partition下的tablet在不同磁盘上的分布情况 + +``` +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition +``` + +返回值就是BE节点上每一个partition下的tablet在各个磁盘上的数量分布，只包含tablet数量。 + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + }, + { + disk_path:"***", + tablets_num:***, + }, + + ... + + ] + }, + + ... + + ] + }, + count: *** +} +``` + +``` +curl -X GET http://be_host:webserver_port/api/tablets_distribution?group_by=partition&partition_id=xxx +``` + +返回值就是BE节点上指定id的partition下的tablet在各个磁盘上的分布，包含tablet数量以及每一个tablet的id、schema hash和tablet size信息。 + +``` +{ + msg: "OK", + code: 0, + data: { + host: "***", + tablets_distribution: [ + { + partition_id:***, + disks:[ + { + disk_path:"***", + tablets_num:***, + tablets:[ + { + tablet_id:***, + schema_hash:***, + tablet_size:*** + }, + + ... + + ] + }, + + ... + + ] + } + ] + }, + count: *** +} +```