diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index 5f27b81b917d..12d30fbce848 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -1476,7 +1476,7 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { TGuard guard(StateMutex); const ui64 totalSize = Format.DiskSize; const ui64 availableSize = (ui64)Format.ChunkSize * Keeper.GetFreeChunkCount(); - + if (*Mon.PDiskBriefState != TPDiskMon::TPDisk::Error) { *Mon.FreeSpaceBytes = availableSize; *Mon.UsedSpaceBytes = totalSize - availableSize; @@ -1486,7 +1486,7 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { *Mon.UsedSpaceBytes = 32_KB; *Mon.TotalSpaceBytes = 32_KB; } - + NKikimrWhiteboard::TPDiskStateInfo& pdiskState = reportResult->PDiskState->Record; pdiskState.SetPDiskId(PDiskId); pdiskState.SetPath(Cfg->GetDevicePath()); @@ -1498,6 +1498,7 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { pdiskState.SetSystemSize(Format.ChunkSize * (Keeper.GetOwnerHardLimit(OwnerSystemLog) + Keeper.GetOwnerHardLimit(OwnerSystemReserve))); pdiskState.SetLogUsedSize(Format.ChunkSize * (Keeper.GetOwnerHardLimit(OwnerCommonStaticLog) - Keeper.GetOwnerFree(OwnerCommonStaticLog))); pdiskState.SetLogTotalSize(Format.ChunkSize * Keeper.GetOwnerHardLimit(OwnerCommonStaticLog)); + pdiskState.SetNumActiveSlots(TotalOwners); if (ExpectedSlotCount) { pdiskState.SetExpectedSlotCount(ExpectedSlotCount); } diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index 090fd2909c75..d4888c8c4bed 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -131,6 +131,7 @@ message TPDiskStateInfo { optional uint64 LogTotalSize = 21; optional uint32 ExpectedSlotCount = 22; optional uint64 EnforcedDynamicSlotSize = 23; + optional uint32 NumActiveSlots = 24; } message TEvPDiskStateRequest { diff --git a/ydb/core/viewer/json_handlers_storage.cpp b/ydb/core/viewer/json_handlers_storage.cpp new file mode 100644 index 000000000000..352f8c3d7c4a --- /dev/null +++ b/ydb/core/viewer/json_handlers_storage.cpp @@ -0,0 +1,12 @@ + +#include "json_handlers.h" + +namespace NKikimr::NViewer { + +void InitStorageGroupsJsonHandler(TJsonHandlers& jsonHandlers); + +void InitStorageJsonHandlers(TJsonHandlers& jsonHandlers) { + InitStorageGroupsJsonHandler(jsonHandlers); +} + +} diff --git a/ydb/core/viewer/json_pipe_req.h b/ydb/core/viewer/json_pipe_req.h index 8c944454fa4d..8ccc293fbec8 100644 --- a/ydb/core/viewer/json_pipe_req.h +++ b/ydb/core/viewer/json_pipe_req.h @@ -76,13 +76,16 @@ class TViewerPipeClient : public TActorBootstrapped { Set(std::unique_ptr(response->Release().Release())); } - void Error(const TString& error) { + bool Error(const TString& error) { + bool result = false; if (!IsDone()) { Span.EndError(error); + result = true; } if (!IsOk()) { Response = error; } + return result; } bool IsOk() const { @@ -105,10 +108,34 @@ class TViewerPipeClient : public TActorBootstrapped { return std::get>(Response).get(); } - T* operator ->() { + const T* Get() const { return std::get>(Response).get(); } + T& GetRef() { + return *Get(); + } + + const T& GetRef() const { + return *Get(); + } + + T* operator ->() { + return Get(); + } + + const T* operator ->() const { + return Get(); + } + + T& operator *() { + return GetRef(); + } + + const T& operator *() const { + return GetRef(); + } + TString GetError() const { return std::get(Response); } @@ -236,6 +263,17 @@ class TViewerPipeClient : public TActorBootstrapped { SendRequestToPipe(pipeClient, request.Release(), hiveId); } + TRequestResponse MakeRequestHiveStorageStats(NNodeWhiteboard::TTabletId hiveId) { + TActorId pipeClient = ConnectTabletPipe(hiveId); + THolder request = MakeHolder(); + auto response = MakeRequestToPipe(pipeClient, request.Release(), hiveId); + if (response.Span) { + auto hive_id = "#" + ::ToString(hiveId); + response.Span.Attribute("hive_id", hive_id); + } + return response; + } + NNodeWhiteboard::TTabletId GetConsoleId() { return MakeConsoleID(); } @@ -246,6 +284,12 @@ class TViewerPipeClient : public TActorBootstrapped { SendRequestToPipe(pipeClient, request.Release()); } + TRequestResponse MakeRequestConsoleListTenants() { + TActorId pipeClient = ConnectTabletPipe(GetConsoleId()); + THolder request = MakeHolder(); + return MakeRequestToPipe(pipeClient, request.Release()); + } + void RequestConsoleGetTenantStatus(const TString& path) { TActorId pipeClient = ConnectTabletPipe(GetConsoleId()); THolder request = MakeHolder(); @@ -272,6 +316,14 @@ class TViewerPipeClient : public TActorBootstrapped { SendRequestToPipe(pipeClient, request.Release()); } + TRequestResponse MakeRequestBSControllerConfigWithStoragePools() { + TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); + THolder request = MakeHolder(); + request->Record.MutableRequest()->AddCommand()->MutableQueryBaseConfig(); + request->Record.MutableRequest()->AddCommand()->MutableReadStoragePool()->SetBoxId(Max()); + return MakeRequestToPipe(pipeClient, request.Release()); + } + void RequestBSControllerInfo() { TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); THolder request = MakeHolder(); @@ -283,6 +335,11 @@ class TViewerPipeClient : public TActorBootstrapped { SendRequestToPipe(pipeClient, request.Release()); } + TRequestResponse MakeRequestBSControllerSelectGroups(THolder request, ui64 cookie) { + TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); + return MakeRequestToPipe(pipeClient, request.Release(), cookie); + } + void RequestBSControllerPDiskRestart(ui32 nodeId, ui32 pdiskId, bool force = false) { TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); THolder request = MakeHolder(); @@ -336,6 +393,30 @@ class TViewerPipeClient : public TActorBootstrapped { return MakeRequestToPipe(pipeClient, request.release(), 0/*cookie*/); } + TRequestResponse RequestBSControllerGroups() { + TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); + auto request = std::make_unique(); + return MakeRequestToPipe(pipeClient, request.release(), 0/*cookie*/); + } + + TRequestResponse RequestBSControllerPools() { + TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); + auto request = std::make_unique(); + return MakeRequestToPipe(pipeClient, request.release(), 0/*cookie*/); + } + + TRequestResponse RequestBSControllerVSlots() { + TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); + auto request = std::make_unique(); + return MakeRequestToPipe(pipeClient, request.release(), 0/*cookie*/); + } + + TRequestResponse RequestBSControllerPDisks() { + TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); + auto request = std::make_unique(); + return MakeRequestToPipe(pipeClient, request.release(), 0/*cookie*/); + } + void RequestBSControllerPDiskUpdateStatus(const NKikimrBlobStorage::TUpdateDriveStatus& driveStatus, bool force = false) { TActorId pipeClient = ConnectTabletPipe(GetBSControllerId()); THolder request = MakeHolder(); @@ -368,6 +449,35 @@ class TViewerPipeClient : public TActorBootstrapped { SendRequest(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); } + TRequestResponse MakeRequestSchemeCacheNavigate(const TString& path, ui64 cookie = 0) { + THolder request = MakeHolder(); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = SplitPath(path); + entry.RedirectRequired = false; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpPath; + request->ResultSet.emplace_back(entry); + auto response = MakeRequest(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0/*flags*/, cookie); + if (response.Span) { + response.Span.Attribute("cookie", "#" + ::ToString(cookie)); + } + return response; + } + + TRequestResponse MakeRequestSchemeCacheNavigate(const TPathId& pathId, ui64 cookie = 0) { + THolder request = MakeHolder(); + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.TableId.PathId = pathId; + entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByTableId; + entry.RedirectRequired = false; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpPath; + request->ResultSet.emplace_back(entry); + auto response = MakeRequest(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()), 0/*flags*/, cookie); + if (response.Span) { + response.Span.Attribute("response", "#" + ::ToString(cookie)); + } + return response; + } + void RequestTxProxyDescribe(const TString& path) { THolder request(new TEvTxUserProxy::TEvNavigate()); request->Record.MutableDescribePath()->SetPath(path); @@ -445,6 +555,10 @@ class TViewerPipeClient : public TActorBootstrapped { } } + bool IsLastRequest() const { + return Requests == 1; + } + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { if (ev->Get()->Status != NKikimrProto::OK) { ui32 requests = FailPipeConnect(ev->Get()->TabletId); diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index fba41698e9ed..0232a1b03fbf 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -393,8 +393,54 @@ message TTenantInfo { repeated string Errors = 2; } +message TStoragePDisk { + string PDiskId = 1; + string Path = 2; + string Type = 3; + string Guid = 4; + uint64 Category = 5; + uint64 TotalSize = 6; + uint64 AvailableSize = 7; + string Status = 8; + EFlag DiskSpace = 9; + string DecommitStatus = 10; + uint64 SlotSize = 11; + NKikimrWhiteboard.TPDiskStateInfo Whiteboard = 50; +} + +message TStorageVDisk { + string VDiskId = 1; + uint32 NodeId = 2; + uint64 AllocatedSize = 3; + uint64 AvailableSize = 4; + string Kind = 5; + string Status = 6; + EFlag DiskSpace = 7; + repeated TStorageVDisk Donors = 8; + TStoragePDisk PDisk = 20; + NKikimrWhiteboard.TVDiskStateInfo Whiteboard = 50; +} + message TStorageGroupInfo { string GroupId = 1; + uint64 GroupGeneration = 2; + string PoolName = 3; + bool Encryption = 4; + EFlag Overall = 5; + EFlag DiskSpace = 6; + string Kind = 7; + string MediaType = 8; + string ErasureSpecies = 9; + uint64 AllocationUnits = 10; + string State = 11; + uint64 MissingDisks = 12; // Degraded + uint64 Used = 13; + uint64 Limit = 14; + uint64 Available = 15; + double Usage = 16; + uint64 Read = 17; + uint64 Write = 18; + repeated TStorageVDisk VDisks = 20; } message TStoragePoolInfo { @@ -420,6 +466,18 @@ message TStorageInfo { repeated TStorageGroupInfo StorageGroups = 5; } +message TStorageGroupsInfo { + uint64 Version = 1; + optional uint64 TotalGroups = 2; + optional uint64 FoundGroups = 3; + optional uint64 FieldsAvailable = 4; + optional uint64 FieldsRequired = 5; + optional bool NeedFilter = 6; + optional bool NeedSort = 7; + optional bool NeedLimit = 8; + repeated TStorageGroupInfo StorageGroups = 10; +} + message TStorageUsageStats { uint32 Pace = 1; repeated uint32 Buckets = 2; diff --git a/ydb/core/viewer/storage_groups.cpp b/ydb/core/viewer/storage_groups.cpp new file mode 100644 index 000000000000..2bdecfa14f4c --- /dev/null +++ b/ydb/core/viewer/storage_groups.cpp @@ -0,0 +1,1810 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "viewer.h" +#include "viewer_helper.h" +#include "json_handlers.h" +#include "json_pipe_req.h" +#include "json_vdiskinfo.h" +#include "json_pdiskinfo.h" +#include "json_bsgroupinfo.h" + +namespace NKikimr { +namespace NViewer { + +using namespace NProtobufJson; + +using TNodeId = ui32; +using TGroupId = ui32; + +struct TPDiskId { + TNodeId NodeId; + ui32 PDiskId; + + TPDiskId() = default; + + TPDiskId(TNodeId nodeId, ui32 pdiskId) + : NodeId(nodeId) + , PDiskId(pdiskId) + {} + + TPDiskId(const NKikimrSysView::TPDiskKey& key) + : NodeId(key.GetNodeId()) + , PDiskId(key.GetPDiskId()) + {} + + TPDiskId(const NKikimrSysView::TVSlotKey& key) + : NodeId(key.GetNodeId()) + , PDiskId(key.GetPDiskId()) + {} + + bool operator ==(const TPDiskId& a) const = default; +}; + +struct TVSlotId : TPDiskId { + ui32 VDiskSlotId; + + TVSlotId() = default; + + TVSlotId(TNodeId nodeId, ui32 pdiskId, ui32 vdiskSlotId) + : TPDiskId(nodeId, pdiskId) + , VDiskSlotId(vdiskSlotId) + {} + + TVSlotId(const NKikimrBlobStorage::TVSlotId& proto) + : TVSlotId(proto.GetNodeId(), proto.GetPDiskId(), proto.GetVSlotId()) + {} + + TVSlotId(const NKikimrSysView::TVSlotKey& proto) + : TVSlotId(proto.GetNodeId(), proto.GetPDiskId(), proto.GetVSlotId()) + {} + + TVSlotId(const NKikimrWhiteboard::TVDiskStateInfo& proto) + : TVSlotId(proto.GetNodeId(), proto.GetPDiskId(), proto.GetVDiskSlotId()) + {} + + bool operator ==(const TVSlotId& a) const = default; +}; + +} +} + +template<> +struct std::hash { + std::size_t operator ()(const NKikimr::NViewer::TPDiskId& s) const { + return ::std::hash()(s.NodeId) + ^ (::std::hash()(s.PDiskId) << 1); + } +}; + +template<> +struct std::hash { + std::size_t operator ()(const NKikimr::NViewer::TVSlotId& s) const { + return ::std::hash()(s.NodeId) + ^ (::std::hash()(s.PDiskId) << 1) + ^ (::std::hash()(s.VDiskSlotId) << 2); + } +}; + +namespace NKikimr { +namespace NViewer { + +using namespace NActors; +using namespace NNodeWhiteboard; + +using ::google::protobuf::FieldDescriptor; + +enum class EGroupFields : ui8 { + GroupId, + PoolName, + Kind, + MediaType, + Erasure, + MissingDisks, + State, + Usage, + Encryption, + Used, + Limit, + Read, + Write, + Available, + AllocationUnits, + NodeId, + PDiskId, + VDisk, // VDisk information + PDisk, // PDisk information + COUNT +}; + +constexpr ui8 operator +(EGroupFields e) { + return static_cast(e); +} + +using TFieldsType = std::bitset<+EGroupFields::COUNT>; + +class TStorageGroups : public TViewerPipeClient { +public: + using TBase = TViewerPipeClient; + using TThis = TStorageGroups; + + // Common + std::optional> DatabaseNavigateResult; + std::unordered_map> NavigateKeySetResult; + std::unordered_map PathId2HiveId; + std::unordered_map> HiveStorageStats; + ui64 HiveStorageStatsInFlight = 0; + + // BSC + bool FallbackToWhiteboard = false; + bool FillDisksFromWhiteboard = false; + std::optional> GetGroupsResponse; + std::optional> GetStoragePoolsResponse; + std::optional> GetVSlotsResponse; + std::optional> GetPDisksResponse; + + // Whiteboard + std::optional> NodesInfo; + std::unordered_map> BSGroupStateResponse; + ui64 BSGroupStateRequestsInFlight = 0; + std::unordered_map> VDiskStateResponse; + ui64 VDiskStateRequestsInFlight = 0; + std::unordered_map> PDiskStateResponse; + ui64 PDiskStateRequestsInFlight = 0; + + ui32 Timeout = 0; + TString Database; + bool Direct = false; + TString Filter; + std::unordered_set FilterStoragePools; + std::unordered_set FilterGroupIds; + std::unordered_set FilterNodeIds; + std::unordered_set FilterPDiskIds; + std::vector SubscriptionNodeIds; + + enum class EWith { + Everything, + MissingDisks, + SpaceProblems, + }; + + enum ETimeoutTag { + TimeoutBSC, + TimeoutFinal, + }; + + EGroupFields GroupSort = EGroupFields::PoolName; + EWith With = EWith::Everything; + bool ReverseSort = false; + std::optional Offset; + std::optional Limit; + ui32 SpaceUsageProblem = 90; // % + + struct TPDisk { + ui32 PDiskId = 0; + TNodeId NodeId = 0; + TString Type; + TString Path; + ui64 Guid = 0; + ui64 AvailableSize = 0; + ui64 TotalSize = 0; + TString Status; + TInstant StatusChangeTimestamp; + ui64 EnforcedDynamicSlotSize = 0; + ui32 ExpectedSlotCount = 0; + ui32 NumActiveSlots = 0; + ui64 Category = 0; + TString DecommitStatus; + NKikimrViewer::EFlag DiskSpace = NKikimrViewer::EFlag::Grey; + + void SetCategory(ui64 category) { + Category = category; + switch (TPDiskCategory(Category).Type()) { + case NPDisk::EDeviceType::DEVICE_TYPE_ROT: + Type = "hdd"; + break; + case NPDisk::EDeviceType::DEVICE_TYPE_SSD: + Type = "ssd"; + break; + case NPDisk::EDeviceType::DEVICE_TYPE_NVME: + Type = "nvme"; + break; + case NPDisk::EDeviceType::DEVICE_TYPE_UNKNOWN: + break; + } + } + + ui64 GetSlotTotalSize() const { + if (EnforcedDynamicSlotSize) { + return EnforcedDynamicSlotSize; + } + if (ExpectedSlotCount) { + return TotalSize / ExpectedSlotCount; + } + if (NumActiveSlots) { + return TotalSize / NumActiveSlots; + } + return TotalSize; + } + + TString GetPDiskId() const { + return TStringBuilder() << NodeId << '-' << PDiskId; + } + }; + + struct TVDisk { + TVDiskID VDiskId; + TVSlotId VSlotId; + ui64 AllocatedSize = 0; + ui64 AvailableSize = 0; + TString Status; + NKikimrBlobStorage::EVDiskStatus VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR; + ui64 Read = 0; + ui64 Write = 0; + NKikimrViewer::EFlag DiskSpace = NKikimrViewer::EFlag::Grey; + bool Donor = false; + std::vector Donors; + + TString GetVDiskId() const { + return TStringBuilder() << VDiskId.GroupID.GetRawId() << '-' + << VDiskId.GroupGeneration << '-' + << int(VDiskId.FailRealm) << '-' + << int(VDiskId.FailDomain) << '-' + << int(VDiskId.VDisk); + } + }; + + struct TGroup { + TString PoolName; + TGroupId GroupId = 0; + ui32 GroupGeneration = 0; + ui64 BoxId = 0; + ui64 PoolId = 0; + ui64 SchemeShardId = 0; + ui64 PathId = 0; + TString Kind; + TString MediaType; + TString Erasure; + TErasureType::EErasureSpecies ErasureSpecies = TErasureType::ErasureNone; + TString State; + ui32 EncryptionMode = 0; + ui64 AllocationUnits = 0; + float Usage = 0; + ui64 Used = 0; + ui64 Limit = 0; + ui64 Available = 0; + ui64 Read = 0; + ui64 Write = 0; + ui32 MissingDisks = 0; + ui64 PutTabletLogLatency = 0; + ui64 PutUserDataLatency = 0; + ui64 GetFastLatency = 0; + NKikimrViewer::EFlag Overall = NKikimrViewer::EFlag::Grey; + NKikimrViewer::EFlag DiskSpace = NKikimrViewer::EFlag::Grey; + + std::vector VDisks; + std::vector VDiskNodeIds; // filter nodes to request disk info from the whiteboard. could be duplicated. + + TString PrintDomains(const std::vector& failedDomains) { + TString result; + result += ::ToString(failedDomains.size()); + result += '('; + for (ui8 domains : failedDomains) { + if (!result.empty()) { + result += ','; + } + result += ::ToString(domains); + } + result += ')'; + return result; + } + + // none: ok, dead:1 + // block-4-2: ok, replicating:1 starting:1, degraded:1, degraded:2, dead:3 + // mirror-3-dc: ok, degraded:1(1), degraded:1(2), degraded:1(3), degraded:2(3,1), dead:3(3,1,1) + + void CalcState() { + MissingDisks = 0; + ui64 allocated = 0; + ui64 limit = 0; + ui32 startingDisks = 0; + ui32 replicatingDisks = 0; + static_assert(sizeof(TVDiskID::FailDomain) == 1, "expecting byte"); + static_assert(sizeof(TVDiskID::FailRealm) == 1, "expecting byte"); + std::vector failedDomainsPerRealm; + for (const TVDisk& vdisk : VDisks) { + if (vdisk.VDiskStatus != NKikimrBlobStorage::EVDiskStatus::READY) { + if (ErasureSpecies == TErasureType::ErasureMirror3dc) { + if (failedDomainsPerRealm.size() <= vdisk.VDiskId.FailRealm) { + failedDomainsPerRealm.resize(vdisk.VDiskId.FailRealm + 1); + } + failedDomainsPerRealm[vdisk.VDiskId.FailRealm]++; + } + ++MissingDisks; + if (vdisk.VDiskStatus == NKikimrBlobStorage::EVDiskStatus::INIT_PENDING) { + ++startingDisks; + } + if (vdisk.VDiskStatus == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { + ++replicatingDisks; + } + } + allocated += vdisk.AllocatedSize; + limit += vdisk.AllocatedSize + vdisk.AvailableSize; + DiskSpace = std::max(DiskSpace, vdisk.DiskSpace); + } + if (MissingDisks == 0) { + Overall = NKikimrViewer::EFlag::Green; + State = "ok"; + } else { + if (ErasureSpecies == TErasureType::ErasureNone) { + TString state; + Overall = NKikimrViewer::EFlag::Red; + if (MissingDisks == startingDisks) { + state = "starting"; + } else { + state = "dead"; + } + State = TStringBuilder() << state << ':' << MissingDisks; + } else if (ErasureSpecies == TErasureType::Erasure4Plus2Block) { + TString state; + if (MissingDisks > 2) { + Overall = NKikimrViewer::EFlag::Red; + state = "dead"; + } else if (MissingDisks == 2) { + Overall = NKikimrViewer::EFlag::Orange; + state = "degraded"; + } else if (MissingDisks == 1) { + if (MissingDisks == replicatingDisks + startingDisks) { + Overall = NKikimrViewer::EFlag::Blue; + if (replicatingDisks) { + state = "replicating"; + } else { + state = "starting"; + } + } else { + Overall = NKikimrViewer::EFlag::Yellow; + state = "degraded"; + } + } + State = TStringBuilder() << state << ':' << MissingDisks; + } else if (ErasureSpecies == TErasureType::ErasureMirror3dc) { + std::sort(failedDomainsPerRealm.begin(), failedDomainsPerRealm.end(), std::greater()); + while (!failedDomainsPerRealm.empty() && failedDomainsPerRealm.back() == 0) { + failedDomainsPerRealm.pop_back(); + } + TString state; + if (failedDomainsPerRealm.size() > 2 || (failedDomainsPerRealm.size() == 2 && failedDomainsPerRealm[1] > 1)) { + Overall = NKikimrViewer::EFlag::Red; + state = "dead"; + } else if (failedDomainsPerRealm.size() == 2) { + Overall = NKikimrViewer::EFlag::Orange; + state = "degraded"; + } else if (failedDomainsPerRealm.size()) { + if (MissingDisks == replicatingDisks + startingDisks) { + Overall = NKikimrViewer::EFlag::Blue; + if (replicatingDisks > startingDisks) { + state = "replicating"; + } else { + state = "starting"; + } + } else { + Overall = NKikimrViewer::EFlag::Yellow; + state = "degraded"; + } + } + State = TStringBuilder() << state << ':' << PrintDomains(failedDomainsPerRealm); + } + } + Used = allocated; + Limit = limit; + Usage = Limit ? 100.0 * Used / Limit : 0; + if (Usage >= 95) { + DiskSpace = std::max(DiskSpace, NKikimrViewer::EFlag::Red); + } else if (Usage >= 90) { + DiskSpace = std::max(DiskSpace, NKikimrViewer::EFlag::Orange); + } else if (Usage >= 85) { + DiskSpace = std::max(DiskSpace, NKikimrViewer::EFlag::Yellow); + } else { + DiskSpace = std::max(DiskSpace, NKikimrViewer::EFlag::Green); + } + } + + void CalcAvailable(const std::unordered_map& pDisks) { + ui64 available = 0; + for (const TVDisk& vdisk : VDisks) { + auto itPDisk = pDisks.find(vdisk.VSlotId); + if (itPDisk != pDisks.end()) { + available += std::min(itPDisk->second.GetSlotTotalSize() - vdisk.AllocatedSize, vdisk.AvailableSize); + DiskSpace = std::max(DiskSpace, vdisk.DiskSpace); + } + } + Available = available; + } + + void CalcReadWrite() { + ui64 read = 0; + ui64 write = 0; + for (const TVDisk& vdisk : VDisks) { + read += vdisk.Read; + write += vdisk.Write; + } + Read = read; + Write = write; + } + }; + + std::vector Groups; + std::unordered_map GroupsByGroupId; + std::unordered_map PDisks; + std::unordered_map VSlotsByVSlotId; + std::unordered_map VDisksByVSlotId; + std::unordered_map PDisksByPDiskId; + + TFieldsType FieldsRequired; + TFieldsType FieldsAvailable; + const TFieldsType FieldsAll = TFieldsType().set(); + const TFieldsType FieldsBsGroups = TFieldsType().set(+EGroupFields::GroupId) + .set(+EGroupFields::Erasure) + .set(+EGroupFields::Usage) + .set(+EGroupFields::Used) + .set(+EGroupFields::Limit); + const TFieldsType FieldsBsPools = TFieldsType().set(+EGroupFields::PoolName) + .set(+EGroupFields::Kind) + .set(+EGroupFields::MediaType) + .set(+EGroupFields::Encryption); + const TFieldsType FieldsBsVSlots = TFieldsType().set(+EGroupFields::NodeId) + .set(+EGroupFields::PDiskId) + .set(+EGroupFields::VDisk); + const TFieldsType FieldsBsPDisks = TFieldsType().set(+EGroupFields::PDisk); + const TFieldsType FieldsGroupState = TFieldsType().set(+EGroupFields::Used) + .set(+EGroupFields::Limit) + .set(+EGroupFields::Available) + .set(+EGroupFields::Usage) + .set(+EGroupFields::MissingDisks) + .set(+EGroupFields::State); + const TFieldsType FieldsHive = TFieldsType().set(+EGroupFields::AllocationUnits); + const TFieldsType FieldsWbGroups = TFieldsType().set(+EGroupFields::GroupId) + .set(+EGroupFields::Erasure) + .set(+EGroupFields::PoolName) + .set(+EGroupFields::Encryption); + const TFieldsType FieldsWbDisks = TFieldsType().set(+EGroupFields::NodeId) + .set(+EGroupFields::PDiskId) + .set(+EGroupFields::VDisk) + .set(+EGroupFields::PDisk) + .set(+EGroupFields::Read) + .set(+EGroupFields::Write); + + bool FieldsNeeded(TFieldsType fields) const { + return (FieldsRequired & (fields & ~FieldsAvailable)).any(); + } + + bool NeedFilter = false; + bool NeedSort = false; + bool NeedLimit = false; + ui64 TotalGroups = 0; + ui64 FoundGroups = 0; + + static EGroupFields ParseEGroupFields(TStringBuf field) { + EGroupFields result = EGroupFields::COUNT; + if (field == "PoolName") { + result = EGroupFields::PoolName; + } else if (field == "Kind") { + result = EGroupFields::Kind; + } else if (field == "MediaType") { + result = EGroupFields::MediaType; + } else if (field == "Erasure") { + result = EGroupFields::Erasure; + } else if (field == "Degraded" || field == "MissingDisks") { + result = EGroupFields::MissingDisks; + } else if (field == "State") { + result = EGroupFields::State; + } else if (field == "Usage") { + result = EGroupFields::Usage; + } else if (field == "GroupId") { + result = EGroupFields::GroupId; + } else if (field == "Encryption") { + result = EGroupFields::Encryption; + } else if (field == "Used") { + result = EGroupFields::Used; + } else if (field == "Limit") { + result = EGroupFields::Limit; + } else if (field == "Read") { + result = EGroupFields::Read; + } else if (field == "Write") { + result = EGroupFields::Write; + } else if (field == "AllocationUnits") { + result = EGroupFields::AllocationUnits; + } else if (field == "VDisk") { + result = EGroupFields::VDisk; + } else if (field == "PDisk") { + result = EGroupFields::PDisk; + } + return result; + } + + TStorageGroups(IViewer* viewer, NMon::TEvHttpInfo::TPtr& ev) + : TBase(viewer, ev) + { + const auto& params(Event->Get()->Request.GetParams()); + InitConfig(params); + Timeout = FromStringWithDefault(params.Get("timeout"), 10000); + Database = params.Get("tenant"); + if (Database.empty()) { + Database = params.Get("database"); + } + if (!Database.empty()) { + NeedFilter = true; + } + Direct = FromStringWithDefault(params.Get("direct"), Direct); + TString filterStoragePool = params.Get("pool"); + if (!filterStoragePool.empty()) { + FilterStoragePools.emplace(filterStoragePool); + } + SplitIds(params.Get("node_id"), ',', FilterNodeIds); + SplitIds(params.Get("pdisk_id"), ',', FilterPDiskIds); + SplitIds(params.Get("group_id"), ',', FilterGroupIds); + if (!FilterStoragePools.empty() || !FilterNodeIds.empty() || !FilterPDiskIds.empty() || !FilterGroupIds.empty()) { + NeedFilter = true; + } + if (params.Has("filter")) { + Filter = params.Get("filter"); + NeedFilter = true; + } + if (params.Get("with") == "missing") { + With = EWith::MissingDisks; + NeedFilter = true; + } if (params.Get("with") == "space") { + With = EWith::SpaceProblems; + NeedFilter = true; + } + if (params.Has("offset")) { + Offset = FromStringWithDefault(params.Get("offset"), 0); + NeedLimit = true; + } + if (params.Has("limit")) { + Limit = FromStringWithDefault(params.Get("limit"), std::numeric_limits::max()); + NeedLimit = true; + } + TStringBuf sort = params.Get("sort"); + if (sort) { + NeedSort = true; + if (sort.StartsWith("-") || sort.StartsWith("+")) { + ReverseSort = (sort[0] == '-'); + sort.Skip(1); + } + GroupSort = ParseEGroupFields(sort); + FieldsRequired.set(+GroupSort); + } + bool whiteboardOnly = FromStringWithDefault(params.Get("whiteboard_only"), false); + if (whiteboardOnly) { + FieldsRequired |= FieldsWbGroups; + FieldsRequired |= FieldsWbDisks; + FallbackToWhiteboard = true; + } + bool bscOnly = FromStringWithDefault(params.Get("bsc_only"), false); + if (bscOnly) { + FieldsRequired |= FieldsBsGroups; + FieldsRequired |= FieldsBsPools; + FieldsRequired |= FieldsBsVSlots; + FieldsRequired |= FieldsBsPDisks; + } + bool groupsOnly = FromStringWithDefault(params.Get("groups_only"), false); + if (groupsOnly) { + FieldsRequired.set(+EGroupFields::GroupId); + } + FillDisksFromWhiteboard = FromStringWithDefault(params.Get("fill_disks_from_whiteboard"), FillDisksFromWhiteboard); + TString fieldsRequired = FromStringWithDefault(params.Get("fields_required")); + if (!fieldsRequired.empty()) { + if (fieldsRequired == "all") { + FieldsRequired = FieldsAll; + } else { + TStringBuf source = fieldsRequired; + for (TStringBuf value = source.NextTok(", "); !value.empty(); value = source.NextTok(", ")) { + EGroupFields field = ParseEGroupFields(value); + if (field != EGroupFields::COUNT) { + FieldsRequired.set(+field); + } + } + } + } + if (FieldsRequired.none()) { + FieldsRequired = FieldsAll; + } + } + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::VIEWER_HANDLER; + } + + virtual void Bootstrap() { + Direct |= TBase::Event->Get()->Request.GetUri().StartsWith("/node/"); // we're already forwarding + Direct |= (Database == AppData()->TenantName) || Database.empty(); // we're already on the right node or don't use database filter + + if (Database && !Direct) { + BLOG_TRACE("Requesting StateStorageEndpointsLookup for " << Database); + RequestStateStorageEndpointsLookup(Database); // to find some dynamic node and redirect query there + } else { + if (Database) { + DatabaseNavigateResult = MakeRequestSchemeCacheNavigate(Database, 0); + } + if (FallbackToWhiteboard) { + RequestWhiteboard(); + } else { + GetGroupsResponse = RequestBSControllerGroups(); + if (FieldsNeeded(FieldsBsPools)) { + GetStoragePoolsResponse = RequestBSControllerPools(); + } + if (FieldsNeeded(FieldsBsVSlots)) { + GetVSlotsResponse = RequestBSControllerVSlots(); + } + if (FieldsNeeded(FieldsBsPDisks)) { + GetPDisksResponse = RequestBSControllerPDisks(); + } + } + } + TBase::Become(&TThis::StateWork); + Schedule(TDuration::MilliSeconds(Timeout * 50 / 100), new TEvents::TEvWakeup(TimeoutBSC)); // 50% timeout (for bsc) + Schedule(TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup(TimeoutFinal)); // timeout for the rest + } + + void Handle(TEvStateStorage::TEvBoardInfo::TPtr& ev) { + BLOG_TRACE("Received TEvBoardInfo"); + TBase::ReplyAndPassAway(MakeForward(GetNodesFromBoardReply(ev))); + } + + void PassAway() override { + std::vector passedNodes; + for (const TNodeId nodeId : SubscriptionNodeIds) { + if (passedNodes.size() <= nodeId) { + passedNodes.resize(nodeId + 1); + } else { + if (passedNodes[nodeId]) { + continue; + } + } + Send(TActivationContext::InterconnectProxy(nodeId), new TEvents::TEvUnsubscribe()); + passedNodes[nodeId] = true; + } + TBase::PassAway(); + } + + void ApplyFilter() { + if (NeedFilter) { + if (!FilterGroupIds.empty()) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + if (FilterGroupIds.count(itGroup->GroupId)) { + continue; + } + itGroup = Groups.erase(itGroup); + } + FilterGroupIds.clear(); + GroupsByGroupId.clear(); + } + if (!FilterStoragePools.empty() && FieldsAvailable.test(+EGroupFields::PoolName)) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + if (FilterStoragePools.count(itGroup->PoolName)) { + continue; + } + itGroup = Groups.erase(itGroup); + } + FilterStoragePools.clear(); + GroupsByGroupId.clear(); + } + if (!FilterNodeIds.empty() && FieldsAvailable.test(+EGroupFields::NodeId)) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + bool found = false; + for (const auto& vdisk : itGroup->VDisks) { + if (FilterNodeIds.count(vdisk.VSlotId.NodeId)) { + found = true; + break; + } + } + if (found) { + continue; + } + itGroup = Groups.erase(itGroup); + } + FilterNodeIds.clear(); + GroupsByGroupId.clear(); + } + if (!FilterPDiskIds.empty() && FieldsAvailable.test(+EGroupFields::PDiskId)) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + bool found = false; + for (const auto& vdisk : itGroup->VDisks) { + if (FilterPDiskIds.count(vdisk.VSlotId.PDiskId)) { + found = true; + break; + } + } + if (found) { + continue; + } + itGroup = Groups.erase(itGroup); + } + FilterPDiskIds.clear(); + GroupsByGroupId.clear(); + } + if (With == EWith::MissingDisks && FieldsAvailable.test(+EGroupFields::MissingDisks)) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + if (itGroup->MissingDisks != 0) { + continue; + } + itGroup = Groups.erase(itGroup); + } + With = EWith::Everything; + GroupsByGroupId.clear(); + } + if (With == EWith::SpaceProblems && FieldsAvailable.test(+EGroupFields::Usage)) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + if (itGroup->Usage >= SpaceUsageProblem) { + continue; + } + itGroup = Groups.erase(itGroup); + } + With = EWith::Everything; + GroupsByGroupId.clear(); + } + if (!Filter.empty() && FieldsAvailable.test(+EGroupFields::PoolName) && FieldsAvailable.test(+EGroupFields::GroupId)) { + for (auto itGroup = Groups.begin(); itGroup != Groups.end(); ++itGroup) { + if (itGroup->PoolName.Contains(Filter)) { + continue; + } + if (::ToString(itGroup->GroupId).Contains(Filter)) { + continue; + } + itGroup = Groups.erase(itGroup); + } + Filter.clear(); + GroupsByGroupId.clear(); + } + NeedFilter = (With == EWith::Everything) && Filter.empty() && FilterNodeIds.empty() && FilterPDiskIds.empty() && FilterGroupIds.empty(); + FoundGroups = Groups.size(); + } + } + + void ApplySort() { + if (NeedSort && FieldsAvailable.test(+GroupSort)) { + switch (GroupSort) { + case EGroupFields::GroupId: + SortCollection(Groups, [](const TGroup& group) { return group.GroupId; }, ReverseSort); + break; + case EGroupFields::Erasure: + SortCollection(Groups, [](const TGroup& group) { return group.Erasure; }, ReverseSort); + break; + case EGroupFields::Usage: + SortCollection(Groups, [](const TGroup& group) { return group.Usage; }, ReverseSort); + break; + case EGroupFields::Used: + SortCollection(Groups, [](const TGroup& group) { return group.Used; }, ReverseSort); + break; + case EGroupFields::Limit: + SortCollection(Groups, [](const TGroup& group) { return group.Limit; }, ReverseSort); + break; + case EGroupFields::Available: + SortCollection(Groups, [](const TGroup& group) { return group.Available; }, ReverseSort); + break; + case EGroupFields::PoolName: + SortCollection(Groups, [](const TGroup& group) { return group.PoolName; }, ReverseSort); + break; + case EGroupFields::Kind: + SortCollection(Groups, [](const TGroup& group) { return group.Kind; }, ReverseSort); + break; + case EGroupFields::Encryption: + SortCollection(Groups, [](const TGroup& group) { return group.EncryptionMode; }, ReverseSort); + break; + case EGroupFields::AllocationUnits: + SortCollection(Groups, [](const TGroup& group) { return group.AllocationUnits; }, ReverseSort); + break; + case EGroupFields::MediaType: + SortCollection(Groups, [](const TGroup& group) { return group.MediaType; }, ReverseSort); + break; + case EGroupFields::MissingDisks: + SortCollection(Groups, [](const TGroup& group) { return group.MissingDisks; }, ReverseSort); + break; + case EGroupFields::Read: + SortCollection(Groups, [](const TGroup& group) { return group.Read; }, ReverseSort); + break; + case EGroupFields::Write: + SortCollection(Groups, [](const TGroup& group) { return group.Write; }, ReverseSort); + break; + case EGroupFields::State: + SortCollection(Groups, [](const TGroup& group) { return group.State; }, ReverseSort); + break; + case EGroupFields::PDiskId: + case EGroupFields::NodeId: + case EGroupFields::PDisk: + case EGroupFields::VDisk: + case EGroupFields::COUNT: + break; + } + NeedSort = false; + GroupsByGroupId.clear(); + } + } + + void ApplyLimit() { + if (!NeedFilter && !NeedSort && NeedLimit) { + if (Offset) { + Groups.erase(Groups.begin(), Groups.begin() + std::min(*Offset, Groups.size())); + GroupsByGroupId.clear(); + } + if (Limit) { + Groups.resize(std::min(*Limit, Groups.size())); + GroupsByGroupId.clear(); + } + NeedLimit = false; + } + } + + void ApplyEverything() { + ApplyFilter(); + ApplySort(); + ApplyLimit(); + } + + void CollectHiveData() { + if (FieldsNeeded(FieldsHive)) { + if (!Groups.empty()) { + ui64 hiveId = AppData()->DomainsInfo->GetHive(); + if (hiveId != TDomainsInfo::BadTabletId) { + if (HiveStorageStats.count(hiveId) == 0) { + HiveStorageStats.emplace(hiveId, MakeRequestHiveStorageStats(hiveId)); + ++HiveStorageStatsInFlight; + } + } + } + for (const TGroup& group : Groups) { + TPathId pathId(group.SchemeShardId, group.PathId); + if (NavigateKeySetResult.count(pathId) == 0) { + ui64 cookie = NavigateKeySetResult.size(); + NavigateKeySetResult.emplace(pathId, MakeRequestSchemeCacheNavigate(pathId, cookie)); + } + } + } + } + + void RebuildGroupsByGroupId() { + GroupsByGroupId.clear(); + for (TGroup& group : Groups) { + GroupsByGroupId.emplace(group.GroupId, &group); + } + } + + static TPathId GetPathId(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + if (ev->Get()->Request->ResultSet.size() == 1) { + if (ev->Get()->Request->ResultSet.begin()->Self) { + const auto& info = ev->Get()->Request->ResultSet.begin()->Self->Info; + return TPathId(info.GetSchemeshardId(), info.GetPathId()); + } + if (ev->Get()->Request->ResultSet.begin()->TableId) { + return ev->Get()->Request->ResultSet.begin()->TableId.PathId; + } + } + return {}; + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + bool firstNavigate = (ev->Cookie == 0); + TPathId pathId = GetPathId(ev); + if (firstNavigate && DatabaseNavigateResult.has_value() && pathId) { + NavigateKeySetResult.emplace(pathId, std::move(*DatabaseNavigateResult)); + } + auto itNavigateKeySetResult = NavigateKeySetResult.find(pathId); + if (itNavigateKeySetResult == NavigateKeySetResult.end()) { + BLOG_W("Invalid NavigateKeySetResult PathId: " << pathId << " Path: " << CanonizePath(ev->Get()->Request->ResultSet.begin()->Path)); + return RequestDone(); + } + auto& navigateResult(itNavigateKeySetResult->second); + if (ev->Get()->Request->ResultSet.size() == 1) { + if (ev->Get()->Request->ResultSet.begin()->Status == NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { + navigateResult.Set(std::move(ev)); + } else { + navigateResult.Error(TStringBuilder() << "Error " << ev->Get()->Request->ResultSet.begin()->Status); + } + } else { + navigateResult.Error(TStringBuilder() << "Invalid number of results: " << ev->Get()->Request->ResultSet.size()); + } + if (navigateResult.IsOk()) { + TString path = CanonizePath(navigateResult->Request->ResultSet.begin()->Path); + TIntrusiveConstPtr domainDescription = navigateResult->Request->ResultSet.begin()->DomainDescription; + TIntrusiveConstPtr domainInfo = navigateResult->Request->ResultSet.begin()->DomainInfo; + if (domainInfo != nullptr && domainDescription != nullptr) { + if (FieldsNeeded(FieldsHive)) { + TTabletId hiveId = domainInfo->Params.GetHive(); + if (hiveId != 0 && HiveStorageStats.count(hiveId) == 0) { + HiveStorageStats.emplace(hiveId, MakeRequestHiveStorageStats(hiveId)); + ++HiveStorageStatsInFlight; + } + } + if (Database && firstNavigate) { // filter by database + for (const auto& storagePool : domainDescription->Description.GetStoragePools()) { + TString storagePoolName = storagePool.GetName(); + FilterStoragePools.emplace(storagePoolName); + } + } + } + } + RequestDone(); + } + + void Handle(TEvHive::TEvResponseHiveStorageStats::TPtr& ev) { + auto itHiveStorageStats = HiveStorageStats.find(ev->Cookie); + if (itHiveStorageStats != HiveStorageStats.end()) { + itHiveStorageStats->second.Set(std::move(ev)); + if (GroupsByGroupId.empty()) { + RebuildGroupsByGroupId(); + } + for (const auto& pbPool : itHiveStorageStats->second->Record.GetPools()) { + for (const auto& pbGroup : pbPool.GetGroups()) { + auto itGroup = GroupsByGroupId.find(pbGroup.GetGroupID()); + if (itGroup != GroupsByGroupId.end()) { + itGroup->second->AllocationUnits += pbGroup.GetAcquiredUnits(); + } + } + } + + } + if (--HiveStorageStatsInFlight == 0) { + FieldsAvailable |= FieldsHive; + ApplyEverything(); + } + RequestDone(); + } + + static TString GetMediaType(const TString& mediaType) { + if (mediaType.StartsWith("Type:")) { + return mediaType.substr(5); + } + return mediaType; + } + + void FillVDiskFromVSlotInfo(TVDisk& vDisk, TVSlotId vSlotId, const NKikimrSysView::TVSlotInfo& info) { + vDisk.VDiskId = TVDiskID(info.GetGroupId(), + info.GetGroupGeneration(), + static_cast(info.GetFailRealm()), + static_cast(info.GetFailDomain()), + static_cast(info.GetVDisk())); + vDisk.VSlotId = vSlotId; + vDisk.AllocatedSize = info.GetAllocatedSize(); + vDisk.AvailableSize = info.GetAvailableSize(); + //vDisk.Kind = info.GetKind(); + vDisk.Status = info.GetStatusV2(); + NKikimrBlobStorage::EVDiskStatus_Parse(info.GetStatusV2(), &vDisk.VDiskStatus); + } + + void ProcessBSControllerResponses() { + if (GetGroupsResponse && GetGroupsResponse->IsOk() && FieldsNeeded(FieldsBsGroups)) { + Groups.reserve(GetGroupsResponse->Get()->Record.EntriesSize()); + for (const NKikimrSysView::TGroupEntry& entry : GetGroupsResponse->Get()->Record.GetEntries()) { + const NKikimrSysView::TGroupInfo& info = entry.GetInfo(); + TGroup& group = Groups.emplace_back(); + group.GroupId = entry.GetKey().GetGroupId(); + group.GroupGeneration = info.GetGeneration(); + group.BoxId = info.GetBoxId(); + group.PoolId = info.GetStoragePoolId(); + group.Erasure = info.GetErasureSpeciesV2(); + group.ErasureSpecies = TErasureType::ErasureSpeciesByName(group.Erasure); + //group.Used = info.GetAllocatedSize(); + //group.Limit = info.GetAllocatedSize() + info.GetAvailableSize(); + //group.Usage = group.Limit ? 100.0 * group.Used / group.Limit : 0; + group.PutTabletLogLatency = info.GetPutTabletLogLatency(); + group.PutUserDataLatency = info.GetPutUserDataLatency(); + group.GetFastLatency = info.GetGetFastLatency(); + } + GroupsByGroupId.clear(); + FoundGroups = TotalGroups = Groups.size(); + FieldsAvailable |= FieldsBsGroups; + ApplyEverything(); + RequestDone(); + } + if (FieldsAvailable.test(+EGroupFields::GroupId) && GetStoragePoolsResponse && GetStoragePoolsResponse->IsOk() && FieldsNeeded(FieldsBsPools)) { + std::unordered_map, const NKikimrSysView::TStoragePoolInfo*> indexStoragePool; // (box, id) -> pool + for (const NKikimrSysView::TStoragePoolEntry& entry : GetStoragePoolsResponse->Get()->Record.GetEntries()) { + const auto& key = entry.GetKey(); + const NKikimrSysView::TStoragePoolInfo& pool = entry.GetInfo(); + indexStoragePool.emplace(std::make_pair(key.GetBoxId(), key.GetStoragePoolId()), &pool); + } + ui64 rootSchemeshardId = AppData()->DomainsInfo->Domain->SchemeRoot; + for (TGroup& group : Groups) { + if (group.BoxId == 0 && group.PoolId == 0) { + group.PoolName = "static"; + group.Kind = ""; // TODO ? + group.MediaType = ""; // TODO ? + group.SchemeShardId = rootSchemeshardId; + group.PathId = 1; + group.EncryptionMode = 0; // TODO ? + } else { + auto itStoragePool = indexStoragePool.find({group.BoxId, group.PoolId}); + if (itStoragePool != indexStoragePool.end()) { + const NKikimrSysView::TStoragePoolInfo* pool = itStoragePool->second; + group.PoolName = pool->GetName(); + group.Kind = pool->GetKind(); + group.SchemeShardId = pool->GetSchemeshardId(); + group.PathId = pool->GetPathId(); + group.MediaType = GetMediaType(pool->GetPDiskFilter()); + if (!group.Erasure) { + group.Erasure = pool->GetErasureSpeciesV2(); + group.ErasureSpecies = TErasureType::ErasureSpeciesByName(group.Erasure); + } + group.EncryptionMode = pool->GetEncryptionMode(); + } else { + BLOG_W("Storage pool not found for group " << group.GroupId << " box " << group.BoxId << " pool " << group.PoolId); + } + } + } + FieldsAvailable |= FieldsBsPools; + ApplyEverything(); + CollectHiveData(); + RequestDone(); + } + if (FieldsAvailable.test(+EGroupFields::GroupId) && GetVSlotsResponse && GetVSlotsResponse->IsOk() && FieldsNeeded(FieldsBsVSlots)) { + if (GroupsByGroupId.empty()) { + RebuildGroupsByGroupId(); + } + for (const NKikimrSysView::TVSlotEntry& entry : GetVSlotsResponse->Get()->Record.GetEntries()) { + const NKikimrSysView::TVSlotKey& key = entry.GetKey(); + const NKikimrSysView::TVSlotInfo& info = entry.GetInfo(); + VSlotsByVSlotId[key] = &info; + auto itGroup = GroupsByGroupId.find(info.GetGroupId()); + if (itGroup != GroupsByGroupId.end() && itGroup->second->GroupGeneration == info.GetGroupGeneration()) { + TGroup& group = *itGroup->second; + TVDisk& vDisk = group.VDisks.emplace_back(); + FillVDiskFromVSlotInfo(vDisk, key, info); + group.VDiskNodeIds.push_back(vDisk.VSlotId.NodeId); + } + } + FieldsAvailable |= FieldsBsVSlots; + ApplyEverything(); + for (TGroup& group : Groups) { + group.CalcState(); + } + ApplyEverything(); + if (FieldsAvailable.test(+EGroupFields::PDisk) && FieldsNeeded(+EGroupFields::Available)) { + for (TGroup& group : Groups) { + group.CalcAvailable(PDisks); + } + FieldsAvailable.set(+EGroupFields::Available); + ApplyEverything(); + } + if (FieldsNeeded(FieldsWbDisks)) { + for (TGroup& group : Groups) { + for (TNodeId nodeId : group.VDiskNodeIds) { + SendWhiteboardDisksRequest(nodeId); + } + } + } + RequestDone(); + } + if (GetPDisksResponse && GetPDisksResponse->IsOk() && FieldsNeeded(FieldsBsPDisks)) { + for (const NKikimrSysView::TPDiskEntry& entry : GetPDisksResponse->Get()->Record.GetEntries()) { + const NKikimrSysView::TPDiskKey& key = entry.GetKey(); + const NKikimrSysView::TPDiskInfo& info = entry.GetInfo(); + TPDisk& pDisk = PDisks[key]; + pDisk.PDiskId = key.GetPDiskId(); + pDisk.NodeId = key.GetNodeId(); + pDisk.SetCategory(info.GetCategory()); + pDisk.Path = info.GetPath(); + pDisk.Guid = info.GetGuid(); + pDisk.AvailableSize = info.GetAvailableSize(); + pDisk.TotalSize = info.GetTotalSize(); + pDisk.Status = info.GetStatusV2(); + pDisk.StatusChangeTimestamp = TInstant::MicroSeconds(info.GetStatusChangeTimestamp()); + pDisk.EnforcedDynamicSlotSize = info.GetEnforcedDynamicSlotSize(); + pDisk.ExpectedSlotCount = info.GetExpectedSlotCount(); + pDisk.NumActiveSlots = info.GetNumActiveSlots(); + pDisk.Category = info.GetCategory(); + pDisk.DecommitStatus = info.GetDecommitStatus(); + } + FieldsAvailable |= FieldsBsPDisks; + if (FieldsAvailable.test(+EGroupFields::VDisk) && FieldsNeeded(+EGroupFields::Available)) { + for (TGroup& group : Groups) { + group.CalcAvailable(PDisks); + } + FieldsAvailable.set(+EGroupFields::Available); + ApplyEverything(); + } + RequestDone(); + } + } + + void Handle(NSysView::TEvSysView::TEvGetGroupsResponse::TPtr& ev) { + GetGroupsResponse->Set(std::move(ev)); + if (FallbackToWhiteboard) { + RequestDone(); + return; + } + ProcessBSControllerResponses(); + } + + void Handle(NSysView::TEvSysView::TEvGetStoragePoolsResponse::TPtr& ev) { + GetStoragePoolsResponse->Set(std::move(ev)); + if (FallbackToWhiteboard) { + RequestDone(); + return; + } + ProcessBSControllerResponses(); + } + + void Handle(NSysView::TEvSysView::TEvGetVSlotsResponse::TPtr& ev) { + GetVSlotsResponse->Set(std::move(ev)); + if (FallbackToWhiteboard) { + RequestDone(); + return; + } + ProcessBSControllerResponses(); + } + + void Handle(NSysView::TEvSysView::TEvGetPDisksResponse::TPtr& ev) { + GetPDisksResponse->Set(std::move(ev)); + if (FallbackToWhiteboard) { + RequestDone(); + return; + } + ProcessBSControllerResponses(); + } + + void RequestNodesList() { + if (!NodesInfo.has_value()) { + NodesInfo = MakeRequest(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); + } + } + + void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { + NodesInfo->Set(std::move(ev)); + ui32 maxAllowedNodeId = std::numeric_limits::max(); + TIntrusivePtr dynamicNameserviceConfig = AppData()->DynamicNameserviceConfig; + if (dynamicNameserviceConfig) { + maxAllowedNodeId = dynamicNameserviceConfig->MaxStaticNodeId; + } + for (const auto& ni : NodesInfo->Get()->Nodes) { + if (ni.NodeId <= maxAllowedNodeId) { + SendWhiteboardGroupRequest(ni.NodeId); + } + } + RequestDone(); + } + + void Handle(TEvWhiteboard::TEvBSGroupStateResponse::TPtr& ev) { + ui64 nodeId = ev.Get()->Cookie; + BSGroupStateResponse[nodeId].Set(std::move(ev)); + BSGroupRequestDone(); + RequestDone(); + } + + void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) { + ui64 nodeId = ev.Get()->Cookie; + auto& vDiskStateResponse = VDiskStateResponse[nodeId]; + vDiskStateResponse.Set(std::move(ev)); + for (const NKikimrWhiteboard::TVDiskStateInfo& info : vDiskStateResponse->Record.GetVDiskStateInfo()) { + for (const auto& vSlotId : info.GetDonors()) { + SendWhiteboardDisksRequest(vSlotId.GetNodeId()); + } + } + + VDiskRequestDone(); + RequestDone(); + } + + void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { + ui64 nodeId = ev.Get()->Cookie; + PDiskStateResponse[nodeId].Set(std::move(ev)); + PDiskRequestDone(); + RequestDone(); + } + + void ProcessWhiteboardGroups() { + std::unordered_map latestGroupInfo; + for (const auto& [nodeId, bsGroupStateResponse] : BSGroupStateResponse) { + if (bsGroupStateResponse.IsOk()) { + for (const NKikimrWhiteboard::TBSGroupStateInfo& info : bsGroupStateResponse->Record.GetBSGroupStateInfo()) { + TString storagePoolName = info.GetStoragePoolName(); + if (storagePoolName.empty()) { + continue; + } + if (info.VDiskNodeIdsSize() == 0) { + continue; + } + auto itLatest = latestGroupInfo.find(info.GetGroupID()); + if (itLatest == latestGroupInfo.end()) { + latestGroupInfo.emplace(info.GetGroupID(), &info); + } else { + if (info.GetGroupGeneration() > itLatest->second->GetGroupGeneration()) { + itLatest->second = &info; + } + } + } + } + } + Groups.reserve(latestGroupInfo.size()); // to keep cache stable after emplace + RebuildGroupsByGroupId(); + size_t capacity = Groups.capacity(); + for (const auto& [groupId, info] : latestGroupInfo) { + auto itGroup = GroupsByGroupId.find(groupId); + if (itGroup == GroupsByGroupId.end()) { + TGroup& group = Groups.emplace_back(); + group.GroupId = groupId; + group.GroupGeneration = info->GetGroupGeneration(); + group.Erasure = info->GetErasureSpecies(); + group.ErasureSpecies = TErasureType::ErasureSpeciesByName(group.Erasure); + group.PoolName = info->GetStoragePoolName(); + group.EncryptionMode = info->GetEncryption(); + for (auto nodeId : info->GetVDiskNodeIds()) { + group.VDiskNodeIds.push_back(nodeId); + } + if (capacity != Groups.capacity()) { + // we expect to never do this + RebuildGroupsByGroupId(); + capacity = Groups.capacity(); + } + } else { + TGroup& group = *itGroup->second; + if (group.VDiskNodeIds.empty()) { + for (auto nodeId : info->GetVDiskNodeIds()) { + group.VDiskNodeIds.push_back(nodeId); + } + } + } + } + FieldsAvailable |= FieldsWbGroups; + FoundGroups = TotalGroups = Groups.size(); + ApplyEverything(); + if (FieldsNeeded(FieldsWbDisks)) { + std::unordered_set nodeIds; + for (const TGroup& group : Groups) { + for (const TVDisk& vdisk : group.VDisks) { + TNodeId nodeId = vdisk.VSlotId.NodeId; + if (nodeIds.insert(nodeId).second) { + SendWhiteboardDisksRequest(nodeId); + } + } + } + } + } + + void FillVDiskFromVDiskStateInfo(TVDisk& vDisk, const TVSlotId& vSlotId, const NKikimrWhiteboard::TVDiskStateInfo& info) { + vDisk.VDiskId = VDiskIDFromVDiskID(info.GetVDiskId()); + vDisk.VSlotId = vSlotId; + if (!vDisk.AllocatedSize) { + vDisk.AllocatedSize = info.GetAllocatedSize(); + } + if (!vDisk.AvailableSize) { + vDisk.AvailableSize = info.GetAvailableSize(); + } + if (!vDisk.Read) { + vDisk.Read = info.GetReadThroughput(); + } + if (!vDisk.Write) { + vDisk.Write = info.GetWriteThroughput(); + } + if (!vDisk.Status) { + switch (info.GetVDiskState()) { + case NKikimrWhiteboard::EVDiskState::Initial: + case NKikimrWhiteboard::EVDiskState::SyncGuidRecovery: + vDisk.VDiskStatus = NKikimrBlobStorage::EVDiskStatus::INIT_PENDING; + break; + case NKikimrWhiteboard::EVDiskState::LocalRecoveryError: + case NKikimrWhiteboard::EVDiskState::SyncGuidRecoveryError: + case NKikimrWhiteboard::EVDiskState::PDiskError: + vDisk.VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR; + break; + case NKikimrWhiteboard::EVDiskState::OK: + vDisk.VDiskStatus = info.GetReplicated() ? NKikimrBlobStorage::EVDiskStatus::READY : NKikimrBlobStorage::EVDiskStatus::REPLICATING; + break; + } + vDisk.Status = NKikimrBlobStorage::EVDiskStatus_Name(vDisk.VDiskStatus); + } + vDisk.DiskSpace = static_cast(info.GetDiskSpace()); + vDisk.Donor = info.GetDonorMode(); + for (auto& donor : info.GetDonors()) { + vDisk.Donors.emplace_back(donor); + } + } + + void ProcessWhiteboardDisks() { + if (GroupsByGroupId.empty()) { + RebuildGroupsByGroupId(); + } + for (const auto& [nodeId, vDiskStateResponse] : VDiskStateResponse) { + if (vDiskStateResponse.IsOk()) { + for (const NKikimrWhiteboard::TVDiskStateInfo& info : vDiskStateResponse->Record.GetVDiskStateInfo()) { + TVSlotId vSlotId(nodeId, info.GetPDiskId(), info.GetVDiskSlotId()); + VDisksByVSlotId[vSlotId] = &info; + ui32 groupId = info.GetVDiskId().GetGroupID(); + ui32 groupGeneration = info.GetVDiskId().GetGroupGeneration(); + auto itGroup = GroupsByGroupId.find(groupId); + if (itGroup != GroupsByGroupId.end() && itGroup->second->GroupGeneration == groupGeneration) { + TGroup& group = *(itGroup->second); + TVDisk* vDisk = nullptr; + TVDiskID vDiskId = VDiskIDFromVDiskID(info.GetVDiskId()); + for (TVDisk& disk : group.VDisks) { + if (disk.VDiskId.SameDisk(vDiskId)) { + vDisk = &disk; + break; + } + } + if (vDisk == nullptr) { + vDisk = &(group.VDisks.emplace_back()); + vDisk->VDiskId = vDiskId; + } + FillVDiskFromVDiskStateInfo(*vDisk, vSlotId, info); + } + } + } + } + for (const auto& [nodeId, pDiskStateResponse] : PDiskStateResponse) { + if (pDiskStateResponse.IsOk()) { + for (const NKikimrWhiteboard::TPDiskStateInfo& info : pDiskStateResponse->Record.GetPDiskStateInfo()) { + PDisksByPDiskId[TPDiskId(nodeId, info.GetPDiskId())] = &info; + TPDisk& pDisk = PDisks[{nodeId, info.GetPDiskId()}]; + pDisk.PDiskId = info.GetPDiskId(); + pDisk.NodeId = nodeId; + //pDisk.Type = info.GetType(); + //pDisk.Kind = info.GetKind(); + pDisk.Path = info.GetPath(); + pDisk.Guid = info.GetGuid(); + pDisk.AvailableSize = info.GetAvailableSize(); + pDisk.TotalSize = info.GetTotalSize(); + //pDisk.Status = info.GetStatus(); + if (pDisk.EnforcedDynamicSlotSize < info.GetEnforcedDynamicSlotSize()) { + pDisk.EnforcedDynamicSlotSize = info.GetEnforcedDynamicSlotSize(); + } + if (pDisk.ExpectedSlotCount < info.GetExpectedSlotCount()) { + pDisk.ExpectedSlotCount = info.GetExpectedSlotCount(); + } + if (pDisk.NumActiveSlots < info.GetNumActiveSlots()) { + pDisk.NumActiveSlots = info.GetNumActiveSlots(); + } + pDisk.SetCategory(info.GetCategory()); + //pDisk.DecommitStatus = info.GetDecommitStatus(); + float usage = pDisk.TotalSize ? 100.0 * (pDisk.TotalSize - pDisk.AvailableSize) / pDisk.TotalSize : 0; + if (usage >= 95) { + pDisk.DiskSpace = NKikimrViewer::EFlag::Red; + } else if (usage >= 90) { + pDisk.DiskSpace = NKikimrViewer::EFlag::Orange; + } else if (usage >= 85) { + pDisk.DiskSpace = NKikimrViewer::EFlag::Yellow; + } else { + pDisk.DiskSpace = NKikimrViewer::EFlag::Green; + } + } + } + } + FieldsAvailable |= FieldsWbDisks; + for (TGroup& group : Groups) { + group.CalcReadWrite(); + } + ApplyEverything(); + if (FieldsNeeded(+EGroupFields::Available)) { + for (TGroup& group : Groups) { + group.CalcAvailable(PDisks); + } + FieldsAvailable.set(+EGroupFields::Available); + ApplyEverything(); + } + if (FieldsNeeded(FieldsGroupState)) { + for (TGroup& group : Groups) { + group.CalcState(); + } + FieldsAvailable |= FieldsGroupState; + ApplyEverything(); + } + } + + void BSGroupRequestDone() { + if (--BSGroupStateRequestsInFlight == 0) { + ProcessWhiteboardGroups(); + } + } + + void VDiskRequestDone() { + --VDiskStateRequestsInFlight; + if (VDiskStateRequestsInFlight == 0 && PDiskStateRequestsInFlight == 0) { + ProcessWhiteboardDisks(); + } + } + + void PDiskRequestDone() { + --PDiskStateRequestsInFlight; + if (VDiskStateRequestsInFlight == 0 && PDiskStateRequestsInFlight == 0) { + ProcessWhiteboardDisks(); + } + } + + void SendWhiteboardGroupRequest(ui32 nodeId) { + if (BSGroupStateResponse.count(nodeId) == 0) { + TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); + BSGroupStateResponse.emplace(nodeId, MakeRequest(whiteboardServiceId, + new TEvWhiteboard::TEvBSGroupStateRequest(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, + nodeId)); + SubscriptionNodeIds.push_back(nodeId); + ++BSGroupStateRequestsInFlight; + } + } + + void SendWhiteboardDisksRequest(ui32 nodeId) { + TActorId whiteboardServiceId = MakeNodeWhiteboardServiceId(nodeId); + if (VDiskStateResponse.count(nodeId) == 0) { + VDiskStateResponse.emplace(nodeId, MakeRequest(whiteboardServiceId, + new TEvWhiteboard::TEvVDiskStateRequest(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, + nodeId)); + ++VDiskStateRequestsInFlight; + SubscriptionNodeIds.push_back(nodeId); + } + if (PDiskStateResponse.count(nodeId) == 0) { + PDiskStateResponse.emplace(nodeId, MakeRequest(whiteboardServiceId, + new TEvWhiteboard::TEvPDiskStateRequest(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, + nodeId)); + ++PDiskStateRequestsInFlight; + SubscriptionNodeIds.push_back(nodeId); + } + } + + void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + ui32 nodeId = ev->Get()->NodeId; + { + auto itVDiskStateResponse = VDiskStateResponse.find(nodeId); + if (itVDiskStateResponse != VDiskStateResponse.end()) { + if (itVDiskStateResponse->second.Error("NodeDisconnected")) { + VDiskRequestDone(); + RequestDone(); + } + } + } + { + auto itPDiskStateResponse = PDiskStateResponse.find(nodeId); + if (itPDiskStateResponse != PDiskStateResponse.end()) { + if (itPDiskStateResponse->second.Error("NodeDisconnected")) { + PDiskRequestDone(); + RequestDone(); + } + } + } + { + auto itBSGroupStateResponse = BSGroupStateResponse.find(nodeId); + if (itBSGroupStateResponse != BSGroupStateResponse.end()) { + if (itBSGroupStateResponse->second.Error("NodeDisconnected")) { + BSGroupRequestDone(); + RequestDone(); + } + } + } + } + + void RequestWhiteboard() { + FallbackToWhiteboard = true; + RequestNodesList(); + } + + void OnBscError(const TString& error) { + if (GetGroupsResponse.has_value()) { + GetGroupsResponse->Error(error); + } + if (GetStoragePoolsResponse.has_value()) { + GetStoragePoolsResponse->Error(error); + } + if (GetVSlotsResponse.has_value()) { + GetVSlotsResponse->Error(error); + } + if (GetPDisksResponse.has_value()) { + GetPDisksResponse->Error(error); + } + RequestWhiteboard(); + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->Status != NKikimrProto::OK) { + TString error = TStringBuilder() << "Failed to establish pipe: " << NKikimrProto::EReplyStatus_Name(ev->Get()->Status); + auto it = HiveStorageStats.find(ev->Get()->TabletId); + if (it != HiveStorageStats.end()) { + it->second.Error(error); + if (--HiveStorageStatsInFlight == 0) { + FieldsAvailable |= FieldsHive; + } + } + if (ev->Get()->TabletId == GetBSControllerId()) { + OnBscError(error); + } + } + TBase::Handle(ev); // all RequestDone() are handled by base handler + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvStateStorage::TEvBoardInfo, Handle); + hFunc(NSysView::TEvSysView::TEvGetGroupsResponse, Handle); + hFunc(NSysView::TEvSysView::TEvGetStoragePoolsResponse, Handle); + hFunc(NSysView::TEvSysView::TEvGetVSlotsResponse, Handle); + hFunc(NSysView::TEvSysView::TEvGetPDisksResponse, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + hFunc(TEvHive::TEvResponseHiveStorageStats, Handle); + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvents::TEvWakeup, HandleTimeout); + hFunc(TEvInterconnect::TEvNodesInfo, Handle); + hFunc(TEvWhiteboard::TEvVDiskStateResponse, Handle); + hFunc(TEvWhiteboard::TEvPDiskStateResponse, Handle); + hFunc(TEvWhiteboard::TEvBSGroupStateResponse, Handle); + hFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected); + IgnoreFunc(TEvents::TEvUndelivered/* , Undelivered */); + } + } + + void HandleTimeout(TEvents::TEvWakeup::TPtr& ev) { + switch (ev->Get()->Tag) { + case TimeoutBSC: + OnBscError("timeout"); + break; + case TimeoutFinal: + FilterNodeIds.clear(); + break; + } + ReplyAndPassAway(); + } + + void RenderVDisk(NKikimrViewer::TStorageVDisk& jsonVDisk, const TVDisk& vdisk) { + jsonVDisk.SetVDiskId(vdisk.GetVDiskId()); + jsonVDisk.SetNodeId(vdisk.VSlotId.NodeId); + jsonVDisk.SetAllocatedSize(vdisk.AllocatedSize); + jsonVDisk.SetAvailableSize(vdisk.AvailableSize); + jsonVDisk.SetStatus(vdisk.Status); + if (vdisk.DiskSpace != NKikimrViewer::Grey) { + jsonVDisk.SetDiskSpace(vdisk.DiskSpace); + } + auto itVDiskByVSlotId = VDisksByVSlotId.find(vdisk.VSlotId); + if (itVDiskByVSlotId != VDisksByVSlotId.end()) { + auto& whiteboard = *jsonVDisk.MutableWhiteboard(); + whiteboard.CopyFrom(*(itVDiskByVSlotId->second)); + if (whiteboard.GetReplicated() || (whiteboard.GetReplicationProgress() == NAN)) { + whiteboard.ClearReplicationProgress(); + whiteboard.ClearReplicationSecondsRemaining(); + } + } + + auto itPDisk = PDisks.find(vdisk.VSlotId); + if (itPDisk != PDisks.end()) { + const TPDisk& pdisk = itPDisk->second; + NKikimrViewer::TStoragePDisk& jsonPDisk = *jsonVDisk.MutablePDisk(); + jsonPDisk.SetPDiskId(pdisk.GetPDiskId()); + jsonPDisk.SetPath(pdisk.Path); + jsonPDisk.SetType(pdisk.Type); + jsonPDisk.SetGuid(::ToString(pdisk.Guid)); + jsonPDisk.SetCategory(pdisk.Category); + jsonPDisk.SetTotalSize(pdisk.TotalSize); + jsonPDisk.SetAvailableSize(pdisk.AvailableSize); + jsonPDisk.SetStatus(pdisk.Status); + jsonPDisk.SetDecommitStatus(pdisk.DecommitStatus); + jsonPDisk.SetSlotSize(pdisk.GetSlotTotalSize()); + if (pdisk.DiskSpace != NKikimrViewer::Grey) { + jsonPDisk.SetDiskSpace(pdisk.DiskSpace); + } + auto itPDiskByPDiskId = PDisksByPDiskId.find(vdisk.VSlotId); + if (itPDiskByPDiskId != PDisksByPDiskId.end()) { + jsonPDisk.MutableWhiteboard()->CopyFrom(*(itPDiskByPDiskId->second)); + } + } + if (!vdisk.Donors.empty()) { + for (const TVSlotId& donorId : vdisk.Donors) { + NKikimrViewer::TStorageVDisk& jsonDonor = *jsonVDisk.AddDonors(); + TVDisk donor; + auto itVSlotInfo = VSlotsByVSlotId.find(donorId); + if (itVSlotInfo != VSlotsByVSlotId.end()) { + FillVDiskFromVSlotInfo(donor, donorId, *(itVSlotInfo->second)); + } + auto itVDiskInfo = VDisksByVSlotId.find(donorId); + if (itVDiskInfo != VDisksByVSlotId.end()) { + FillVDiskFromVDiskStateInfo(donor, donorId, *(itVDiskInfo->second)); + } + RenderVDisk(jsonDonor, donor); + } + } + } + + void ReplyAndPassAway() { + ApplyEverything(); + NKikimrViewer::TStorageGroupsInfo json; + json.SetVersion(1); + json.SetFieldsAvailable(FieldsAvailable.to_ulong()); + json.SetFieldsRequired(FieldsRequired.to_ulong()); + if (NeedFilter) { + json.SetNeedFilter(true); + } + if (NeedSort) { + json.SetNeedSort(true); + } + if (NeedLimit) { + json.SetNeedLimit(true); + } + json.SetTotalGroups(TotalGroups); + json.SetFoundGroups(FoundGroups); + for (const TGroup& group : Groups) { + NKikimrViewer::TStorageGroupInfo& jsonGroup = *json.AddStorageGroups(); + jsonGroup.SetGroupId(::ToString(group.GroupId)); + if (group.GroupGeneration) { + jsonGroup.SetGroupGeneration(group.GroupGeneration); + } + if (FieldsAvailable.test(+EGroupFields::PoolName)) { + jsonGroup.SetPoolName(group.PoolName); + } + for (const TVDisk& vdisk : group.VDisks) { + RenderVDisk(*jsonGroup.AddVDisks(), vdisk); + } + if (FieldsAvailable.test(+EGroupFields::Encryption)) { + jsonGroup.SetEncryption(group.EncryptionMode); + } + if (group.Overall != NKikimrViewer::Grey) { + jsonGroup.SetOverall(group.Overall); + } + if (group.DiskSpace != NKikimrViewer::Grey) { + jsonGroup.SetDiskSpace(group.DiskSpace); + } + if (FieldsAvailable.test(+EGroupFields::Kind)) { + jsonGroup.SetKind(group.Kind); + } + if (FieldsAvailable.test(+EGroupFields::MediaType)) { + jsonGroup.SetMediaType(group.MediaType); + } + if (FieldsAvailable.test(+EGroupFields::Erasure)) { + jsonGroup.SetErasureSpecies(group.Erasure); + } + if (FieldsAvailable.test(+EGroupFields::AllocationUnits)) { + jsonGroup.SetAllocationUnits(group.AllocationUnits); + } + if (FieldsAvailable.test(+EGroupFields::State)) { + jsonGroup.SetState(group.State); + } + if (FieldsAvailable.test(+EGroupFields::MissingDisks)) { + jsonGroup.SetMissingDisks(group.MissingDisks); + } + if (FieldsAvailable.test(+EGroupFields::Used)) { + jsonGroup.SetUsed(group.Used); + } + if (FieldsAvailable.test(+EGroupFields::Limit)) { + jsonGroup.SetLimit(group.Limit); + } + if (FieldsAvailable.test(+EGroupFields::Read)) { + jsonGroup.SetRead(group.Read); + } + if (FieldsAvailable.test(+EGroupFields::Write)) { + jsonGroup.SetWrite(group.Write); + } + if (FieldsAvailable.test(+EGroupFields::Usage)) { + jsonGroup.SetUsage(group.Usage); + } + if (FieldsAvailable.test(+EGroupFields::Available)) { + jsonGroup.SetAvailable(group.Available); + } + } + TStringStream out; + Proto2Json(json, out, { + .EnumMode = TProto2JsonConfig::EnumValueMode::EnumName, + .StringifyNumbers = TProto2JsonConfig::EStringifyNumbersMode::StringifyInt64Always, + .WriteNanAsString = true, + }); + TBase::ReplyAndPassAway(GetHTTPOKJSON(out.Str())); + } +}; + +template <> +YAML::Node TJsonRequestSwagger::GetSwagger() { + YAML::Node node = YAML::Load(R"___( + post: + tags: + - viewer + summary: Storage groups + description: Information about storage groups + parameters: + - name: database + in: query + description: database name + required: false + type: string + - name: pool + in: query + description: storage pool name + required: false + type: string + - name: node_id + in: query + description: node id + required: false + type: integer + - name: pdisk_id + in: query + description: pdisk id + required: false + type: integer + - name: group_id + in: query + description: group id + required: false + type: integer + - name: need_groups + in: query + description: return groups information + required: false + type: boolean + default: true + - name: need_disks + in: query + description: return disks information + required: false + type: boolean + default: true + - name: with + in: query + description: > + filter groups by missing or space: + * `missing` + * `space` + required: false + type: string + - name: sort + in: query + description: > + sort by: + * `PoolName` + * `Kind` + * `MediaType` + * `Erasure` + * `MissingDisks` + * `Usage` + * `GroupId` + * `Used` + * `Limit` + * `Usage` + * `Read` + * `Write` + required: false + type: string + - name: offset + in: query + description: skip N nodes + required: false + type: integer + - name: limit + in: query + description: limit to N nodes + required: false + type: integer + - name: timeout + in: query + description: timeout in ms + required: false + type: integer + responses: + 200: + description: OK + content: + application/json: + schema: + type: object + description: format depends on schema parameter + 400: + description: Bad Request + 403: + description: Forbidden + 504: + description: Gateway Timeout + )___"); + node["get"]["responses"]["200"]["content"]["application/json"]["schema"] = TProtoToYaml::ProtoToYamlSchema(); + return node; +} + + +void InitStorageGroupsJsonHandler(TJsonHandlers& jsonHandlers) { + jsonHandlers.AddHandler("/storage/groups", new TJsonHandler); +} + +} // namespace NViewer +} // namespace NKikimr diff --git a/ydb/core/viewer/viewer.cpp b/ydb/core/viewer/viewer.cpp index 53473f772872..90f57b2495e8 100644 --- a/ydb/core/viewer/viewer.cpp +++ b/ydb/core/viewer/viewer.cpp @@ -45,6 +45,7 @@ extern void InitPDiskJsonHandlers(TJsonHandlers& jsonHandlers); extern void InitVDiskJsonHandlers(TJsonHandlers& jsonHandlers); extern void InitOperationJsonHandlers(TJsonHandlers& jsonHandlers); extern void InitSchemeJsonHandlers(TJsonHandlers& jsonHandlers); +extern void InitStorageJsonHandlers(TJsonHandlers& jsonHandlers); void SetupPQVirtualHandlers(IViewer* viewer) { viewer->RegisterVirtualHandler( @@ -169,6 +170,13 @@ class TViewer : public TActorBootstrapped, public IViewer { .UseAuth = true, .AllowedSIDs = viewerAllowedSIDs, }); + mon->RegisterActorPage({ + .RelPath = "storage", + .ActorSystem = ctx.ExecutorThread.ActorSystem, + .ActorId = ctx.SelfID, + .UseAuth = true, + .AllowedSIDs = viewerAllowedSIDs, + }); auto whiteboardServiceId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()); ctx.Send(whiteboardServiceId, new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateAddEndpoint( "http-mon", Sprintf(":%d", KikimrRunConfig.AppConfig.GetMonitoringConfig().GetMonitoringPort()))); @@ -178,6 +186,7 @@ class TViewer : public TActorBootstrapped, public IViewer { InitViewerJsonHandlers(JsonHandlers); InitPDiskJsonHandlers(JsonHandlers); InitVDiskJsonHandlers(JsonHandlers); + InitStorageJsonHandlers(JsonHandlers); InitOperationJsonHandlers(JsonHandlers); InitSchemeJsonHandlers(JsonHandlers); @@ -560,12 +569,13 @@ class TViewer : public TActorBootstrapped, public IViewer { } auto handler = JsonHandlers.FindHandler(path); if (handler) { + auto sender(ev->Sender); try { ctx.ExecutorThread.RegisterActor(handler->CreateRequestActor(this, ev)); return; } catch (const std::exception& e) { - ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(TString("HTTP/1.1 400 Bad Request\r\n\r\n") + e.what(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); + Send(sender, new NMon::TEvHttpInfoRes(TString("HTTP/1.1 400 Bad Request\r\n\r\n") + e.what(), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); return; } } diff --git a/ydb/core/viewer/ya.make b/ydb/core/viewer/ya.make index d3b71fd5814e..f1a37de383c9 100644 --- a/ydb/core/viewer/ya.make +++ b/ydb/core/viewer/ya.make @@ -30,6 +30,7 @@ SRCS( json_handlers_operation.cpp json_handlers_pdisk.cpp json_handlers_scheme.cpp + json_handlers_storage.cpp json_handlers_vdisk.cpp json_handlers_viewer.cpp json_healthcheck.h @@ -67,6 +68,7 @@ SRCS( pdisk_info.h pdisk_status.h scheme_directory.h + storage_groups.cpp query_autocomplete_helper.h viewer_request.cpp viewer_request.h