Skip to content

fix group sorting #9892

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adameat merged 2 commits into from
Sep 30, 2024
Merged

fix group sorting #9892

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
df1c05a
groups sorting
adameat Sep 30, 2024
23f1172
one more fix to groups stats
adameat Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ydb/core/viewer/json_handlers_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
namespace NKikimr::NViewer {

void InitStorageGroupsJsonHandler(TJsonHandlers& jsonHandlers) {
jsonHandlers.AddHandler("/storage/groups", new TJsonHandler<TStorageGroups>(TStorageGroups::GetSwagger()), 5);
jsonHandlers.AddHandler("/storage/groups", new TJsonHandler<TStorageGroups>(TStorageGroups::GetSwagger()), 6);
}

void InitStorageJsonHandlers(TJsonHandlers& jsonHandlers) {
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/viewer/json_handlers_viewer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ void InitViewerHealthCheckJsonHandler(TJsonHandlers& handlers) {
}

void InitViewerNodesJsonHandler(TJsonHandlers& handlers) {
handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 7);
handlers.AddHandler("/viewer/nodes", new TJsonHandler<TJsonNodes>(TJsonNodes::GetSwagger()), 8);
}

void InitViewerACLJsonHandler(TJsonHandlers &jsonHandlers) {
Expand Down
10 changes: 10 additions & 0 deletions ydb/core/viewer/protos/viewer.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,16 @@ message TStorageGroupInfo {
message TStorageGroupGroup {
string GroupName = 1;
uint64 GroupCount = 2;
optional uint64 Used = 13;
optional uint64 Limit = 14;
optional uint64 Available = 15;
optional float Usage = 16;
optional uint64 Read = 17;
optional uint64 Write = 18;
optional float DiskSpaceUsage = 19;
optional uint64 LatencyPutTabletLog = 20;
optional uint64 LatencyPutUserData = 21;
optional uint64 LatencyGetFast = 22;
}

message TStoragePoolInfo {
Expand Down
101 changes: 95 additions & 6 deletions ydb/core/viewer/storage_groups.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ class TStorageGroups : public TViewerPipeClient {
std::optional<std::size_t> Limit;
ui32 SpaceUsageProblem = 90; // %

using TGroupSortKey = std::variant<TString, ui64, float, bool>;

struct TPDisk {
ui32 PDiskId = 0;
TNodeId NodeId = 0;
Expand Down Expand Up @@ -273,7 +275,7 @@ class TStorageGroups : public TViewerPipeClient {
ui64 Available = 0;
ui64 Read = 0;
ui64 Write = 0;
ui32 MissingDisks = 0;
ui64 MissingDisks = 0;
ui64 PutTabletLogLatency = 0;
ui64 PutUserDataLatency = 0;
ui64 GetFastLatency = 0;
Expand Down Expand Up @@ -523,14 +525,73 @@ class TStorageGroups : public TViewerPipeClient {
}
return groupName;
}

TGroupSortKey GetSortKey(EGroupFields groupBy) const {
switch (groupBy) {
case EGroupFields::GroupId:
case EGroupFields::Erasure:
case EGroupFields::PoolName:
case EGroupFields::Kind:
case EGroupFields::MediaType:
case EGroupFields::State:
return GetGroupName(groupBy);
case EGroupFields::Usage:
return Usage;
case EGroupFields::DiskSpaceUsage:
return DiskSpaceUsage;
case EGroupFields::Encryption:
return EncryptionMode;
case EGroupFields::MissingDisks:
return MissingDisks;
case EGroupFields::Latency:
return PutTabletLogLatency;
default:
return TString();
}
}
};

using TGroupData = std::vector<TGroup>;
using TGroupView = std::vector<TGroup*>;

struct TGroupGroup {
TString Name;
TGroupSortKey SortKey;
TGroupView Groups;

// stats
float Usage = 0; // avg
ui64 Used = 0; // sum
ui64 Limit = 0; // sum
ui64 Available = 0; // sum
ui64 Read = 0; // sum
ui64 Write = 0; // sum
ui64 PutTabletLogLatency = 0; // max
ui64 PutUserDataLatency = 0; // max
ui64 GetFastLatency = 0; // max
float DiskSpaceUsage = 0; // max

void CalcStats() {
for (TGroup* group : Groups) {
Usage += group->Usage;
Used += group->Used;
Limit += group->Limit;
Read += group->Read;
Write += group->Write;
if (Available) {
Available = std::min(Available, group->Available);
} else {
Available = group->Available;
}
PutTabletLogLatency = std::max(PutTabletLogLatency, group->PutTabletLogLatency);
PutUserDataLatency = std::max(PutUserDataLatency, group->PutUserDataLatency);
GetFastLatency = std::max(GetFastLatency, group->GetFastLatency);
DiskSpaceUsage = std::max(DiskSpaceUsage, group->DiskSpaceUsage);
}
if (!Groups.empty()) {
Usage /= Groups.size();
}
}
};

TGroupData GroupData;
Expand All @@ -547,8 +608,6 @@ class TStorageGroups : public TViewerPipeClient {
const TFieldsType FieldsAll = TFieldsType().set();
const TFieldsType FieldsBsGroups = TFieldsType().set(+EGroupFields::GroupId)
.set(+EGroupFields::Erasure)
.set(+EGroupFields::Used)
.set(+EGroupFields::Limit)
.set(+EGroupFields::Latency);
const TFieldsType FieldsBsPools = TFieldsType().set(+EGroupFields::PoolName)
.set(+EGroupFields::Kind)
Expand Down Expand Up @@ -655,6 +714,8 @@ class TStorageGroups : public TViewerPipeClient {
result = EGroupFields::PDiskId;
} else if (field == "Latency") {
result = EGroupFields::Latency;
} else if (field == "Available") {
result = EGroupFields::Available;
}
return result;
}
Expand Down Expand Up @@ -1013,6 +1074,7 @@ class TStorageGroups : public TViewerPipeClient {
groupGroups.emplace(gb, GroupGroups.size());
groupGroup = &GroupGroups.emplace_back();
groupGroup->Name = gb;
groupGroup->SortKey = group->GetSortKey(GroupBy);
} else {
groupGroup = &GroupGroups[it->second];
}
Expand All @@ -1031,15 +1093,15 @@ class TStorageGroups : public TViewerPipeClient {
case EGroupFields::MediaType:
case EGroupFields::State:
GroupCollection();
SortCollection(GroupGroups, [](const TGroupGroup& groupGroup) { return groupGroup.Name; });
SortCollection(GroupGroups, [](const TGroupGroup& groupGroup) { return groupGroup.SortKey; });
NeedGroup = false;
break;
case EGroupFields::Usage:
case EGroupFields::DiskSpaceUsage:
case EGroupFields::MissingDisks:
case EGroupFields::Latency:
GroupCollection();
SortCollection(GroupGroups, [](const TGroupGroup& groupGroup) { return groupGroup.Name; }, true);
SortCollection(GroupGroups, [](const TGroupGroup& groupGroup) { return groupGroup.SortKey; }, true);
NeedGroup = false;
break;
case EGroupFields::Read:
Expand Down Expand Up @@ -2072,10 +2134,37 @@ class TStorageGroups : public TViewerPipeClient {
}
}
} else {
for (const TGroupGroup& groupGroup : GroupGroups) {
for (TGroupGroup& groupGroup : GroupGroups) {
NKikimrViewer::TStorageGroupGroup& jsonGroupGroup = *json.AddStorageGroupGroups();
jsonGroupGroup.SetGroupName(groupGroup.Name);
jsonGroupGroup.SetGroupCount(groupGroup.Groups.size());
groupGroup.CalcStats();
if (FieldsAvailable.test(+EGroupFields::Used)) {
jsonGroupGroup.SetUsed(groupGroup.Used);
}
if (FieldsAvailable.test(+EGroupFields::Limit)) {
jsonGroupGroup.SetLimit(groupGroup.Limit);
}
if (FieldsAvailable.test(+EGroupFields::Read)) {
jsonGroupGroup.SetRead(groupGroup.Read);
}
if (FieldsAvailable.test(+EGroupFields::Write)) {
jsonGroupGroup.SetWrite(groupGroup.Write);
}
if (FieldsAvailable.test(+EGroupFields::Usage)) {
jsonGroupGroup.SetUsage(groupGroup.Usage);
}
if (FieldsAvailable.test(+EGroupFields::Available)) {
jsonGroupGroup.SetAvailable(groupGroup.Available);
}
if (FieldsAvailable.test(+EGroupFields::DiskSpaceUsage)) {
jsonGroupGroup.SetDiskSpaceUsage(groupGroup.DiskSpaceUsage);
}
if (FieldsAvailable.test(+EGroupFields::Latency)) {
jsonGroupGroup.SetLatencyPutTabletLog(groupGroup.PutTabletLogLatency);
jsonGroupGroup.SetLatencyPutUserData(groupGroup.PutUserDataLatency);
jsonGroupGroup.SetLatencyGetFast(groupGroup.GetFastLatency);
}
}
}
AddEvent("RenderingResult");
Expand Down
37 changes: 31 additions & 6 deletions ydb/core/viewer/viewer_nodes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ class TJsonNodes : public TViewerPipeClient {
bool OffloadMerge = true;
size_t OffloadMergeAttempts = 2;

using TGroupSortKey = std::variant<TString, ui64, float>;

struct TNode {
TEvInterconnect::TNodeInfo NodeInfo;
NKikimrWhiteboard::TSystemStateInfo SystemState;
Expand Down Expand Up @@ -470,6 +472,27 @@ class TJsonNodes : public TViewerPipeClient {
return groupName;
}

TGroupSortKey GetGroupSortKey(ENodeFields groupBy, TInstant now) const {
switch (groupBy) {
case ENodeFields::NodeId:
case ENodeFields::HostName:
case ENodeFields::NodeName:
case ENodeFields::Database:
case ENodeFields::DC:
case ENodeFields::Rack:
case ENodeFields::Version:
return GetGroupName(groupBy, now);
case ENodeFields::DiskSpaceUsage:
return DiskSpaceUsage;
case ENodeFields::Missing:
return MissingDisks;
case ENodeFields::Uptime:
return static_cast<ui64>(now.Seconds()) - (Disconnected ? SystemState.GetDisconnectTime() : SystemState.GetStartTime());
default:
return TString();
}
}

void MergeFrom(const NKikimrWhiteboard::TSystemStateInfo& systemState) {
SystemState.MergeFrom(systemState);
Cleanup();
Expand All @@ -479,11 +502,6 @@ class TJsonNodes : public TViewerPipeClient {
}
};

struct TNodeGroup {
TString Name;
std::vector<TNode*> Nodes;
};

struct TNodeBatch {
std::vector<TNode*> NodesToAskFor;
std::vector<TNode*> NodesToAskAbout;
Expand All @@ -501,6 +519,12 @@ class TJsonNodes : public TViewerPipeClient {
using TNodeData = std::vector<TNode>;
using TNodeView = std::deque<TNode*>;

struct TNodeGroup {
TString Name;
TGroupSortKey SortKey;
TNodeView Nodes;
};

TNodeData NodeData;
TNodeView NodeView;
std::vector<TNodeGroup> NodeGroups;
Expand Down Expand Up @@ -1031,6 +1055,7 @@ class TJsonNodes : public TViewerPipeClient {
nodeGroups.emplace(gb, NodeGroups.size());
nodeGroup = &NodeGroups.emplace_back();
nodeGroup->Name = gb;
nodeGroup->SortKey = node->GetGroupSortKey(GroupBy, now);
} else {
nodeGroup = &NodeGroups[it->second];
}
Expand All @@ -1052,7 +1077,7 @@ class TJsonNodes : public TViewerPipeClient {
case ENodeFields::Uptime:
case ENodeFields::Version:
GroupCollection();
SortCollection(NodeGroups, [](const TNodeGroup& nodeGroup) { return nodeGroup.Name; });
SortCollection(NodeGroups, [](const TNodeGroup& nodeGroup) { return nodeGroup.SortKey; });
NeedGroup = false;
break;
case ENodeFields::NodeInfo:
Expand Down