From c9bb42e442b08a95a46414dac82a7649ee53910c Mon Sep 17 00:00:00 2001 From: Andrei Rykov Date: Thu, 21 Dec 2023 18:33:43 +0000 Subject: [PATCH 1/3] YDB-2101 storage handler added check usage when out-of-space filter --- ydb/core/viewer/json_storage.h | 134 ++++++++++++++++------------ ydb/core/viewer/json_storage_base.h | 30 ------- ydb/core/viewer/viewer_ut.cpp | 84 +++++++++++++++++ 3 files changed, 160 insertions(+), 88 deletions(-) diff --git a/ydb/core/viewer/json_storage.h b/ydb/core/viewer/json_storage.h index a64fcf8db414..ce5cdb8bbf93 100644 --- a/ydb/core/viewer/json_storage.h +++ b/ydb/core/viewer/json_storage.h @@ -54,13 +54,13 @@ class TJsonStorage : public TJsonStorageBase { uint64 Read; uint64 Write; - TGroupRow() + TGroupRow() : Degraded(0) , Usage(0) , Used(0) , Limit(0) , Read(0) - , Write(0) + , Write(0) {} }; THashMap GroupRowsByGroupId; @@ -245,6 +245,36 @@ class TJsonStorage : public TJsonStorageBase { } } + bool CheckGroupFilters(const TString& groupId, const TString& poolName, const TGroupRow& groupRow) { + if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) { + return false; + } + switch (With) { + case EWith::MissingDisks: + if (BSGroupWithMissingDisks.count(groupId) == 0) { + return false; + } + break; + case EWith::SpaceProblems: + if (BSGroupWithSpaceProblems.count(groupId) == 0 && groupRow.Usage < 0.8) { + return false; + } + break; + case EWith::Everything: + break; + } + if (Filter) { + if (poolName.Contains(Filter)) { + return true; + } + if (groupId.Contains(Filter)) { + return true; + } + return false; + } + return true; + } + void ReplyAndPassAway() override { if (CheckAdditionalNodesInfoNeeded()) { return; @@ -270,19 +300,62 @@ class TJsonStorage : public TJsonStorageBase { } ui64 foundGroups = 0; ui64 totalGroups = 0; + TVector GroupRows; for (const auto& [poolName, poolInfo] : StoragePoolInfo) { if ((!FilterTenant.empty() || !FilterStoragePools.empty()) && FilterStoragePools.count(poolName) == 0) { continue; } NKikimrViewer::TStoragePoolInfo* pool = StorageInfo.AddStoragePools(); for (TString groupId : poolInfo.Groups) { + TGroupRow row; + row.PoolName = poolName; + row.GroupId = groupId; + row.Kind = poolInfo.Kind; + auto ib = BSGroupIndex.find(groupId); + if (ib != BSGroupIndex.end()) { + row.Erasure = ib->second.GetErasureSpecies(); + const auto& vDiskIds = ib->second.GetVDiskIds(); + for (auto iv = vDiskIds.begin(); iv != vDiskIds.end(); ++iv) { + const NKikimrBlobStorage::TVDiskID& vDiskId = *iv; + auto ie = VDisksIndex.find(vDiskId); + bool degraded = false; + if (ie != VDisksIndex.end()) { + ui32 nodeId = ie->second.GetNodeId(); + ui32 pDiskId = ie->second.GetPDiskId(); + degraded |= !ie->second.GetReplicated() || ie->second.GetVDiskState() != NKikimrWhiteboard::EVDiskState::OK; + row.Used += ie->second.GetAllocatedSize(); + row.Limit += ie->second.GetAllocatedSize() + ie->second.GetAvailableSize(); + row.Read += ie->second.GetReadThroughput(); + row.Write += ie->second.GetWriteThroughput(); + + auto ip = PDisksIndex.find(std::make_pair(nodeId, pDiskId)); + if (ip != PDisksIndex.end()) { + degraded |= ip->second.GetState() != NKikimrBlobStorage::TPDiskState::Normal; + if (!ie->second.HasAvailableSize()) { + row.Limit += ip->second.GetAvailableSize(); + } + } + } + if (degraded) { + row.Degraded++; + } + } + } + row.Usage = row.Limit == 0 ? 100 : (float)row.Used / row.Limit; + ++totalGroups; - if (!CheckGroupFilters(groupId, poolName)) { + if (!CheckGroupFilters(groupId, poolName, row)) { continue; } ++foundGroups; if (Version == EVersion::v1) { pool->AddGroups()->SetGroupId(groupId); + } else if (Version == EVersion::v2) { + if (!UsageBuckets.empty() && !BinarySearch(UsageBuckets.begin(), UsageBuckets.end(), (ui32)(row.Usage * 100) / UsagePace)) { + continue; + } + GroupRows.emplace_back(row); + GroupRowsByGroupId[groupId] = row; } auto itHiveGroup = BSGroupHiveIndex.find(groupId); if (itHiveGroup != BSGroupHiveIndex.end()) { @@ -309,61 +382,6 @@ class TJsonStorage : public TJsonStorageBase { } if (Version == EVersion::v2) { - TVector GroupRows; - for (const auto& [poolName, poolInfo] : StoragePoolInfo) { - if ((!FilterTenant.empty() || !FilterStoragePools.empty()) && FilterStoragePools.count(poolName) == 0) { - continue; - } - for (TString groupId : poolInfo.Groups) { - if (!CheckGroupFilters(groupId, poolName)) { - continue; - } - - TGroupRow row; - row.PoolName = poolName; - row.GroupId = groupId; - row.Kind = poolInfo.Kind; - - auto ib = BSGroupIndex.find(groupId); - if (ib != BSGroupIndex.end()) { - row.Erasure = ib->second.GetErasureSpecies(); - const auto& vDiskIds = ib->second.GetVDiskIds(); - for (auto iv = vDiskIds.begin(); iv != vDiskIds.end(); ++iv) { - const NKikimrBlobStorage::TVDiskID& vDiskId = *iv; - auto ie = VDisksIndex.find(vDiskId); - bool degraded = false; - if (ie != VDisksIndex.end()) { - ui32 nodeId = ie->second.GetNodeId(); - ui32 pDiskId = ie->second.GetPDiskId(); - degraded |= !ie->second.GetReplicated() || ie->second.GetVDiskState() != NKikimrWhiteboard::EVDiskState::OK; - row.Used += ie->second.GetAllocatedSize(); - row.Limit += ie->second.GetAllocatedSize() + ie->second.GetAvailableSize(); - row.Read += ie->second.GetReadThroughput(); - row.Write += ie->second.GetWriteThroughput(); - - auto ip = PDisksIndex.find(std::make_pair(nodeId, pDiskId)); - if (ip != PDisksIndex.end()) { - degraded |= ip->second.GetState() != NKikimrBlobStorage::TPDiskState::Normal; - if (!ie->second.HasAvailableSize()) { - row.Limit += ip->second.GetAvailableSize(); - } - } - } - if (degraded) { - row.Degraded++; - } - } - } - - row.Usage = row.Limit == 0 ? 100 : (float)row.Used / row.Limit; - if (!UsageBuckets.empty() && !BinarySearch(UsageBuckets.begin(), UsageBuckets.end(), (ui32)(row.Usage * 100) / UsagePace)) { - continue; - } - GroupRows.emplace_back(row); - GroupRowsByGroupId[groupId] = row; - } - } - switch (GroupSort) { case EGroupSort::PoolName: SortCollection(GroupRows, [](const TGroupRow& node) { return node.PoolName;}, ReverseSort); diff --git a/ydb/core/viewer/json_storage_base.h b/ydb/core/viewer/json_storage_base.h index 47d97678fb9d..26046561476f 100644 --- a/ydb/core/viewer/json_storage_base.h +++ b/ydb/core/viewer/json_storage_base.h @@ -407,36 +407,6 @@ class TJsonStorageBase : public TViewerPipeClient { TList PDisksAppended; TList VDisksAppended; - bool CheckGroupFilters(const TString& groupId, const TString& poolName) { - if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) { - return false; - } - switch (With) { - case EWith::MissingDisks: - if (BSGroupWithMissingDisks.count(groupId) == 0) { - return false; - } - break; - case EWith::SpaceProblems: - if (BSGroupWithSpaceProblems.count(groupId) == 0) { - return false; - } - break; - case EWith::Everything: - break; - } - if (Filter) { - if (poolName.Contains(Filter)) { - return true; - } - if (groupId.Contains(Filter)) { - return true; - } - return false; - } - return true; - } - bool CheckAdditionalNodesInfoNeeded() { if (NeedAdditionalNodesRequests) { NeedAdditionalNodesRequests = false; diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index f88f1a6f0d16..40c989d9f65a 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -291,6 +292,19 @@ Y_UNIT_TEST_SUITE(Viewer) { nodeId++; } + void ChangeVDiskStateResponse(TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev, NKikimrWhiteboard::EFlag diskSpace, ui64 used, ui64 limit) { + auto& pbRecord = (*ev)->Get()->Record; + auto state = pbRecord.add_vdiskstateinfo(); + state->mutable_vdiskid()->set_vdisk(0); + state->mutable_vdiskid()->set_groupid(0); + state->mutable_vdiskid()->set_groupgeneration(1); + state->set_diskspace(diskSpace); + state->set_vdiskstate(NKikimrWhiteboard::EVDiskState::OK); + state->set_nodeid(0); + state->set_allocatedsize(used); + state->set_availablesize(limit - used); + } + void ChangeDescribeSchemeResult(TEvSchemeShard::TEvDescribeSchemeResult::TPtr* ev, int tabletsTotal) { auto record = (*ev)->Get()->MutableRecord(); auto params = record->mutable_pathdescription()->mutable_domaindescription()->mutable_processingparams(); @@ -516,4 +530,74 @@ Y_UNIT_TEST_SUITE(Viewer) { { QueryTest("select \"Hello\"", false, "Hello"); } + + void StorageSpaceTest(const TString& withValue, const NKikimrWhiteboard::EFlag diskSpace, const ui64 used, const ui64 limit, const bool isExpectingGroup) { + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + auto settings = TServerSettings(port); + settings.InitKikimrRunConfig() + .SetNodeCount(1) + .SetUseRealThreads(false) + .SetDomainName("Root"); + TServer server(settings); + server.EnableGRpc(grpcPort); + TClient client(settings); + TTestActorRuntime& runtime = *server.GetRuntime(); + + TActorId sender = runtime.AllocateEdgeActor(); + TAutoPtr handle; + + THttpRequest httpReq(HTTP_METHOD_GET); + httpReq.CgiParameters.emplace("with", withValue); + httpReq.CgiParameters.emplace("version", "v2"); + auto page = MakeHolder("viewer", "title"); + TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/storage", nullptr); + auto request = MakeHolder(monReq); + + auto observerFunc = [&](TAutoPtr& ev) { + Y_UNUSED(ev); + if (ev->GetTypeRewrite() == TEvWhiteboard::EvVDiskStateResponse) { + auto *x = reinterpret_cast(&ev); + ChangeVDiskStateResponse(x, diskSpace, used, limit); + } + + return TTestActorRuntime::EEventAction::PROCESS; + }; + runtime.SetObserverFunc(observerFunc); + + runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0)); + NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent(handle); + + size_t pos = result->Answer.find('{'); + TString jsonResult = result->Answer.substr(pos); + Ctest << "json result: " << jsonResult << Endl; + NJson::TJsonValue json; + try { + NJson::ReadJsonTree(jsonResult, &json, true); + } + catch (yexception ex) { + Ctest << ex.what() << Endl; + } + UNIT_ASSERT_VALUES_EQUAL(json.GetMap().contains("StorageGroups"), isExpectingGroup); + } + + Y_UNIT_TEST(StorageGroupOutputWithoutFilterNoDepends) + { + StorageSpaceTest("all", NKikimrWhiteboard::EFlag::Green, 10, 100, true); + StorageSpaceTest("all", NKikimrWhiteboard::EFlag::Red, 90, 100, true); + } + + Y_UNIT_TEST(StorageGroupOutputWithSpaceCheckDependsOnVDiskSpaceStatus) + { + StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 10, 100, false); + StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Red, 10, 100, true); + } + + Y_UNIT_TEST(StorageGroupOutputWithSpaceCheckDependsOnUsage) + { + StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 70, 100, false); + StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 80, 100, true); + StorageSpaceTest("space", NKikimrWhiteboard::EFlag::Green, 90, 100, true); + } } From 389cd69e89aba31d6eed16d73628378cfa70be20 Mon Sep 17 00:00:00 2001 From: Andrei Rykov Date: Fri, 22 Dec 2023 16:29:52 +0000 Subject: [PATCH 2/3] remove base64 tests --- ydb/core/viewer/viewer_ut.cpp | 65 ----------------------------------- 1 file changed, 65 deletions(-) diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index 40c989d9f65a..279977c1b6cf 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -466,71 +466,6 @@ Y_UNIT_TEST_SUITE(Viewer) { #endif } - NJson::TJsonValue SendQuery(const TString& query, const TString& schema, const bool base64) { - TPortManager tp; - ui16 port = tp.GetPort(2134); - ui16 grpcPort = tp.GetPort(2135); - auto settings = TServerSettings(port); - settings.InitKikimrRunConfig() - .SetNodeCount(1) - .SetUseRealThreads(false) - .SetDomainName("Root"); - TServer server(settings); - server.EnableGRpc(grpcPort); - TClient client(settings); - TTestActorRuntime& runtime = *server.GetRuntime(); - - TActorId sender = runtime.AllocateEdgeActor(); - TAutoPtr handle; - - THttpRequest httpReq(HTTP_METHOD_GET); - httpReq.CgiParameters.emplace("schema", schema); - httpReq.CgiParameters.emplace("base64", base64 ? "true" : "false"); - httpReq.CgiParameters.emplace("query", query); - auto page = MakeHolder("viewer", "title"); - TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/query", nullptr); - auto request = MakeHolder(monReq); - - runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0)); - NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent(handle); - - size_t pos = result->Answer.find('{'); - TString jsonResult = result->Answer.substr(pos); - Ctest << "json result: " << jsonResult << Endl; - NJson::TJsonValue json; - try { - NJson::ReadJsonTree(jsonResult, &json, true); - } - catch (yexception ex) { - Ctest << ex.what() << Endl; - } - return json; - } - - void QueryTest(const TString& query, const bool base64, const TString& reply) { - NJson::TJsonValue result = SendQuery(query, "classic", base64); - UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("column0").GetString(), reply); - - result = SendQuery(query, "ydb", base64); - UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("result").GetArray()[0].GetMap().at("column0").GetString(), reply); - - result = SendQuery(query, "modern", base64); - UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("result").GetArray()[0].GetArray()[0].GetString(), reply); - - result = SendQuery(query, "multi", base64); - UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("result").GetArray()[0].GetMap().at("rows").GetArray()[0].GetArray()[0].GetString(), reply); - } - - Y_UNIT_TEST(SelectStringWithBase64Encoding) - { - QueryTest("select \"Hello\"", true, "SGVsbG8="); - } - - Y_UNIT_TEST(SelectStringWithNoBase64Encoding) - { - QueryTest("select \"Hello\"", false, "Hello"); - } - void StorageSpaceTest(const TString& withValue, const NKikimrWhiteboard::EFlag diskSpace, const ui64 used, const ui64 limit, const bool isExpectingGroup) { TPortManager tp; ui16 port = tp.GetPort(2134); From 0242179d97f93608ecf0de7dc391f18f8d2b8523 Mon Sep 17 00:00:00 2001 From: Andrei Rykov Date: Fri, 29 Dec 2023 13:48:58 +0000 Subject: [PATCH 3/3] added SendQuery --- ydb/core/viewer/viewer_ut.cpp | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index 279977c1b6cf..40c989d9f65a 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -466,6 +466,71 @@ Y_UNIT_TEST_SUITE(Viewer) { #endif } + NJson::TJsonValue SendQuery(const TString& query, const TString& schema, const bool base64) { + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + auto settings = TServerSettings(port); + settings.InitKikimrRunConfig() + .SetNodeCount(1) + .SetUseRealThreads(false) + .SetDomainName("Root"); + TServer server(settings); + server.EnableGRpc(grpcPort); + TClient client(settings); + TTestActorRuntime& runtime = *server.GetRuntime(); + + TActorId sender = runtime.AllocateEdgeActor(); + TAutoPtr handle; + + THttpRequest httpReq(HTTP_METHOD_GET); + httpReq.CgiParameters.emplace("schema", schema); + httpReq.CgiParameters.emplace("base64", base64 ? "true" : "false"); + httpReq.CgiParameters.emplace("query", query); + auto page = MakeHolder("viewer", "title"); + TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/query", nullptr); + auto request = MakeHolder(monReq); + + runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0)); + NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent(handle); + + size_t pos = result->Answer.find('{'); + TString jsonResult = result->Answer.substr(pos); + Ctest << "json result: " << jsonResult << Endl; + NJson::TJsonValue json; + try { + NJson::ReadJsonTree(jsonResult, &json, true); + } + catch (yexception ex) { + Ctest << ex.what() << Endl; + } + return json; + } + + void QueryTest(const TString& query, const bool base64, const TString& reply) { + NJson::TJsonValue result = SendQuery(query, "classic", base64); + UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("column0").GetString(), reply); + + result = SendQuery(query, "ydb", base64); + UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("result").GetArray()[0].GetMap().at("column0").GetString(), reply); + + result = SendQuery(query, "modern", base64); + UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("result").GetArray()[0].GetArray()[0].GetString(), reply); + + result = SendQuery(query, "multi", base64); + UNIT_ASSERT_VALUES_EQUAL(result.GetMap().at("result").GetArray()[0].GetMap().at("rows").GetArray()[0].GetArray()[0].GetString(), reply); + } + + Y_UNIT_TEST(SelectStringWithBase64Encoding) + { + QueryTest("select \"Hello\"", true, "SGVsbG8="); + } + + Y_UNIT_TEST(SelectStringWithNoBase64Encoding) + { + QueryTest("select \"Hello\"", false, "Hello"); + } + void StorageSpaceTest(const TString& withValue, const NKikimrWhiteboard::EFlag diskSpace, const ui64 used, const ui64 limit, const bool isExpectingGroup) { TPortManager tp; ui16 port = tp.GetPort(2134);