Make separate app data for different nodes in test actor system

ydb/core/blobstorage/ut_blobstorage/blob_depot_test_env.h

@@ -222,7 +222,7 @@ struct TBlobDepotTestEnvironment {
         {
             auto *cmd = request.AddCommand()->MutableAllocateVirtualGroup();
             cmd->SetName("vg");
-            cmd->SetHiveId(envPtr->Runtime->GetAppData()->DomainsInfo->HivesByHiveUid.begin()->second);
+            cmd->SetHiveId(envPtr->Runtime->GetDomainsInfo()->HivesByHiveUid.begin()->second);
             cmd->SetStoragePoolName(virtualPool);
             auto *prof = cmd->AddChannelProfiles();
             prof->SetStoragePoolName(envPtr->StoragePoolName);

ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.cpp

@@ -14,7 +14,7 @@ void DecommitGroup(TBlobDepotTestEnvironment& tenv, ui32 groupId) {
 
     auto *cmd = request.AddCommand()->MutableDecommitGroups();
     cmd->AddGroupIds(groupId);
-    cmd->SetHiveId(tenv.Env->Runtime->GetAppData()->DomainsInfo->HivesByHiveUid.begin()->second);
+    cmd->SetHiveId(tenv.Env->Runtime->GetDomainsInfo()->HivesByHiveUid.begin()->second);
     auto *prof = cmd->AddChannelProfiles();
     prof->SetStoragePoolName(blobDepotPool);
     prof->SetCount(2);

ydb/core/blobstorage/ut_blobstorage/lib/env.h

@@ -126,22 +126,27 @@ struct TEnvironmentSetup {
             Settings.Erasure.GetErasure() == TBlobStorageGroupType::ErasureMirror3dc ? 3 : 1;
     }
 
+    std::unique_ptr<TTestActorSystem> MakeRuntime() {
+        TFeatureFlags featureFlags;
+        featureFlags.SetSuppressCompatibilityCheck(Settings.SuppressCompatibilityCheck);
+
+        auto domainsInfo = MakeIntrusive<TDomainsInfo>();
+        auto domain = TDomainsInfo::TDomain::ConstructEmptyDomain(DomainName, DomainId);
+        domainsInfo->AddDomain(domain.Get());
+        if (Settings.SetupHive) {
+            domainsInfo->AddHive(domain->DefaultHiveUid, MakeDefaultHiveID(domain->DefaultStateStorageGroup));
+        }
+
+        return std::make_unique<TTestActorSystem>(Settings.NodeCount, NLog::PRI_ERROR, domainsInfo, featureFlags);
+    }
+
     void Initialize() {
-        Runtime = std::make_unique<TTestActorSystem>(Settings.NodeCount, NLog::PRI_ERROR);
+        Runtime = MakeRuntime();
         if (Settings.PrepareRuntime) {
             Settings.PrepareRuntime(*Runtime);
         }
         SetupLogging();
         Runtime->Start();
-        auto *appData = Runtime->GetAppData();
-
-        appData->FeatureFlags.SetSuppressCompatibilityCheck(Settings.SuppressCompatibilityCheck);
-
-        auto domain = TDomainsInfo::TDomain::ConstructEmptyDomain(DomainName, DomainId);
-        appData->DomainsInfo->AddDomain(domain.Get());
-        if (Settings.SetupHive) {
-            appData->DomainsInfo->AddHive(domain->DefaultHiveUid, MakeDefaultHiveID(domain->DefaultStateStorageGroup));
-        }
 
         if (Settings.LocationGenerator) {
             Runtime->SetupTabletRuntime(Settings.LocationGenerator, Settings.ControllerNodeId);
@@ -335,9 +340,8 @@ struct TEnvironmentSetup {
             {MakeBSControllerID(DomainId), TTabletTypes::BSController, &CreateFlatBsController},
         };
 
-        auto *appData = Runtime->GetAppData();
 
-        for (const auto& [uid, tabletId] : appData->DomainsInfo->HivesByHiveUid) {
+        for (const auto& [uid, tabletId] : Runtime->GetDomainsInfo()->HivesByHiveUid) {
             tablets.push_back(TTabletInfo{tabletId, TTabletTypes::Hive, &CreateDefaultHive});
         }
 
@@ -353,8 +357,8 @@ struct TEnvironmentSetup {
         auto localConfig = MakeIntrusive<TLocalConfig>();
 
         localConfig->TabletClassInfo[TTabletTypes::BlobDepot] = TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
-            &NBlobDepot::CreateBlobDepot, TMailboxType::ReadAsFilled, appData->SystemPoolId, TMailboxType::ReadAsFilled,
-            appData->SystemPoolId));
+            &NBlobDepot::CreateBlobDepot, TMailboxType::ReadAsFilled, Runtime->SYSTEM_POOL_ID, TMailboxType::ReadAsFilled,
+            Runtime->SYSTEM_POOL_ID));
 
         auto tenantPoolConfig = MakeIntrusive<TTenantPoolConfig>(localConfig);
         tenantPoolConfig->AddStaticSlot(DomainName);

ydb/core/blobstorage/ut_blobstorage/monitoring.cpp

@@ -91,29 +91,32 @@ void Test(const TBlobStorageGroupInfo::TTopology& topology, TInflightActor* acto
     ui64 dsproxyCost = 0;
     ui64 vdiskCost = 0;
 
-    auto* appData = env.Runtime->GetAppData();
-    Y_ABORT_UNLESS(appData);
-
     auto vdisksTotal = [&](TString subsystem, TString counter, bool derivative = false) {
         ui64 ctr = 0;
-        for (ui32 i = 0; i < groupSize; ++i) {
-            ctr += GetServiceCounters(appData->Counters, "vdisks")->
-                    GetSubgroup("storagePool", env.StoragePoolName)->
-                    GetSubgroup("group", std::to_string(groupId))->
-                    GetSubgroup("orderNumber", "0" + std::to_string(i))->
-                    GetSubgroup("pdisk", "00000" + std::to_string(pdiskIds[i]))->
-                    GetSubgroup("media", "rot")->
-                    GetSubgroup("subsystem", subsystem)->
-                    GetCounter(counter, derivative)->Val();
+        for (const auto& vslot : baseConfig.GetVSlot()) {
+            auto* appData = env.Runtime->GetNode(vslot.GetVSlotId().GetNodeId())->AppData.get();
+            for (ui32 i = 0; i < groupSize; ++i) {
+                ctr += GetServiceCounters(appData->Counters, "vdisks")->
+                        GetSubgroup("storagePool", env.StoragePoolName)->
+                        GetSubgroup("group", std::to_string(groupId))->
+                        GetSubgroup("orderNumber", "0" + std::to_string(i))->
+                        GetSubgroup("pdisk", "00000" + std::to_string(pdiskIds[i]))->
+                        GetSubgroup("media", "rot")->
+                        GetSubgroup("subsystem", subsystem)->
+                        GetCounter(counter, derivative)->Val();
+            }
         }
         return ctr;
     };
 
     auto updateCounters = [&]() {
-        dsproxyCost = GetServiceCounters(appData->Counters, "dsproxynode")->
-                GetSubgroup("subsystem", "request")->
-                GetSubgroup("storagePool", env.StoragePoolName)->
-                GetCounter("DSProxyDiskCostNs")->Val();
+        for (const auto& vslot : baseConfig.GetVSlot()) {
+            auto* appData = env.Runtime->GetNode(vslot.GetVSlotId().GetNodeId())->AppData.get();
+            dsproxyCost += GetServiceCounters(appData->Counters, "dsproxynode")->
+                    GetSubgroup("subsystem", "request")->
+                    GetSubgroup("storagePool", env.StoragePoolName)->
+                    GetCounter("DSProxyDiskCostNs")->Val();
+        }
         vdiskCost = vdisksTotal("cost", "SkeletonFrontUserCostNs");
     };
 
@@ -135,7 +138,6 @@ void Test(const TBlobStorageGroupInfo::TTopology& topology, TInflightActor* acto
 
     if constexpr(VERBOSE) {
         Cerr << str.Str() << Endl;
-        // env.Runtime->GetAppData()->Counters->OutputPlainText(Cerr);
     }
     UNIT_ASSERT_VALUES_EQUAL_C(dsproxyCost, vdiskCost, str.Str());
 }

ydb/core/blobstorage/ut_testshard/env.h

@@ -75,8 +75,16 @@ struct TEnvironmentSetup {
         Cerr << "RandomSeed# " << seed << Endl;
     }
 
+    std::unique_ptr<TTestActorSystem> MakeRuntime() {
+        auto domainsInfo = MakeIntrusive<TDomainsInfo>();
+        auto domain = TDomainsInfo::TDomain::ConstructEmptyDomain(DomainName, DomainId);
+        domainsInfo->AddDomain(domain.Get());
+        domainsInfo->AddHive(domain->DefaultHiveUid, MakeDefaultHiveID(domain->DefaultStateStorageGroup));
+        return std::make_unique<TTestActorSystem>(Settings.NodeCount, NLog::PRI_ERROR, domainsInfo);
+    }
+
     void Initialize() {
-        Runtime = std::make_unique<TTestActorSystem>(Settings.NodeCount, NLog::PRI_ERROR);
+        Runtime = MakeRuntime();
 
         Runtime->FilterFunction = [this](ui32 nodeId, std::unique_ptr<IEventHandle>& ev) {
             GroupOverseer.ExamineEvent(nodeId, *ev);
@@ -89,12 +97,6 @@ struct TEnvironmentSetup {
 
         SetupLogging();
         Runtime->Start();
-        auto *appData = Runtime->GetAppData();
-
-        auto domain = TDomainsInfo::TDomain::ConstructEmptyDomain(DomainName, DomainId);
-        appData->DomainsInfo->AddDomain(domain.Get());
-        appData->DomainsInfo->AddHive(domain->DefaultHiveUid, MakeDefaultHiveID(domain->DefaultStateStorageGroup));
-
         Runtime->SetupTabletRuntime(1, Settings.ControllerNodeId);
         SetupStaticStorage();
         for (ui32 nodeId : Runtime->GetNodes()) {
@@ -223,17 +225,15 @@ struct TEnvironmentSetup {
     }
 
     void SetupTablet(ui32 nodeId) {
-        auto *appData = Runtime->GetAppData();
-
         auto localConfig = MakeIntrusive<TLocalConfig>();
 
         localConfig->TabletClassInfo[TTabletTypes::BlobDepot] = TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
-            &NBlobDepot::CreateBlobDepot, TMailboxType::ReadAsFilled, appData->SystemPoolId, TMailboxType::ReadAsFilled,
-            appData->SystemPoolId));
+            &NBlobDepot::CreateBlobDepot, TMailboxType::ReadAsFilled, Runtime->SYSTEM_POOL_ID, TMailboxType::ReadAsFilled,
+            Runtime->SYSTEM_POOL_ID));
 
         localConfig->TabletClassInfo[TTabletTypes::TestShard] = TLocalConfig::TTabletClassInfo(new TTabletSetupInfo(
-            &NKikimr::NTestShard::CreateTestShard, TMailboxType::ReadAsFilled, appData->SystemPoolId,
-            TMailboxType::ReadAsFilled, appData->SystemPoolId));
+            &NKikimr::NTestShard::CreateTestShard, TMailboxType::ReadAsFilled, Runtime->SYSTEM_POOL_ID,
+            TMailboxType::ReadAsFilled, Runtime->SYSTEM_POOL_ID));
 
         auto tenantPoolConfig = MakeIntrusive<TTenantPoolConfig>(localConfig);
         tenantPoolConfig->AddStaticSlot(DomainName);
@@ -256,7 +256,7 @@ struct TEnvironmentSetup {
             {MakeBSControllerID(DomainId), TTabletTypes::BSController, &CreateFlatBsController},
         };
 
-        for (const auto& [uid, tabletId] : appData->DomainsInfo->HivesByHiveUid) {
+        for (const auto& [uid, tabletId] : Runtime->GetDomainsInfo()->HivesByHiveUid) {
             tablets.push_back(TTabletInfo{tabletId, TTabletTypes::Hive, &CreateDefaultHive});
         }
 
@@ -335,7 +335,7 @@ struct TEnvironmentSetup {
             auto *cmd = request.AddCommand();
             auto *vg = cmd->MutableAllocateVirtualGroup();
             vg->SetName("vg");
-            vg->SetHiveId(Runtime->GetAppData()->DomainsInfo->HivesByHiveUid.begin()->second);
+            vg->SetHiveId(Runtime->GetDomainsInfo()->HivesByHiveUid.begin()->second);
             vg->SetStoragePoolName("virtual");
             auto *prof = vg->AddChannelProfiles();
             prof->SetStoragePoolName(StoragePoolName);

ydb/core/mind/bscontroller/ut_selfheal/env.h

@@ -30,13 +30,17 @@ struct TEnvironmentSetup {
         Cleanup();
     }
 
+    std::unique_ptr<TTestActorSystem> MakeRuntime() {
+        auto domainsInfo = MakeIntrusive<TDomainsInfo>();
+        domainsInfo->AddDomain(TDomainsInfo::TDomain::ConstructEmptyDomain("dom", Domain).Release());
+        return std::make_unique<TTestActorSystem>(NodeCount, NLog::PRI_ERROR, domainsInfo);
+    }
+
     void Initialize() {
-        Runtime = std::make_unique<TTestActorSystem>(NodeCount);
+        Runtime = MakeRuntime();
         TimerActor = Runtime->Register(new TTimerActor, NodeId);
         SetupLogging();
         Runtime->Start();
-        auto *appData = Runtime->GetAppData();
-        appData->DomainsInfo->AddDomain(TDomainsInfo::TDomain::ConstructEmptyDomain("dom", Domain).Release());
         if (LocationGenerator) {
             Runtime->SetupTabletRuntime(LocationGenerator);
         } else {

ydb/core/util/testactorsys.cpp

@@ -225,8 +225,7 @@ void TTestActorSystem::SetupTabletRuntime(const std::function<TNodeLocation(ui32
 }
 
 void TTestActorSystem::SetupStateStorage(ui32 nodeId, ui32 stateStorageNodeId) {
-    auto *appData = GetAppData();
-    for (const auto& [id, domain] : appData->DomainsInfo->Domains) {
+    for (const auto& [id, domain] : GetDomainsInfo()->Domains) {
         const ui64 stateStorageGroup = domain->DefaultStateStorageGroup;
         ui32 numReplicas = 3;
 
@@ -283,4 +282,6 @@ TIntrusivePtr<IMonotonicTimeProvider> TTestActorSystem::CreateMonotonicTimeProvi
     return MakeIntrusive<TTestActorMonotonicTimeProvider>();
 }
 
+const ui32 TTestActorSystem::SYSTEM_POOL_ID = 0;
+
 }

ydb/core/util/testactorsys.h

@@ -92,6 +92,7 @@ class TTestActorSystem {
     };
 
     struct TPerNodeInfo {
+        std::unique_ptr<TAppData> AppData;
         std::unique_ptr<TActorSystem> ActorSystem;
         std::unique_ptr<TMailboxTable> MailboxTable;
         std::unique_ptr<TExecutorThread> ExecutorThread;
@@ -104,13 +105,19 @@ class TTestActorSystem {
         TMailboxHeader Header{TMailboxType::Simple};
     };
 
+    struct TAppDataInfo {
+        TIntrusivePtr<TDomainsInfo> DomainsInfo;
+        TFeatureFlags FeatureFlags;
+        TIntrusivePtr<IMonotonicTimeProvider> MonotonicTimeProvider;
+    };
+
     const ui32 MaxNodeId;
     std::map<TInstant, std::deque<TScheduleItem>> ScheduleQ;
     TInstant Clock = TInstant::Zero();
     ui64 ActorLocalId = 1;
     std::unordered_map<TMailboxId, TMailboxInfo, THash<std::tuple<ui32, ui32, ui32>>> Mailboxes;
     TProgramShouldContinue ProgramShouldContinue;
-    TAppData AppData;
+    TAppDataInfo AppDataInfo;
     TIntrusivePtr<NLog::TSettings> LoggerSettings_;
     NActors::NLog::EPrio OwnLogPriority = NActors::NLog::EPrio::Error;
     TActorId CurrentRecipient;
@@ -175,14 +182,12 @@ class TTestActorSystem {
     IOutputStream *LogStream = &Cerr;
 
 public:
-    TTestActorSystem(ui32 numNodes, NLog::EPriority defaultPrio = NLog::PRI_ERROR)
+    TTestActorSystem(ui32 numNodes, NLog::EPriority defaultPrio = NLog::PRI_ERROR, TIntrusivePtr<TDomainsInfo> domainsInfo = nullptr, TFeatureFlags featureFlags = {})
         : MaxNodeId(numNodes)
-        , AppData(0, 0, 0, 0, {{"IC", 0}}, nullptr, nullptr, nullptr, &ProgramShouldContinue)
+        , AppDataInfo({.DomainsInfo=domainsInfo, .FeatureFlags=featureFlags, .MonotonicTimeProvider=CreateMonotonicTimeProvider()})
         , LoggerSettings_(MakeIntrusive<NLog::TSettings>(TActorId(0, "logger"), NActorsServices::LOGGER, defaultPrio))
         , InterconnectMock(0, Max<ui64>(), this) // burst capacity (bytes), bytes per second
     {
-        AppData.Counters = MakeIntrusive<::NMonitoring::TDynamicCounters>();
-        AppData.DomainsInfo = MakeIntrusive<TDomainsInfo>();
         LoggerSettings_->Append(
             NActorsServices::EServiceCommon_MIN,
             NActorsServices::EServiceCommon_MAX,
@@ -199,14 +204,6 @@ class TTestActorSystem {
 
         Y_ABORT_UNLESS(!CurrentTestActorSystem);
         CurrentTestActorSystem = this;
-
-        AppData.MonotonicTimeProvider = CreateMonotonicTimeProvider();
-
-        AppData.HiveConfig.SetWarmUpBootWaitingPeriod(10);
-        AppData.HiveConfig.SetMaxNodeUsageToKick(100);
-        AppData.HiveConfig.SetMinCounterScatterToBalance(100);
-        AppData.HiveConfig.SetMinScatterToBalance(100);
-        AppData.HiveConfig.SetObjectImbalanceToBalance(100);
     }
 
     ~TTestActorSystem() {
@@ -217,8 +214,27 @@ class TTestActorSystem {
     static TIntrusivePtr<ITimeProvider> CreateTimeProvider();
     static TIntrusivePtr<IMonotonicTimeProvider> CreateMonotonicTimeProvider();
 
-    TAppData *GetAppData() {
-        return &AppData;
+    const static ui32 SYSTEM_POOL_ID;
+
+    const TIntrusivePtr<TDomainsInfo> GetDomainsInfo() const {
+        return AppDataInfo.DomainsInfo;
+    }
+
+    std::unique_ptr<TAppData> MakeAppData() {
+        auto appData = std::make_unique<TAppData>(SYSTEM_POOL_ID, 0, 0, 0, TMap<TString, ui32>{{"IC", 0}}, nullptr, nullptr, nullptr, &ProgramShouldContinue);
+        appData->Counters = MakeIntrusive<::NMonitoring::TDynamicCounters>();
+
+        appData->DomainsInfo = AppDataInfo.DomainsInfo;
+        appData->MonotonicTimeProvider = AppDataInfo.MonotonicTimeProvider;
+        appData->FeatureFlags = AppDataInfo.FeatureFlags;
+
+        appData->HiveConfig.SetWarmUpBootWaitingPeriod(10);
+        appData->HiveConfig.SetMaxNodeUsageToKick(100);
+        appData->HiveConfig.SetMinCounterScatterToBalance(100);
+        appData->HiveConfig.SetMinScatterToBalance(100);
+        appData->HiveConfig.SetObjectImbalanceToBalance(100);
+
+        return appData;
     }
 
     template<typename T>
@@ -261,7 +277,8 @@ class TTestActorSystem {
             }
         }
 
-        info.ActorSystem = std::make_unique<TActorSystem>(setup, &AppData, LoggerSettings_);
+        info.AppData = std::move(MakeAppData());
+        info.ActorSystem = std::make_unique<TActorSystem>(setup, info.AppData.get(), LoggerSettings_);
         info.MailboxTable = std::make_unique<TMailboxTable>();
         info.ExecutorThread = std::make_unique<TExecutorThread>(0, 0, info.ActorSystem.get(), pool,
             info.MailboxTable.get(), "TestExecutor");
@@ -715,6 +732,12 @@ class TTestActorSystem {
         }
     }
 
+    TPerNodeInfo *GetNode(ui32 nodeId) {
+        const auto nodeIt = PerNodeInfo.find(nodeId);
+        Y_ABORT_UNLESS(nodeIt != PerNodeInfo.end());
+        return &nodeIt->second;
+    }
+
     TInstant GetClock() const { return Clock; }
     ui64 GetEventsProcessed() const { return EventsProcessed; }
 
@@ -752,12 +775,6 @@ class TTestActorSystem {
         Y_ABORT_UNLESS(!recip || (recip.NodeId() == nodeId && !recip.IsService()));
         return recip;
     }
-
-    TPerNodeInfo *GetNode(ui32 nodeId) {
-        const auto nodeIt = PerNodeInfo.find(nodeId);
-        Y_ABORT_UNLESS(nodeIt != PerNodeInfo.end());
-        return &nodeIt->second;
-    }
 };
 
 class TFakeSchedulerCookie : public ISchedulerCookie {