Apply patch between 24.1.18 and 24.1.18-stream-nb-1 tags - Nebius specific

.github/CODEOWNERS

@@ -1 +1 @@
-* @ydb-platform/ReleaseApprovers
+* @ydb-platform/ReleaseApprovers @mvgorbunov @dcherednik @va-kuznecov

ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h

@@ -302,7 +302,7 @@ using TColor = NKikimrBlobStorage::TPDiskSpaceColor;
         }
 
         SharedQuota->SetName("SharedQuota");
-        TColorLimits chunkLimits = TColorLimits::MakeChunkLimits();
+        TColorLimits chunkLimits = TColorLimits::MakeChunkLimits(params.ChunkBaseLimit);
         SharedQuota->ForceHardLimit(GlobalQuota->GetHardLimit(OwnerBeginUser), chunkLimits);
         OwnerQuota->Reset(GlobalQuota->GetHardLimit(OwnerBeginUser), chunkLimits);
         OwnerQuota->SetExpectedOwnerCount(params.ExpectedOwnerCount);

ydb/core/blobstorage/pdisk/blobstorage_pdisk_color_limits.h

@@ -51,16 +51,26 @@ struct TColorLimits {
         str << "  Cyan = Total * " << Cyan.ToString() << "\n";
     }
 
-    static TColorLimits MakeChunkLimits() {
+    static TColorLimits MakeChunkLimits(i64 cyan) {
+        cyan = Min<i64>(130, cyan);
+        cyan = Max<i64>(13, cyan);
+
+        i64 lightYellow = cyan / 130.0 * 100;
+        i64 yellow = cyan / 130.0 * 80;
+        i64 lightOrange = cyan / 130.0 * 65;
+        i64 preOrange = cyan / 130.0 * 50;
+        i64 orange = cyan / 130.0 * 30;
+        i64 red = cyan / 130.0 * 10;
+
         return {
             {1,   1000, 2}, // Black: Leave bare minimum for disaster recovery
-            {10,  1000, 3}, // Red
-            {30,  1000, 4}, // Orange
-            {50,  1000, 4}, // PreOrange
-            {65,  1000, 5}, // LightOrange
-            {80,  1000, 6}, // Yellow: Stop serving user writes at 8% free space
-            {100, 1000, 7}, // LightYellow: Ask tablets to move to another group at 10% free space
-            {130, 1000, 8}, // Cyan: 13% free space or less
+            {red,  1000, 3}, // Red
+            {orange,  1000, 4}, // Orange
+            {preOrange,  1000, 4}, // PreOrange
+            {lightOrange,  1000, 5}, // LightOrange
+            {yellow,  1000, 6}, // Yellow: Stop serving user writes at 8% free space
+            {lightYellow, 1000, 7}, // LightYellow: Ask tablets to move to another group at 10% free space
+            {cyan, 1000, 8}, // Cyan: 13% free space or less
         };
     }
 
@@ -107,8 +117,25 @@ struct TColorLimits {
                 Y_ABORT();
         }
     }
+
+    i64 GetQuotaForColor(NKikimrBlobStorage::TPDiskSpaceColor::E color, i64 total) {
+        switch (color) {
+            case NKikimrBlobStorage::TPDiskSpaceColor::CYAN:           return Cyan.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_YELLOW:   return LightYellow.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::YELLOW:         return Yellow.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_ORANGE:   return LightOrange.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::PRE_ORANGE:     return PreOrange.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::ORANGE:         return Orange.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::RED:            return Red.CalculateQuota(total);
+            case NKikimrBlobStorage::TPDiskSpaceColor::BLACK:          return Black.CalculateQuota(total);
+
+            case NKikimrBlobStorage::TPDiskSpaceColor_E_TPDiskSpaceColor_E_INT_MIN_SENTINEL_DO_NOT_USE_:
+            case NKikimrBlobStorage::TPDiskSpaceColor_E_TPDiskSpaceColor_E_INT_MAX_SENTINEL_DO_NOT_USE_:
+            default:
+                Y_ABORT();
+        }
+    }
 };
 
 } // NPDisk
 } // NKikimr
-

ydb/core/blobstorage/pdisk/blobstorage_pdisk_config.h

@@ -132,6 +132,9 @@ struct TPDiskConfig : public TThrRefBase {
 
     ui64 ExpectedSlotCount = 0;
 
+    // Free chunk permille that triggers Cyan color (e.g. 100 is 10%). Between 130 (default) and 13.
+    ui32 ChunkBaseLimit = 130;
+
     NKikimrConfig::TFeatureFlags FeatureFlags;
 
     ui64 MinLogChunksTotal = 4ull; // for tiny disks
@@ -375,6 +378,10 @@ struct TPDiskConfig : public TThrRefBase {
         if (cfg->HasExpectedSlotCount()) {
             ExpectedSlotCount = cfg->GetExpectedSlotCount();
         }
+
+        if (cfg->HasChunkBaseLimit()) {
+            ChunkBaseLimit = cfg->GetChunkBaseLimit();
+        }
     }
 };
 

ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp

@@ -1449,6 +1449,7 @@ void TPDisk::ProcessReadLogResult(const NPDisk::TEvReadLogResult &evReadLogResul
                 }
                 params.CommonLogSize = LogChunks.size();
                 params.SpaceColorBorder = Cfg->SpaceColorBorder;
+                params.ChunkBaseLimit = Cfg->ChunkBaseLimit;
                 for (ui32 ownerId = OwnerBeginUser; ownerId < OwnerEndUser; ++ownerId) {
                     if (OwnerData[ownerId].VDiskId != TVDiskID::InvalidId) {
                         params.OwnersInfo[ownerId] = {usedForOwner[ownerId], OwnerData[ownerId].VDiskId};

ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper_params.h

@@ -40,6 +40,9 @@ struct TKeeperParams {
 
     // Small disk
     bool SeparateCommonLog = true;
+
+    // Free chunk permille that triggers Cyan color (e.g. 100 is 10%). Between 130 (default) and 13.
+    ui32 ChunkBaseLimit = 130;
 };
 
 } // NPDisk

ydb/core/blobstorage/pdisk/blobstorage_pdisk_quota_record.h

@@ -78,7 +78,7 @@ class TQuotaRecord {
 #undef PRINT_DISK_SPACE_COLOR
     }
 
-    // Called only from the main trhead
+    // Called only from the main thread
     // Returns number of chunks released (negative for chunks acquired)
     i64 ForceHardLimit(i64 hardLimit, const TColorLimits &limits) {
         i64 oldHardLimit = AtomicGet(HardLimit);

ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_color_limits.cpp

@@ -0,0 +1,142 @@
+#include "blobstorage_pdisk_abstract.h"
+#include "blobstorage_pdisk_impl.h"
+
+#include "blobstorage_pdisk_ut.h"
+#include "blobstorage_pdisk_ut_actions.h"
+#include "blobstorage_pdisk_ut_helpers.h"
+#include "blobstorage_pdisk_ut_run.h"
+#include "blobstorage_pdisk_color_limits.h"
+
+#include <ydb/core/blobstorage/crypto/default.h>
+
+#include <ydb/core/testlib/actors/test_runtime.h>
+
+namespace NKikimr {
+
+Y_UNIT_TEST_SUITE(TColorLimitsTest) {
+    // Define color codes
+    enum class TDiskColor {
+        Black,
+        Red,
+        Orange,
+        PreOrange,
+        LightOrange,
+        Yellow,
+        LightYellow,
+        Cyan,
+        Default
+    };
+
+    // Function to set text color based on TDiskColor enum
+    void SetColor(NKikimrBlobStorage::TPDiskSpaceColor_E color) {
+        switch (color) {
+            case NKikimrBlobStorage::TPDiskSpaceColor::BLACK:
+                Cout << "\033[30m"; // Black
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::RED:
+                Cout << "\033[31m"; // Red
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::GREEN:
+                Cout << "\033[32m"; // Green
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::ORANGE:
+                Cout << "\033[33m"; // Orange
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::PRE_ORANGE:
+                Cout << "\033[38;5;208m"; // PreOrange (closest approximation)
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_ORANGE:
+                Cout << "\033[38;5;215m"; // LightOrange (closest approximation)
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::YELLOW:
+                Cout << "\033[93m"; // Yellow
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_YELLOW:
+                Cout << "\033[38;5;229m"; // LightYellow (closest approximation)
+                break;
+            case NKikimrBlobStorage::TPDiskSpaceColor::CYAN:
+                Cout << "\033[36m"; // Cyan
+                break;
+            default:
+                // Default color (reset)
+                break;
+        }
+    }
+
+    void ClearColor() {
+        Cout << "\033[0m";
+    }
+
+    Y_UNIT_TEST(Colors) {
+        NKikimrBlobStorage::TPDiskSpaceColor_E colors[] = {
+            NKikimrBlobStorage::TPDiskSpaceColor::BLACK,
+            NKikimrBlobStorage::TPDiskSpaceColor::RED,
+            NKikimrBlobStorage::TPDiskSpaceColor::ORANGE,
+            NKikimrBlobStorage::TPDiskSpaceColor::PRE_ORANGE,
+            NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_ORANGE,
+            NKikimrBlobStorage::TPDiskSpaceColor::YELLOW,
+            NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_YELLOW,
+            NKikimrBlobStorage::TPDiskSpaceColor::CYAN
+        };
+
+        auto printLimitsFn = [&colors](int percent) {
+            NPDisk::TColorLimits limits = NPDisk::TColorLimits::MakeChunkLimits(percent);
+
+            Cout << "Print for " << (percent / 10.0) << "%" << Endl;
+
+            i64 chunks = 1000;
+
+            i64 cur = 0;
+            i64 all = 0;
+
+            std::map<NKikimrBlobStorage::TPDiskSpaceColor_E, i64> sizeByColor;
+
+            for (auto color : colors) {
+                i64 curChunks = limits.GetQuotaForColor(color, chunks);
+
+                SetColor(color);
+
+                i64 sz = curChunks - all;
+
+                sizeByColor[color] = sz;
+
+                for (i64 i = 0; i < sz; i++) {
+                    Cout << "#";
+
+                    if ((++cur % 100) == 0) {
+                        cur = 0;
+                        Cout << Endl;
+                    }
+                }
+
+                all = curChunks;
+            }
+
+            SetColor(NKikimrBlobStorage::TPDiskSpaceColor::GREEN);
+            for (i64 i = 0; i < (chunks - all); i++) {
+                Cout << "#";
+
+                if ((++cur % 100) == 0) {
+                    cur = 0;
+                    Cout << Endl;
+                }
+            }
+
+            ClearColor();
+
+            Cout << Endl;
+
+            for (auto color : colors) {
+                Cout << color << ": " << sizeByColor[color] << Endl;
+            }
+
+            Cout << Endl;
+        };
+
+        printLimitsFn(130);
+        printLimitsFn(100);
+        printLimitsFn(65);
+        printLimitsFn(13);
+    }
+}
+} // namespace NKikimr

ydb/core/blobstorage/pdisk/ut/ya.make

@@ -36,6 +36,7 @@ SRCS(
     blobstorage_pdisk_ut_run.cpp
     blobstorage_pdisk_ut_sectormap.cpp
     blobstorage_pdisk_restore_ut.cpp
+    blobstorage_pdisk_ut_color_limits.cpp
     mock/pdisk_mock.cpp
 )
 

ydb/core/mind/bscontroller/ut_bscontroller/main.cpp

@@ -321,6 +321,51 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) {
                 UNIT_ASSERT(env.ParsePDisks(response.GetStatus(baseConfigIndex).GetBaseConfig()) == env.ExpectedPDisks); });
     }
 
+    Y_UNIT_TEST(PDiskUpdate) {
+        TEnvironmentSetup env(10, 1);
+        RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) {
+                TFinalizer finalizer(env);
+                env.Prepare(dispatchName, setup, outActiveZone);
+
+                {
+                    NKikimrBlobStorage::TConfigRequest request;
+                    env.DefineBox(1, "test box", {
+                            {"/dev/disk1", NKikimrBlobStorage::ROT, false, false, 0},
+                        }, env.GetNodes(), request);
+
+                    size_t baseConfigIndex = request.CommandSize();
+                    request.AddCommand()->MutableQueryBaseConfig();
+
+                    NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
+                    UNIT_ASSERT(response.GetSuccess());
+                    UNIT_ASSERT(env.ParsePDisks(response.GetStatus(baseConfigIndex).GetBaseConfig()) == env.ExpectedPDisks);
+                }
+
+                {
+                    NKikimrBlobStorage::TConfigRequest request;
+
+                    auto& hostcfg = *request.AddCommand()->MutableDefineHostConfig();
+                    hostcfg.SetHostConfigId(env.NextHostConfigId - 1);
+                    hostcfg.SetItemConfigGeneration(1);
+
+                    auto& drive = *hostcfg.AddDrive();
+                    drive.SetPath("/dev/disk1");
+                    drive.SetType(NKikimrBlobStorage::ROT);
+                    drive.SetSharedWithOs(false);
+                    drive.SetReadCentric(false);
+                    drive.SetKind(0);
+                    drive.MutablePDiskConfig()->SetChunkBaseLimit(65);
+
+                    size_t baseConfigIndex = request.CommandSize();
+                    request.AddCommand()->MutableQueryBaseConfig();
+
+                    NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
+                    UNIT_ASSERT(response.GetSuccess());
+                    UNIT_ASSERT(env.ParsePDisks(response.GetStatus(baseConfigIndex).GetBaseConfig()) == env.ExpectedPDisks);
+                }
+         });
+    }
+
     Y_UNIT_TEST(ManyPDisksRestarts) {
         int nodes = 100;
         TEnvironmentSetup env(nodes, 1);

ydb/core/protos/blobstorage_pdisk_config.proto

@@ -89,5 +89,6 @@ message TPDiskConfig {
     optional uint64 InsaneLogChunksMultiplier = 2000;  // Log of cutThreshold * InsaneLogChunksMultiplier is insane
 
     optional uint64 ExpectedSlotCount = 2001; // Number of slots to calculate per-vdisk disk space limit.
-};
 
+    optional uint32 ChunkBaseLimit = 2002; // Free chunk permille that triggers Cyan color (e.g. 100 is 10%). Between 130 (default) and 13.
+};