Revert "Support arbitrary chain set in huge blob keeper heap" (#11961)

ydb-platform · Nov 25, 2024 · a328b30 · a328b30
1 parent ec75229
commit a328b30
Show file tree

Hide file tree

Showing 13 changed files with 529 additions and 390 deletions.
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
@@ -137,6 +137,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
 
     bool InitHugeBlobKeeper(const TActorContext &ctx, const TStartingPoints &startingPoints) {
         Y_UNUSED(ctx);
+        const ui32 oldMinHugeBlobInBytes = 64 << 10;
         const ui32 milestoneHugeBlobInBytes = 64 << 10;
         const ui32 maxBlobInBytes = 128 << 10;
         auto logFunc = [] (const TString) { /* empty */ };
@@ -149,6 +150,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
                         HmCtx->PDiskCtx->Dsk->ChunkSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
+                        oldMinHugeBlobInBytes,
                         milestoneHugeBlobInBytes,
                         maxBlobInBytes,
                         HmCtx->Config->HugeBlobOverhead,
@@ -167,6 +169,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
                         HmCtx->PDiskCtx->Dsk->ChunkSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
+                        oldMinHugeBlobInBytes,
                         milestoneHugeBlobInBytes,
                         maxBlobInBytes,
                         HmCtx->Config->HugeBlobOverhead,

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
@@ -136,7 +136,9 @@ namespace NKikimr {
                 MinHugeBlobInBytes = 512u << 10u;
                 break;
         }
+        OldMinHugeBlobInBytes = MinHugeBlobInBytes; // preserved to migrate entry point state correctly 
         MilestoneHugeBlobInBytes = 512u << 10u;  // for compatibility reasons it must be 512KB
+
     }
 
     void TVDiskConfig::Merge(const NKikimrBlobStorage::TVDiskConfig &update) {

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
@@ -120,6 +120,7 @@ namespace NKikimr {
         ui32 HullSstSizeInChunksLevel;
         ui32 HugeBlobsFreeChunkReservation;
         ui32 MinHugeBlobInBytes;
+        ui32 OldMinHugeBlobInBytes;
         ui32 MilestoneHugeBlobInBytes;
         ui32 HugeBlobOverhead;
         ui32 HullCompLevel0MaxSstsAtOnce;

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
@@ -3,23 +3,23 @@
 
 namespace NKikimr {
 
-    THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo,
-            TSearchTable &&searchTable)
+    THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable)
         : AppendBlockSize(appendBlockSize)
-        , MinHugeBlobInBlocks(minHugeBlobInBlocks)
         , AllSlotsInfo(std::move(slotsInfo))
         , SearchTable(std::move(searchTable))
     {}
 
     const THugeSlotsMap::TSlotInfo *THugeSlotsMap::GetSlotInfo(ui32 size) const {
-        const ui32 sizeInBlocks = (size + AppendBlockSize - 1) / AppendBlockSize;
-        Y_ABORT_UNLESS(MinHugeBlobInBlocks <= sizeInBlocks);
-        const ui64 idx = SearchTable.at(sizeInBlocks - MinHugeBlobInBlocks);
+        ui32 sizeInBlocks = size / AppendBlockSize;
+        sizeInBlocks += !(sizeInBlocks * AppendBlockSize == size);
+        const ui64 idx = SearchTable.at(sizeInBlocks);
         return &AllSlotsInfo.at(idx);
     }
 
     ui32 THugeSlotsMap::AlignByBlockSize(ui32 size) const {
-        return Max(MinHugeBlobInBlocks * AppendBlockSize, size - size % AppendBlockSize);
+        ui32 sizeInBlocks = size / AppendBlockSize;
+        Y_ABORT_UNLESS(sizeInBlocks,  "Blob size to align is smaller than a single block. BlobSize# %" PRIu32, size);
+        return sizeInBlocks * AppendBlockSize;
     }
 
     void THugeSlotsMap::Output(IOutputStream &str) const {
@@ -31,7 +31,11 @@ namespace NKikimr {
         str << "]}\n";
         str << "{SearchTable# [";
         for (const auto &idx : SearchTable) {
-            AllSlotsInfo.at(idx).Output(str);
+            if (idx != NoOpIdx) {
+                AllSlotsInfo.at(idx).Output(str);
+            } else {
+                str << "null";
+            }
             str << "\n";
         }
         str << "]}";

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
@@ -38,21 +38,23 @@ namespace NKikimr {
         };
 
         // All slot types
-        using TAllSlotsInfo = std::vector<TSlotInfo>;
+        using TAllSlotsInfo = TVector<TSlotInfo>;
         // Type to address TAllSlotsInfo
         using TIndex = ui16;
         // Size in AppendBlockSize -> index in TAllSlotsInfo
-        using TSearchTable = std::vector<TIndex>;
+        using TSearchTable = TVector<TIndex>;
+        // Idx that indicates there is no record for it in TAllSlotsInfo
+        static constexpr TIndex NoOpIdx = Max<TIndex>();
 
-        THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
+
+        THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
         const TSlotInfo *GetSlotInfo(ui32 size) const;
         ui32 AlignByBlockSize(ui32 size) const;
         void Output(IOutputStream &str) const;
         TString ToString() const;
 
     private:
         const ui32 AppendBlockSize;
-        const ui32 MinHugeBlobInBlocks;
         TAllSlotsInfo AllSlotsInfo;
         TSearchTable SearchTable;
     };

diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
@@ -17,6 +17,7 @@ namespace NKikimr {
         Y_UNIT_TEST(SerializeParse) {
             ui32 chunkSize = 134274560u;
             ui32 appendBlockSize = 56896u;
+            ui32 minHugeBlobInBytes = 512u << 10u;
             ui32 milestoneHugeBlobInBytes = 512u << 10u;
             ui32 maxBlobInBytes = 10u << 20u;
             ui32 overhead = 8;
@@ -28,8 +29,8 @@ namespace NKikimr {
             auto vctx = MakeIntrusive<TVDiskContext>(TActorId(), info->PickTopology(), counters, TVDiskID(0, 1, 0, 0, 0),
                 nullptr, NPDisk::DEVICE_TYPE_UNKNOWN);
             std::unique_ptr<THullHugeKeeperPersState> state(
-                    new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize,
-                        appendBlockSize, milestoneHugeBlobInBytes, maxBlobInBytes,
+                    new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize, appendBlockSize,
+                        minHugeBlobInBytes, milestoneHugeBlobInBytes, maxBlobInBytes,
                         overhead, freeChunksReservation, logf));
 
             state->LogPos = THullHugeRecoveryLogPos(0, 0, 100500, 50000, 70000, 56789, 39482);