Revert "Revert "Support arbitrary chain set in huge blob keeper heap (y…

…db-platform#11566)" (ydb-platform#11891)" This reverts commit f5beef5.
alexvru · Nov 25, 2024 · a88c2af · a88c2af
1 parent b07b2ad
commit a88c2af
Show file tree

Hide file tree

Showing 14 changed files with 402 additions and 516 deletions.
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
@@ -137,7 +137,6 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
 
     bool InitHugeBlobKeeper(const TActorContext &ctx, const TStartingPoints &startingPoints) {
         Y_UNUSED(ctx);
-        const ui32 oldMinHugeBlobInBytes = 64 << 10;
         const ui32 milestoneHugeBlobInBytes = 64 << 10;
         const ui32 maxBlobInBytes = 128 << 10;
         auto logFunc = [] (const TString) { /* empty */ };
@@ -150,7 +149,6 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
                         HmCtx->PDiskCtx->Dsk->ChunkSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
-                        oldMinHugeBlobInBytes,
                         milestoneHugeBlobInBytes,
                         maxBlobInBytes,
                         HmCtx->Config->HugeBlobOverhead,
@@ -169,7 +167,6 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
                         HmCtx->PDiskCtx->Dsk->ChunkSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
                         HmCtx->PDiskCtx->Dsk->AppendBlockSize,
-                        oldMinHugeBlobInBytes,
                         milestoneHugeBlobInBytes,
                         maxBlobInBytes,
                         HmCtx->Config->HugeBlobOverhead,

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
@@ -136,9 +136,7 @@ namespace NKikimr {
                 MinHugeBlobInBytes = 512u << 10u;
                 break;
         }
-        OldMinHugeBlobInBytes = MinHugeBlobInBytes; // preserved to migrate entry point state correctly 
         MilestoneHugeBlobInBytes = 512u << 10u;  // for compatibility reasons it must be 512KB
-
     }
 
     void TVDiskConfig::Merge(const NKikimrBlobStorage::TVDiskConfig &update) {

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
@@ -120,7 +120,6 @@ namespace NKikimr {
         ui32 HullSstSizeInChunksLevel;
         ui32 HugeBlobsFreeChunkReservation;
         ui32 MinHugeBlobInBytes;
-        ui32 OldMinHugeBlobInBytes;
         ui32 MilestoneHugeBlobInBytes;
         ui32 HugeBlobOverhead;
         ui32 HullCompLevel0MaxSstsAtOnce;

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
@@ -3,23 +3,23 @@
 
 namespace NKikimr {
 
-    THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable)
+    THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo,
+            TSearchTable &&searchTable)
         : AppendBlockSize(appendBlockSize)
+        , MinHugeBlobInBlocks(minHugeBlobInBlocks)
         , AllSlotsInfo(std::move(slotsInfo))
         , SearchTable(std::move(searchTable))
     {}
 
     const THugeSlotsMap::TSlotInfo *THugeSlotsMap::GetSlotInfo(ui32 size) const {
-        ui32 sizeInBlocks = size / AppendBlockSize;
-        sizeInBlocks += !(sizeInBlocks * AppendBlockSize == size);
-        const ui64 idx = SearchTable.at(sizeInBlocks);
+        const ui32 sizeInBlocks = (size + AppendBlockSize - 1) / AppendBlockSize;
+        Y_ABORT_UNLESS(MinHugeBlobInBlocks <= sizeInBlocks);
+        const ui64 idx = SearchTable.at(sizeInBlocks - MinHugeBlobInBlocks);
         return &AllSlotsInfo.at(idx);
     }
 
     ui32 THugeSlotsMap::AlignByBlockSize(ui32 size) const {
-        ui32 sizeInBlocks = size / AppendBlockSize;
-        Y_ABORT_UNLESS(sizeInBlocks,  "Blob size to align is smaller than a single block. BlobSize# %" PRIu32, size);
-        return sizeInBlocks * AppendBlockSize;
+        return Max(MinHugeBlobInBlocks * AppendBlockSize, size - size % AppendBlockSize);
     }
 
     void THugeSlotsMap::Output(IOutputStream &str) const {
@@ -31,11 +31,7 @@ namespace NKikimr {
         str << "]}\n";
         str << "{SearchTable# [";
         for (const auto &idx : SearchTable) {
-            if (idx != NoOpIdx) {
-                AllSlotsInfo.at(idx).Output(str);
-            } else {
-                str << "null";
-            }
+            AllSlotsInfo.at(idx).Output(str);
             str << "\n";
         }
         str << "]}";

diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
@@ -38,23 +38,21 @@ namespace NKikimr {
         };
 
         // All slot types
-        using TAllSlotsInfo = TVector<TSlotInfo>;
+        using TAllSlotsInfo = std::vector<TSlotInfo>;
         // Type to address TAllSlotsInfo
         using TIndex = ui16;
         // Size in AppendBlockSize -> index in TAllSlotsInfo
-        using TSearchTable = TVector<TIndex>;
-        // Idx that indicates there is no record for it in TAllSlotsInfo
-        static constexpr TIndex NoOpIdx = Max<TIndex>();
+        using TSearchTable = std::vector<TIndex>;
 
-
-        THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
+        THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
         const TSlotInfo *GetSlotInfo(ui32 size) const;
         ui32 AlignByBlockSize(ui32 size) const;
         void Output(IOutputStream &str) const;
         TString ToString() const;
 
     private:
         const ui32 AppendBlockSize;
+        const ui32 MinHugeBlobInBlocks;
         TAllSlotsInfo AllSlotsInfo;
         TSearchTable SearchTable;
     };

diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
@@ -17,7 +17,6 @@ namespace NKikimr {
         Y_UNIT_TEST(SerializeParse) {
             ui32 chunkSize = 134274560u;
             ui32 appendBlockSize = 56896u;
-            ui32 minHugeBlobInBytes = 512u << 10u;
             ui32 milestoneHugeBlobInBytes = 512u << 10u;
             ui32 maxBlobInBytes = 10u << 20u;
             ui32 overhead = 8;
@@ -29,8 +28,8 @@ namespace NKikimr {
             auto vctx = MakeIntrusive<TVDiskContext>(TActorId(), info->PickTopology(), counters, TVDiskID(0, 1, 0, 0, 0),
                 nullptr, NPDisk::DEVICE_TYPE_UNKNOWN);
             std::unique_ptr<THullHugeKeeperPersState> state(
-                    new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize, appendBlockSize,
-                        minHugeBlobInBytes, milestoneHugeBlobInBytes, maxBlobInBytes,
+                    new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize,
+                        appendBlockSize, milestoneHugeBlobInBytes, maxBlobInBytes,
                         overhead, freeChunksReservation, logf));
 
             state->LogPos = THullHugeRecoveryLogPos(0, 0, 100500, 50000, 70000, 56789, 39482);