Skip to content

Commit

Permalink
Huge defrag smooth linear threshold (#9825)
Browse files Browse the repository at this point in the history
  • Loading branch information
@robdrynkin
robdrynkin authored Nov 7, 2024
1 parent 27f35fa commit 8f0d66f
Show file tree
Hide file tree
Showing 10 changed files with 84 additions and 29 deletions.
2 changes: 2 additions & 0 deletions ydb/core/blobstorage/nodewarden/node_warden_impl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ TNodeWarden::TNodeWarden(const TIntrusivePtr<TNodeWardenConfig> &cfg)
, MaxSyncLogChunksInFlightHDD(10, 1, 1024)
, MaxSyncLogChunksInFlightSSD(10, 1, 1024)
, DefaultHugeGarbagePerMille(300, 1, 1000)
, HugeDefragFreeSpaceBorderPerMille(260, 1, 1000)
, MaxCommonLogChunksHDD(200, 1, 1'000'000)
, MaxCommonLogChunksSSD(200, 1, 1'000'000)
, CostMetricsParametersByMedia({
Expand Down Expand Up @@ -325,6 +326,7 @@ void TNodeWarden::Bootstrap() {
icb->RegisterSharedControl(MaxSyncLogChunksInFlightHDD, "VDiskControls.MaxSyncLogChunksInFlightHDD");
icb->RegisterSharedControl(MaxSyncLogChunksInFlightSSD, "VDiskControls.MaxSyncLogChunksInFlightSSD");
icb->RegisterSharedControl(DefaultHugeGarbagePerMille, "VDiskControls.DefaultHugeGarbagePerMille");
icb->RegisterSharedControl(HugeDefragFreeSpaceBorderPerMille, "VDiskControls.HugeDefragFreeSpaceBorderPerMille");
icb->RegisterSharedControl(MaxCommonLogChunksHDD, "PDiskControls.MaxCommonLogChunksHDD");
icb->RegisterSharedControl(MaxCommonLogChunksSSD, "PDiskControls.MaxCommonLogChunksSSD");

Expand Down
1 change: 1 addition & 0 deletions ydb/core/blobstorage/nodewarden/node_warden_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ namespace NKikimr::NStorage {
TControlWrapper MaxSyncLogChunksInFlightHDD;
TControlWrapper MaxSyncLogChunksInFlightSSD;
TControlWrapper DefaultHugeGarbagePerMille;
TControlWrapper HugeDefragFreeSpaceBorderPerMille;
TControlWrapper MaxCommonLogChunksHDD;
TControlWrapper MaxCommonLogChunksSSD;

Expand Down
1 change: 1 addition & 0 deletions ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ namespace NKikimr::NStorage {
vdiskConfig->ReplPausedAtStart = Cfg->VDiskReplPausedAtStart;
vdiskConfig->EnableVPatch = EnableVPatch;
vdiskConfig->DefaultHugeGarbagePerMille = DefaultHugeGarbagePerMille;
vdiskConfig->HugeDefragFreeSpaceBorderPerMille = HugeDefragFreeSpaceBorderPerMille;

vdiskConfig->EnableLocalSyncLogDataCutting = EnableLocalSyncLogDataCutting;
if (deviceType == NPDisk::EDeviceType::DEVICE_TYPE_ROT) {
Expand Down
20 changes: 12 additions & 8 deletions ydb/core/blobstorage/ut_blobstorage/defrag.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ Y_UNIT_TEST_SUITE(Defragmentation) {
++chunkToBlobs[item.Location.ChunkIdx];
}
}
UNIT_ASSERT_VALUES_EQUAL(chunkToBlobs.size(), 1);
UNIT_ASSERT_VALUES_EQUAL(chunkToBlobs.size(), 9 + 1); // defragmentation stopping if number of can be freed chunks is 9 + 1 chunk really used
}
}

Expand Down Expand Up @@ -232,6 +232,8 @@ Y_UNIT_TEST_SUITE(Defragmentation) {
TAutoPtr<NPDisk::TEvChunkReadResult> readMsg;
bool caughtRestore = false;
bool caughtDone = false;
ui32 rewrittenRecs = 0;
ui32 rewrittenBytes = 0;
env.Runtime->FilterFunction = [&](ui32 nodeId, std::unique_ptr<IEventHandle>& ev) {
Y_UNUSED(nodeId);
switch(ev->Type) {
Expand All @@ -251,13 +253,12 @@ Y_UNIT_TEST_SUITE(Defragmentation) {
}
return true;
case TEvBlobStorage::EvDefragRewritten:
if (rewriterActorId == ev->Sender) {
UNIT_ASSERT_VALUES_EQUAL(caughtDone, false);
{
caughtDone = true;

const TEvDefragRewritten* msg = ev->Get<TEvDefragRewritten>();
UNIT_ASSERT_VALUES_EQUAL(msg->RewrittenRecs, 18);
UNIT_ASSERT_VALUES_EQUAL(msg->RewrittenBytes, 9961567);
rewrittenRecs += msg->RewrittenRecs;
rewrittenBytes += msg->RewrittenBytes;
}
return true;
case TEvBlobStorage::EvRestoreCorruptedBlob:
Expand All @@ -278,9 +279,12 @@ Y_UNIT_TEST_SUITE(Defragmentation) {
return true;
}
};
while (!caughtDone || !caughtRestore) {
env.Sim(TDuration::Minutes(1));
}
env.Sim(TDuration::Minutes(10));
UNIT_ASSERT_VALUES_EQUAL(caughtDone, true);
UNIT_ASSERT_VALUES_EQUAL(caughtRestore, true);
UNIT_ASSERT_VALUES_EQUAL(rewrittenRecs, 20 - (9 + 1)); // // defragmentation stopping if number of can be freed chunks is 9 + 1 chunk really used
UNIT_ASSERT_VALUES_EQUAL(rewrittenBytes, 5767223);

}

Y_UNIT_TEST(CorruptedReadHandling) {
Expand Down
1 change: 1 addition & 0 deletions ydb/core/blobstorage/vdisk/common/vdisk_config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ namespace NKikimr {
bool EnableVDiskCooldownTimeout;
TControlWrapper EnableVPatch = true;
TControlWrapper DefaultHugeGarbagePerMille;
TControlWrapper HugeDefragFreeSpaceBorderPerMille;
bool UseActorSystemTimeInBSQueue = false;

///////////// BALANCING SETTINGS ////////////////////
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/blobstorage/vdisk/common/vdisk_mongroups.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -576,9 +576,11 @@ public:
GROUP_CONSTRUCTOR(TDefragGroup)
{
COUNTER_INIT_IF_EXTENDED(DefragBytesRewritten, true);
COUNTER_INIT_IF_EXTENDED(DefragThreshold, false);
}

COUNTER_DEF(DefragBytesRewritten);
COUNTER_DEF(DefragThreshold);
};

///////////////////////////////////////////////////////////////////////////////////
Expand Down
53 changes: 36 additions & 17 deletions ydb/core/blobstorage/vdisk/defrag/defrag_actor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,29 @@ namespace NKikimr {
{}
};

double DefragThreshold(
const TOutOfSpaceState& oos,
double defaultPercent,
double hugeDefragFreeSpaceBorder
) {
double multiplier = Min(oos.GetFreeSpaceShare() / hugeDefragFreeSpaceBorder, 1.0);
return defaultPercent * multiplier;
}

const ui32 MIN_CAN_BE_FREED_CHUNKS = 9;

bool HugeHeapDefragmentationRequired(
ui32 hugeCanBeFreedChunks,
ui32 hugeTotalChunks,
double defragThreshold) {

if (hugeCanBeFreedChunks <= MIN_CAN_BE_FREED_CHUNKS)
return false;

double percentOfGarbage = static_cast<double>(hugeCanBeFreedChunks) / hugeTotalChunks;
return percentOfGarbage >= defragThreshold;
}

////////////////////////////////////////////////////////////////////////////
// HugeHeapDefragmentationRequired
// We calculate allowd percent of garbage as a percent of chunks
Expand All @@ -51,22 +74,17 @@ namespace NKikimr {
const TOutOfSpaceState& oos,
ui32 hugeCanBeFreedChunks,
ui32 hugeTotalChunks,
double defaultPercent) {

if (hugeCanBeFreedChunks < 10)
return false;

double percentOfGarbage = static_cast<double>(hugeCanBeFreedChunks) / hugeTotalChunks;
double defaultPercent,
double hugeDefragFreeSpaceBorder) {
double defragThreshold = DefragThreshold(oos, defaultPercent, hugeDefragFreeSpaceBorder);
return HugeHeapDefragmentationRequired(hugeCanBeFreedChunks, hugeTotalChunks, defragThreshold);
}

if (oos.GetLocalColor() > TSpaceColor::CYAN) {
// For anything worse than CYAN
return percentOfGarbage >= Min(0.02, defaultPercent);
} else if (oos.GetLocalColor() > TSpaceColor::GREEN) {
// For CYAN
return percentOfGarbage >= Min(0.15, defaultPercent);
ui32 MaxInflyghtDefragChunks(const TOutOfSpaceState& oos, ui32 maxChunksToDefrag, ui32 hugeCanBeFreedChunks) {
if (oos.GetLocalColor() > TSpaceColor::GREEN) {
return Min(maxChunksToDefrag, hugeCanBeFreedChunks - MIN_CAN_BE_FREED_CHUNKS);
} else {
// For GREEN
return percentOfGarbage >= Min(0.30, defaultPercent);
return 1;
}
}

Expand Down Expand Up @@ -120,8 +138,10 @@ namespace NKikimr {
Y_ABORT_UNLESS(usefulChunks <= totalChunks);
const ui32 canBeFreedChunks = totalChunks - usefulChunks;
double defaultPercent = DCtx->VCfg->DefaultHugeGarbagePerMille / 1000.0;
if (HugeHeapDefragmentationRequired(oos, canBeFreedChunks, totalChunks, defaultPercent)) {
TChunksToDefrag chunksToDefrag = calcStat.GetChunksToDefrag(DCtx->MaxChunksToDefrag);
double hugeDefragFreeSpaceBorder = DCtx->VCfg->HugeDefragFreeSpaceBorderPerMille / 1000.0;
DCtx->DefragMonGroup.DefragThreshold() = DefragThreshold(oos, defaultPercent, hugeDefragFreeSpaceBorder);
if (HugeHeapDefragmentationRequired(oos, canBeFreedChunks, totalChunks, defaultPercent, hugeDefragFreeSpaceBorder)) {
TChunksToDefrag chunksToDefrag = calcStat.GetChunksToDefrag(MaxInflyghtDefragChunks(oos, DCtx->MaxChunksToDefrag, canBeFreedChunks));
Y_ABORT_UNLESS(chunksToDefrag);
STLOG(PRI_INFO, BS_VDISK_DEFRAG, BSVDD03, VDISKP(DCtx->VCtx->VDiskLogPrefix, "scan finished"),
(TotalChunks, totalChunks), (UsefulChunks, usefulChunks),
Expand Down Expand Up @@ -280,7 +300,6 @@ namespace NKikimr {
Sublog.Log() << "Defrag quantum has been finished\n";

auto *msg = ev->Get();
Y_ABORT_UNLESS(msg->Stat.Eof || msg->Stat.FreedChunks.size() == DCtx->MaxChunksToDefrag);

auto &task = WaitQueue.front();

Expand Down
3 changes: 2 additions & 1 deletion ydb/core/blobstorage/vdisk/defrag/defrag_actor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ namespace NKikimr {
const TOutOfSpaceState& oos,
ui32 hugeCanBeFreedChunks,
ui32 hugeTotalChunks,
double defaultPercent);
double defaultPercent,
double hugeDefragFreeSpaceShareThreshold);

////////////////////////////////////////////////////////////////////////////
// VDISK DEFRAG ACTOR CREATOR
Expand Down
25 changes: 22 additions & 3 deletions ydb/core/blobstorage/vdisk/defrag/defrag_actor_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,42 @@ namespace NKikimr {
Y_UNIT_TEST(HugeHeapDefragmentationRequired) {
{
TOutOfSpaceState oos(1, 0);
oos.UpdateLocalFreeSpaceShare(ui64(1 << 24) * 0.5);
ui32 hugeCanBeFreedChunks = 9;
ui32 hugeUsedChunks = 20;
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30);
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30, 0.13);
UNIT_ASSERT(!defrag);
}
{
TOutOfSpaceState oos(1, 0);
oos.UpdateLocalFreeSpaceShare(ui64(1 << 24) * 0.5);
ui32 hugeCanBeFreedChunks = 200;
ui32 hugeUsedChunks = 1000;
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30);
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30, 0.13);
UNIT_ASSERT(!defrag);
}
{
TOutOfSpaceState oos(1, 0);
oos.UpdateLocalFreeSpaceShare(ui64(1 << 24) * 0.5);
ui32 hugeCanBeFreedChunks = 301;
ui32 hugeUsedChunks = 1000;
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30);
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30, 0.13);
UNIT_ASSERT(defrag);
}
{
TOutOfSpaceState oos(1, 0);
oos.UpdateLocalFreeSpaceShare(ui64(1 << 24) * 0.05);
ui32 hugeUsedChunks = 1000;
ui32 hugeCanBeFreedChunks = ui32(5.0 / 13 * 0.3 * hugeUsedChunks) - 1;
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30, 0.13);
UNIT_ASSERT(!defrag);
}
{
TOutOfSpaceState oos(1, 0);
oos.UpdateLocalFreeSpaceShare(ui64(1 << 24) * 0.05);
ui32 hugeUsedChunks = 1000;
ui32 hugeCanBeFreedChunks = ui32(0.05 / 0.13 * 0.3 * hugeUsedChunks) + 1;
bool defrag = HugeHeapDefragmentationRequired(oos, hugeCanBeFreedChunks, hugeUsedChunks, 0.30, 0.13);
UNIT_ASSERT(defrag);
}
}
Expand Down
5 changes: 5 additions & 0 deletions ydb/core/protos/config.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1362,6 +1362,11 @@ message TImmediateControlsConfig {
MinValue: 1,
MaxValue: 1000,
DefaultValue: 300 }];
optional uint64 HugeDefragFreeSpaceBorderPerMille = 13 [(ControlOptions) = {
Description: "Huge garbage threshold = DefaultHugeGarbagePerMille * min((free space share) / (this param), 1)",
MinValue: 1,
MaxValue: 1000,
DefaultValue: 260 }];
}

message TTabletControls {
Expand Down

0 comments on commit 8f0d66f

Please sign in to comment.