limit in-fly of tablet deletions (ydb-platform#13993)

GrigoriyPA · Jan 30, 2025 · 43d4c47 · 43d4c47
1 parent 6dedd76
commit 43d4c47
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 10 deletions.
diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp
@@ -516,9 +516,7 @@ void THive::Handle(TEvPrivate::TEvBootTablets::TPtr&) {
         } else if (tablet.IsReadyToBlockStorage()) {
             tablet.InitiateBlockStorage(sideEffects);
         } else if (tablet.IsDeleting()) {
-            if (!tablet.InitiateBlockStorage(sideEffects, std::numeric_limits<ui32>::max())) {
-                DeleteTabletWithoutStorage(&tablet);
-            }
+            BlockStorageForDelete(tablet.Id, sideEffects);
         } else if (tablet.IsLockedToActor()) {
             // we are wating for a lock
         } else if (tablet.IsExternalBoot()) {
@@ -855,9 +853,7 @@ void THive::Handle(TEvHive::TEvInitiateBlockStorage::TPtr& ev) {
     TLeaderTabletInfo* tablet = FindTabletEvenInDeleting(tabletId);
     if (tablet != nullptr) {
         if (tablet->IsDeleting()) {
-            if (!tablet->InitiateBlockStorage(sideEffects, std::numeric_limits<ui32>::max())) {
-                DeleteTabletWithoutStorage(tablet);
-            }
+            BlockStorageForDelete(tabletId, sideEffects);
         } else
         if (tablet->IsReadyToBlockStorage()) {
             tablet->InitiateBlockStorage(sideEffects);
@@ -2880,6 +2876,21 @@ ui64 THive::GetObjectImbalance(TFullObjectId object) {
     return it->second->GetImbalance();
 }
 
+void THive::BlockStorageForDelete(TTabletId tabletId, TSideEffects& sideEffects) {
+    auto* tablet = FindTabletEvenInDeleting(tabletId);
+    if (tablet == nullptr) {
+        return;
+    }
+    if (DeleteTabletInProgress < MAX_DELETE_TABLET_IN_PROGRESS) {
+        ++DeleteTabletInProgress;
+        if (!tablet->InitiateBlockStorage(sideEffects, std::numeric_limits<ui32>::max())) {
+            DeleteTabletWithoutStorage(tablet);
+        }
+    } else {
+        DeleteTabletQueue.push(tabletId);
+    }
+}
+
 THive::THive(TTabletStorageInfo *info, const TActorId &tablet)
     : TActor(&TThis::StateInit)
     , TTabletExecutedFlat(info, tablet, new NMiniKQL::TMiniKQLFactory)

diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h
@@ -436,6 +436,9 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
 
     ui64 UpdateTabletMetricsInProgress = 0;
     static constexpr ui64 MAX_UPDATE_TABLET_METRICS_IN_PROGRESS = 10000; // 10K
+    i64 DeleteTabletInProgress = 0;
+    static constexpr i64 MAX_DELETE_TABLET_IN_PROGRESS = 100;
+    std::queue<TTabletId> DeleteTabletQueue;
 
     TString BootStateBooting = "Booting";
     TString BootStateStarting = "Starting";
@@ -710,6 +713,7 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
     TDuration GetBalancerCooldown(EBalancerType balancerType) const;
     void UpdateObjectCount(const TLeaderTabletInfo& tablet, const TNodeInfo& node, i64 diff);
     ui64 GetObjectImbalance(TFullObjectId object);
+    void BlockStorageForDelete(TTabletId tabletId, TSideEffects& sideEffects);
 
     ui32 GetEventPriority(IEventHandle* ev);
     void PushProcessIncomingEvent();

diff --git a/ydb/core/mind/hive/tx__delete_tablet.cpp b/ydb/core/mind/hive/tx__delete_tablet.cpp
@@ -39,9 +39,7 @@ class TTxDeleteBase : public TTransactionBase<THive> {
                     follower.InitiateStop(SideEffects);
                     db.Table<Schema::TabletFollowerTablet>().Key(follower.GetFullTabletId()).Update<Schema::TabletFollowerTablet::FollowerNode>(0);
                 }
-                if (!tablet->InitiateBlockStorage(SideEffects, std::numeric_limits<ui32>::max())) {
-                    Self->DeleteTabletWithoutStorage(tablet, SideEffects);
-                }
+                Self->BlockStorageForDelete(tabletId, SideEffects);
             } else {
                 BLOG_D("THive::TTxDeleteTablet::Execute Tablet " << tabletId << " already in ETabletState::Deleting");
             }

diff --git a/ydb/core/mind/hive/tx__delete_tablet_result.cpp b/ydb/core/mind/hive/tx__delete_tablet_result.cpp
@@ -8,6 +8,7 @@ class TTxDeleteTabletResult : public TTransactionBase<THive> {
     TEvTabletBase::TEvDeleteTabletResult::TPtr Result;
     TTabletId TabletId;
     TSideEffects SideEffects;
+    bool Success;
 
 public:
     TTxDeleteTabletResult(TEvTabletBase::TEvDeleteTabletResult::TPtr& ev, THive* hive)
@@ -20,6 +21,7 @@ class TTxDeleteTabletResult : public TTransactionBase<THive> {
 
     bool Execute(TTransactionContext& txc, const TActorContext&) override {
         SideEffects.Reset(Self->SelfId());
+        Success = true;
         TEvTabletBase::TEvDeleteTabletResult* msg = Result->Get();
         BLOG_D("THive::TTxDeleteTabletResult::Execute(" << TabletId << " " << NKikimrProto::EReplyStatus_Name(msg->Status) << ")");
         TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(TabletId);
@@ -57,6 +59,7 @@ class TTxDeleteTabletResult : public TTransactionBase<THive> {
                 Self->PendingCreateTablets.erase({tablet->Owner.first, tablet->Owner.second});
                 Self->DeleteTablet(tablet->Id);
             } else {
+                Success = false;
                 BLOG_W("THive::TTxDeleteTabletResult retrying for " << TabletId << " because of " << NKikimrProto::EReplyStatus_Name(msg->Status));
                 Y_ENSURE_LOG(tablet->IsDeleting(), " tablet " << tablet->Id);
                 SideEffects.Schedule(TDuration::MilliSeconds(1000), new TEvHive::TEvInitiateDeleteStorage(tablet->Id));
@@ -67,8 +70,14 @@ class TTxDeleteTabletResult : public TTransactionBase<THive> {
 
     void Complete(const TActorContext& ctx) override {
         BLOG_D("THive::TTxDeleteTabletResult(" << TabletId << ")::Complete SideEffects " << SideEffects);
+        if (Success) {
+            --Self->DeleteTabletInProgress;
+            while (!Self->DeleteTabletQueue.empty() && Self->DeleteTabletInProgress < THive::MAX_DELETE_TABLET_IN_PROGRESS) {
+                Self->BlockStorageForDelete(Self->DeleteTabletQueue.front(), SideEffects);
+                Self->DeleteTabletQueue.pop();
+            }
+        }
         SideEffects.Complete(ctx);
-
     }
 };