diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index be5c664ea078..b56b6dfe11f4 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -131,6 +131,8 @@ void TBlobStorageController::Handle(TEvNodeWardenStorageConfig::TPtr ev) { auto prevStaticVSlots = std::exchange(StaticVSlots, {}); StaticVDiskMap.clear(); + const TMonotonic mono = TActivationContext::Monotonic(); + if (StorageConfig.HasBlobStorageConfig()) { if (const auto& bsConfig = StorageConfig.GetBlobStorageConfig(); bsConfig.HasServiceSet()) { const auto& ss = bsConfig.GetServiceSet(); @@ -143,7 +145,7 @@ void TBlobStorageController::Handle(TEvNodeWardenStorageConfig::TPtr ev) { const auto& location = vslot.GetVDiskLocation(); const TPDiskId pdiskId(location.GetNodeID(), location.GetPDiskID()); const TVSlotId vslotId(pdiskId, location.GetVDiskSlotID()); - StaticVSlots.try_emplace(vslotId, vslot, prevStaticVSlots); + StaticVSlots.try_emplace(vslotId, vslot, prevStaticVSlots, mono); const TVDiskID& vdiskId = VDiskIDFromVDiskID(vslot.GetVDiskID()); StaticVDiskMap.emplace(vdiskId, vslotId); StaticVDiskMap.emplace(TVDiskID(vdiskId.GroupID, 0, vdiskId), vslotId); diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h index 7352658bc107..8c1bb321e5ec 100644 --- a/ydb/core/mind/bscontroller/config.h +++ b/ydb/core/mind/bscontroller/config.h @@ -101,6 +101,7 @@ namespace NKikimr { // when the config cmd received const TInstant Timestamp; + const TMonotonic Mono; // various settings from controller const bool DonorMode; @@ -124,7 +125,8 @@ namespace NKikimr { bool PushStaticGroupsToSelfHeal = false; public: - TConfigState(TBlobStorageController &controller, const THostRecordMap &hostRecords, TInstant timestamp) + TConfigState(TBlobStorageController &controller, const THostRecordMap &hostRecords, TInstant timestamp, + TMonotonic mono) : Self(controller) , HostConfigs(&controller.HostConfigs) , Boxes(&controller.Boxes) @@ -142,6 +144,7 @@ namespace NKikimr { , NextStoragePoolId(&controller.NextStoragePoolId) , HostRecords(hostRecords) , Timestamp(timestamp) + , Mono(mono) , DonorMode(controller.DonorMode) , DefaultMaxSlots(controller.DefaultMaxSlots) , StaticVSlots(controller.StaticVSlots) diff --git a/ydb/core/mind/bscontroller/config_cmd.cpp b/ydb/core/mind/bscontroller/config_cmd.cpp index 88fe17f61f34..ca5e9f91da25 100644 --- a/ydb/core/mind/bscontroller/config_cmd.cpp +++ b/ydb/core/mind/bscontroller/config_cmd.cpp @@ -180,7 +180,7 @@ namespace NKikimr::NBsController { Response->MutableStatus()->RemoveLast(); } - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); State->CheckConsistency(); TString m; diff --git a/ydb/core/mind/bscontroller/config_fit_groups.cpp b/ydb/core/mind/bscontroller/config_fit_groups.cpp index 0953ad03f97b..e3f1f199de01 100644 --- a/ydb/core/mind/bscontroller/config_fit_groups.cpp +++ b/ydb/core/mind/bscontroller/config_fit_groups.cpp @@ -607,6 +607,7 @@ namespace NKikimr { groupInfo->ID, 0, groupInfo->Generation, StoragePool.VDiskKind, failRealmIdx, failDomainIdx, vdiskIdx, TMood::Normal, groupInfo, &VSlotReadyTimestampQ, TInstant::Zero(), TDuration::Zero()); + vslotInfo->VDiskStatusTimestamp = State.Mono; // mark as uncommitted State.UncommittedVSlots.insert(vslotId); diff --git a/ydb/core/mind/bscontroller/drop_donor.cpp b/ydb/core/mind/bscontroller/drop_donor.cpp index 55558a8322ee..b9b0b9cdd944 100644 --- a/ydb/core/mind/bscontroller/drop_donor.cpp +++ b/ydb/core/mind/bscontroller/drop_donor.cpp @@ -18,7 +18,7 @@ class TBlobStorageController::TTxDropDonor TTxType GetTxType() const override { return NBlobStorageController::TXTYPE_DROP_DONOR; } bool Execute(TTransactionContext &txc, const TActorContext&) override { - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); State->CheckConsistency(); for (const TVSlotId& vslotId : VSlotIds) { if (const TVSlotInfo *vslot = State->VSlots.Find(vslotId); vslot && !vslot->IsBeingDeleted()) { diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index d6cd2b30f2fa..c8e24f664880 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -125,7 +125,7 @@ class TBlobStorageController : public TActor, public TTa public: std::optional VDiskStatus; - NHPTimer::STime VDiskStatusTimestamp = GetCycleCountFast(); + TMonotonic VDiskStatusTimestamp; bool IsReady = false; bool OnlyPhantomsRemain = false; @@ -2309,11 +2309,11 @@ class TBlobStorageController : public TActor, public TTa std::optional VDiskMetrics; std::optional VDiskStatus; - NHPTimer::STime VDiskStatusTimestamp = GetCycleCountFast(); + TMonotonic VDiskStatusTimestamp; TMonotonic ReadySince = TMonotonic::Max(); // when IsReady becomes true for this disk; Max() in non-READY state TStaticVSlotInfo(const NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk& vdisk, - std::map& prev) + std::map& prev, TMonotonic mono) : VDiskId(VDiskIDFromVDiskID(vdisk.GetVDiskID())) , VDiskKind(vdisk.GetVDiskKind()) { @@ -2325,6 +2325,8 @@ class TBlobStorageController : public TActor, public TTa VDiskStatus = item.VDiskStatus; VDiskStatusTimestamp = item.VDiskStatusTimestamp; ReadySince = item.ReadySince; + } else { + VDiskStatusTimestamp = mono; } } }; diff --git a/ydb/core/mind/bscontroller/load_everything.cpp b/ydb/core/mind/bscontroller/load_everything.cpp index b8eb7d59a5e2..a6efc5a39457 100644 --- a/ydb/core/mind/bscontroller/load_everything.cpp +++ b/ydb/core/mind/bscontroller/load_everything.cpp @@ -352,6 +352,7 @@ class TBlobStorageController::TTxLoadEverything : public TTransactionBaseVSlots.clear(); { using T = Schema::VSlot; @@ -374,6 +375,7 @@ class TBlobStorageController::TTxLoadEverything : public TTransactionBaseNotReadyVSlotIds.insert(x.VSlotId); } + x.VDiskStatusTimestamp = mono; if (!slot.Next()) { return false; diff --git a/ydb/core/mind/bscontroller/node_report.cpp b/ydb/core/mind/bscontroller/node_report.cpp index 868d8fb98822..ed1b116f1ca9 100644 --- a/ydb/core/mind/bscontroller/node_report.cpp +++ b/ydb/core/mind/bscontroller/node_report.cpp @@ -26,7 +26,7 @@ class TBlobStorageController::TTxNodeReport return true; } - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); State->CheckConsistency(); NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index 8dd84e535250..d0952a8bdf0b 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -160,7 +160,7 @@ class TBlobStorageController::TTxUpdateNodeDrives bool Execute(TTransactionContext& txc, const TActorContext&) override { const TNodeId nodeId = Record.GetNodeId(); - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); State->CheckConsistency(); auto updateIsSuccessful = true; diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp index c2f9f737de45..b9fed5ab6d1d 100644 --- a/ydb/core/mind/bscontroller/sys_view.cpp +++ b/ydb/core/mind/bscontroller/sys_view.cpp @@ -325,8 +325,8 @@ void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder status, NHPTimer::STime statusTimestamp, - NKikimrBlobStorage::TVDiskKind::EVDiskKind kind, bool isBeingDeleted) { + std::optional status, NKikimrBlobStorage::TVDiskKind::EVDiskKind kind, + bool isBeingDeleted) { pb->SetGroupId(vdiskId.GroupID.GetRawId()); pb->SetGroupGeneration(vdiskId.GroupGeneration); pb->SetFailRealm(vdiskId.FailRealm); @@ -338,9 +338,6 @@ void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, if (m.HasAvailableSize()) { pb->SetAvailableSize(m.GetAvailableSize()); } - if (!status && CyclesToDuration(GetCycleCountFast() - statusTimestamp) > TDuration::Seconds(15)) { - status = NKikimrBlobStorage::EVDiskStatus::ERROR; - } if (status) { pb->SetStatusV2(NKikimrBlobStorage::EVDiskStatus_Name(*status)); } @@ -352,7 +349,7 @@ void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, void CopyInfo(NKikimrSysView::TVSlotInfo* info, const THolder& vSlotInfo) { SerializeVSlotInfo(info, vSlotInfo->GetVDiskId(), vSlotInfo->Metrics, vSlotInfo->VDiskStatus, - vSlotInfo->VDiskStatusTimestamp, vSlotInfo->Kind, vSlotInfo->IsBeingDeleted()); + vSlotInfo->Kind, vSlotInfo->IsBeingDeleted()); } void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder& groupInfo) { @@ -428,6 +425,21 @@ void TBlobStorageController::UpdateSystemViews() { return; } + const TMonotonic now = TActivationContext::Monotonic(); + const TDuration expiration = TDuration::Seconds(15); + for (auto& [key, value] : VSlots) { + if (!value->VDiskStatus && value->VDiskStatusTimestamp + expiration <= now) { + value->VDiskStatus = NKikimrBlobStorage::ERROR; + SysViewChangedVSlots.insert(key); + } + } + for (auto& [key, value] : StaticVSlots) { + if (!value.VDiskStatus && value.VDiskStatusTimestamp + expiration <= now) { + value.VDiskStatus = NKikimrBlobStorage::ERROR; + SysViewChangedVSlots.insert(key); + } + } + if (!SysViewChangedPDisks.empty() || !SysViewChangedVSlots.empty() || !SysViewChangedGroups.empty() || !SysViewChangedStoragePools.empty() || SysViewChangedSettings) { auto update = MakeHolder(); @@ -468,7 +480,7 @@ void TBlobStorageController::UpdateSystemViews() { if (SysViewChangedVSlots.count(vslotId)) { static const NKikimrBlobStorage::TVDiskMetrics zero; SerializeVSlotInfo(&state.VSlots[vslotId], vslot.VDiskId, vslot.VDiskMetrics ? *vslot.VDiskMetrics : zero, - vslot.VDiskStatus, vslot.VDiskStatusTimestamp, vslot.VDiskKind, false); + vslot.VDiskStatus, vslot.VDiskKind, false); } } if (StorageConfig.HasBlobStorageConfig()) { diff --git a/ydb/core/mind/bscontroller/virtual_group.cpp b/ydb/core/mind/bscontroller/virtual_group.cpp index 66f3056e06ab..ee3b31fb2ac2 100644 --- a/ydb/core/mind/bscontroller/virtual_group.cpp +++ b/ydb/core/mind/bscontroller/virtual_group.cpp @@ -248,7 +248,7 @@ namespace NKikimr::NBsController { if (const TGroupInfo *group = Self->FindGroup(GroupId); !group || group->VirtualGroupSetupMachineId != MachineId) { return true; // another machine is already running } - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); TGroupInfo *group = State->Groups.FindForUpdate(GroupId); Y_ABORT_UNLESS(group); if (!Callback(*group, *State)) { @@ -294,7 +294,7 @@ namespace NKikimr::NBsController { if (Token.expired()) { return true; // actor is already dead } - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); const size_t n = State->BlobDepotDeleteQueue.Unshare().erase(GroupId); Y_ABORT_UNLESS(n == 1); TString error; @@ -897,7 +897,7 @@ namespace NKikimr::NBsController { TTxType GetTxType() const override { return NBlobStorageController::TXTYPE_DECOMMIT_GROUP; } bool Execute(TTransactionContext& txc, const TActorContext&) override { - State.emplace(*Self, Self->HostRecords, TActivationContext::Now()); + State.emplace(*Self, Self->HostRecords, TActivationContext::Now(), TActivationContext::Monotonic()); Action(*State); TString error; if (State->Changed() && !Self->CommitConfigUpdates(*State, true, true, true, txc, &error)) {