Skip to content

Commit

Permalink
refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
stanislav-shchetinin committed Oct 14, 2024
1 parent 9f222c6 commit f14d78e
Show file tree
Hide file tree
Showing 10 changed files with 46 additions and 40 deletions.
9 changes: 6 additions & 3 deletions ydb/core/driver_lib/run/run.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1734,14 +1734,17 @@ void TKikimrRunner::KikimrStart() {

void TKikimrRunner::KikimrStop(bool graceful) {

EnableDecomissionNode = AppData->FeatureFlags.GetEnableDecomissionNode();
bool enableReleaseNodeNameOnGracefulShutdown = AppData->FeatureFlags.GetEnableReleaseNodeNameOnGracefulShutdown();

if (graceful && EnableDecomissionNode) {
if (graceful && enableReleaseNodeNameOnGracefulShutdown) {
using namespace NKikimr::NNodeBroker;

NTabletPipe::TClientConfig pipeConfig;
pipeConfig.RetryPolicy = {.RetryLimitCount = 10};
auto pipe = NTabletPipe::CreateClient({}, MakeNodeBrokerID(), pipeConfig);
TActorId nodeBrokerPipe = ActorSystem->Register(pipe);
ActorSystem->Send(new IEventHandle(nodeBrokerPipe, {}, new NKikimr::NNodeBroker::TEvNodeBroker::TEvDecommissionRequest));

ActorSystem->Send(new IEventHandle(nodeBrokerPipe, {}, new TEvNodeBroker::TEvGracefulShutdownRequest));
}

if (EnabledGrpcService) {
Expand Down
1 change: 0 additions & 1 deletion ydb/core/driver_lib/run/run.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ class TKikimrRunner : public virtual TThrRefBase, private IGlobalObjectStorage {

bool EnabledGrpcService = false;
bool GracefulShutdownSupported = false;
bool EnabledDecommisionNode = false;

TDuration MinDelayBeforeShutdown;
THolder<NSQS::TAsyncHttpServer> SqsHttp;
Expand Down
10 changes: 5 additions & 5 deletions ydb/core/mind/node_broker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -844,7 +844,7 @@ void TNodeBroker::DbUpdateNodeLocation(const TNodeInfo &node,
db.Table<T>().Key(node.NodeId).Update<T::Location>(node.Location.GetSerializedLocation());
}

void TNodeBroker::DbUpdateSlotIndexToNull(const TNodeInfo &node,
void TNodeBroker::DbReleaseSlotIndex(const TNodeInfo &node,
TTransactionContext &txc)
{
NIceDb::TNiceDb db(txc.DB);
Expand Down Expand Up @@ -1011,12 +1011,12 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev,
ctx.RegisterWithSameMailbox(new TResolveTenantActor(ev, SelfId()));
}

void TNodeBroker::Handle(TEvNodeBroker::TEvDecommissionRequest::TPtr &ev,
void TNodeBroker::Handle(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev,
const TActorContext &ctx) {
LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvNodeBroker::TEvDecommissionRequest"
LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvNodeBroker::TEvGracefulShutdownRequest"
<< ": request# " << ev->Get()->Record.ShortDebugString());

ProcessTx(CreateTxDecommissionNode(ev), ctx);
TabletCounters->Cumulative()[COUNTER_GRACEFUL_SHUTDOWN_REQUESTS].Increment(1);
ProcessTx(CreateTxGracefulShutdown(ev), ctx);
}

void TNodeBroker::Handle(TEvNodeBroker::TEvExtendLeaseRequest::TPtr &ev,
Expand Down
18 changes: 10 additions & 8 deletions ydb/core/mind/node_broker.h
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,12 @@ struct TEvNodeBroker {
EvListNodes = EventSpaceBegin(TKikimrEvents::ES_NODE_BROKER),
EvResolveNode,
EvRegistrationRequest,
EvDecommissionRequest,
EvExtendLeaseRequest,

// responses
EvNodesInfo,
EvResolvedNode,
EvRegistrationResponse,
EvDecommissionResponse,
EvExtendLeaseResponse,

// config
Expand All @@ -97,6 +95,10 @@ struct TEvNodeBroker {
EvSetConfigRequest,
EvSetConfigResponse,

// decommission
EvGracefulShutdownRequest,
EvGracefulShutdownResponse,

// TODO: remove
// internal
//EvNodeExpire = EvListNodes + 512,
Expand Down Expand Up @@ -127,9 +129,9 @@ struct TEvNodeBroker {
EvRegistrationRequest> {
};

struct TEvDecommissionRequest : public TEventPB<TEvDecommissionRequest,
NKikimrNodeBroker::TDecommissionRequest,
EvDecommissionRequest> {
struct TEvGracefulShutdownRequest : public TEventPB<TEvGracefulShutdownRequest,
NKikimrNodeBroker::TGracefulShutdownRequest,
EvGracefulShutdownRequest> {
};

struct TEvExtendLeaseRequest : public TEventPB<TEvExtendLeaseRequest,
Expand Down Expand Up @@ -163,9 +165,9 @@ struct TEvNodeBroker {
EvRegistrationResponse> {
};

struct TEvDecommissionResponse : public TEventPB<TEvDecommissionResponse,
NKikimrNodeBroker::TDecommissionResponse,
EvDecommissionResponse> {
struct TEvGracefulShutdownResponse : public TEventPB<TEvGracefulShutdownResponse,
NKikimrNodeBroker::TGracefulShutdownResponse,
EvGracefulShutdownResponse> {
};

struct TEvExtendLeaseResponse : public TEventPB<TEvExtendLeaseResponse,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ namespace NKikimr::NNodeBroker {

using namespace NKikimrNodeBroker;

class TNodeBroker::TTxDecommissionNode : public TTransactionBase<TNodeBroker> {
class TNodeBroker::TTxGracefulShutdown : public TTransactionBase<TNodeBroker> {
public:
TTxDecommissionNode(TNodeBroker *self, TEvNodeBroker::TEvDecommissionRequest::TPtr &ev)
TTxGracefulShutdown(TNodeBroker *self, TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev)
: TBase(self)
, Event(ev)
{
}

TTxType GetTxType() const override { return TXTYPE_DECOMMISSION_NODE; }
TTxType GetTxType() const override { return TXTYPE_GRACESFUL_SHUTDOWN; }

bool Error(TStatus::ECode code,
const TString &reason,
Expand All @@ -26,7 +26,7 @@ class TNodeBroker::TTxDecommissionNode : public TTransactionBase<TNodeBroker> {
auto host = rec.GetHost();
auto port = rec.GetPort();
LOG_ERROR_S(ctx, NKikimrServices::NODE_BROKER,
"Cannot Decommission node " << host << ":" << port << ": " << code << ": " << reason);
"Cannot Graceful Shutdown " << host << ":" << port << ": " << code << ": " << reason);

Response->Record.MutableStatus()->SetCode(code);
Response->Record.MutableStatus()->SetReason(reason);
Expand All @@ -41,16 +41,18 @@ class TNodeBroker::TTxDecommissionNode : public TTransactionBase<TNodeBroker> {
ui16 port = (ui16)rec.GetPort();
TString addr = rec.GetAddress();

LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxDecommissionNode Execute");
LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxGracefulShutdown Execute");
LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER,
"Decommission request from " << host << ":" << port << " ");
"Graceful Shutdown request from " << host << ":" << port << " ");

Response = new TEvNodeBroker::TEvGracefulShutdownResponse;

auto it = Self->Hosts.find(std::make_tuple(host, addr, port));
if (it != Self->Hosts.end()) {
auto &node = Self->Nodes.find(it->second)->second;

Self->SlotIndexesPools[node.ServicedSubDomain].Release(node.SlotIndex.value());
Self->DbUpdateSlotIndexToNull(node, txc);
Self->DbReleaseSlotIndex(node, txc);

return true;
}
Expand All @@ -62,19 +64,18 @@ class TNodeBroker::TTxDecommissionNode : public TTransactionBase<TNodeBroker> {

void Complete(const TActorContext &ctx) override
{
LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxDecommissionNode Complete");

LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxGracefulShutdown Complete");
ctx.Send(Event->Sender, Response.Release());
}

private:
TEvNodeBroker::TEvDecommissionRequest::TPtr Event;
TAutoPtr<TEvNodeBroker::TEvDecommissionResponse> Response;
TEvNodeBroker::TEvGracefulShutdownRequest::TPtr Event;
TAutoPtr<TEvNodeBroker::TEvGracefulShutdownResponse> Response;
};

ITransaction *TNodeBroker::CreateTxDecommissionNode(TEvNodeBroker::TEvDecommissionRequest::TPtr &ev)
ITransaction *TNodeBroker::CreateTxGracefulShutdown(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev)
{
return new TTxDecommissionNode(this, ev);
return new TTxGracefulShutdown(this, ev);
}

} // NKikimr::NNodeBroker
10 changes: 5 additions & 5 deletions ydb/core/mind/node_broker_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ class TNodeBroker : public TActor<TNodeBroker>
class TTxInitScheme;
class TTxLoadState;
class TTxRegisterNode;
class TTxDecommissionNode;
class TTxGracefulShutdown;
class TTxUpdateConfig;
class TTxUpdateConfigSubscription;
class TTxUpdateEpoch;
Expand All @@ -157,7 +157,7 @@ class TNodeBroker : public TActor<TNodeBroker>
ITransaction *CreateTxInitScheme();
ITransaction *CreateTxLoadState();
ITransaction *CreateTxRegisterNode(TEvPrivate::TEvResolvedRegistrationRequest::TPtr &ev);
ITransaction *CreateTxDecommissionNode(TEvNodeBroker::TEvDecommissionRequest::TPtr &ev);
ITransaction *CreateTxGracefulShutdown(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev);
ITransaction *CreateTxUpdateConfig(TEvConsole::TEvConfigNotificationRequest::TPtr &ev);
ITransaction *CreateTxUpdateConfig(TEvNodeBroker::TEvSetConfigRequest::TPtr &ev);
ITransaction *CreateTxUpdateConfigSubscription(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev);
Expand Down Expand Up @@ -204,7 +204,7 @@ class TNodeBroker : public TActor<TNodeBroker>
HFuncTraced(TEvNodeBroker::TEvListNodes, Handle);
HFuncTraced(TEvNodeBroker::TEvResolveNode, Handle);
HFuncTraced(TEvNodeBroker::TEvRegistrationRequest, Handle);
HFuncTraced(TEvNodeBroker::TEvDecommissionRequest, Handle);
HFuncTraced(TEvNodeBroker::TEvGracefulShutdownRequest, Handle);
HFuncTraced(TEvNodeBroker::TEvExtendLeaseRequest, Handle);
HFuncTraced(TEvNodeBroker::TEvCompactTables, Handle);
HFuncTraced(TEvNodeBroker::TEvGetConfigRequest, Handle);
Expand Down Expand Up @@ -291,7 +291,7 @@ class TNodeBroker : public TActor<TNodeBroker>
TTransactionContext &txc);
void DbUpdateNodeLocation(const TNodeInfo &node,
TTransactionContext &txc);
void DbUpdateSlotIndexToNull(const TNodeInfo &node,
void DbReleaseSlotIndex(const TNodeInfo &node,
TTransactionContext &txc);

void Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev,
Expand All @@ -304,7 +304,7 @@ class TNodeBroker : public TActor<TNodeBroker>
const TActorContext &ctx);
void Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev,
const TActorContext &ctx);
void Handle(TEvNodeBroker::TEvDecommissionRequest::TPtr &ev,
void Handle(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev,
const TActorContext &ctx);
void Handle(TEvNodeBroker::TEvExtendLeaseRequest::TPtr &ev,
const TActorContext &ctx);
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/mind/ya.make
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ SRCS(
lease_holder.h
local.cpp
local.h
node_broker__decommission_node.cpp
node_broker__graceful_shutdown.cpp
node_broker.cpp
node_broker.h
node_broker_impl.h
Expand Down
3 changes: 2 additions & 1 deletion ydb/core/protos/counters_node_broker.proto
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ enum ECumulativeCounters {
COUNTER_RESOLVE_NODE_REQUESTS = 1 [(CounterOpts) = {Name: "ResolveNodeRequests"}];
COUNTER_REGISTRATION_REQUESTS = 2 [(CounterOpts) = {Name: "RegistrationRequests"}];
COUNTER_EXTEND_LEASE_REQUESTS = 3 [(CounterOpts) = {Name: "ExtendLeaseRequests"}];
COUNTER_GRACEFUL_SHUTDOWN_REQUESTS = 4 [(CounterOpts) = {Name: "ExtendLeaseRequests"}];
}

enum EPercentileCounters {
Expand Down Expand Up @@ -52,5 +53,5 @@ enum ETxTypes {
TXTYPE_UPDATE_CONFIG = 4 [(TxTypeOpts) = {Name: "TTxUpdateConfig"}];
TXTYPE_UPDATE_CONFIG_SUBSCRIPTION = 5 [(TxTypeOpts) = {Name: "TTxUpdateConfigSubscription"}];
TXTYPE_UPDATE_EPOCH = 6 [(TxTypeOpts) = {Name: "TTxUpdateEpoch"}];
TXTYPE_DECOMMISSION_NODE = 7 [(TxTypeOpts) = {Name: "TTxDecommissionNode"}];
TXTYPE_GRACESFUL_SHUTDOWN = 7 [(TxTypeOpts) = {Name: "TTxGracefulShutdown"}];
}
2 changes: 1 addition & 1 deletion ydb/core/protos/feature_flags.proto
Original file line number Diff line number Diff line change
Expand Up @@ -166,5 +166,5 @@ message TFeatureFlags {
optional bool EnableInsertWriteIdSpecialColumnCompatibility = 147 [default = false];
optional bool EnableTopicAutopartitioningForCDC = 148 [default = false];
optional bool EnableWritePortionsOnInsert = 149 [default = false];
optional bool EnableDecommissionNode = 150 [default = false];
optional bool EnableReleaseNodeNameOnGracefulShutdown = 150 [default = false];
}
4 changes: 2 additions & 2 deletions ydb/core/protos/node_broker.proto
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,13 @@ message TRegistrationResponse {
optional uint64 ScopePathId = 4;
}

message TDecommissionRequest {
message TGracefulShutdownRequest {
optional string Host = 1;
optional uint32 Port = 2;
optional string Address = 3;
}

message TDecommissionResponse {
message TGracefulShutdownResponse {
optional TStatus Status = 1;
}

Expand Down

0 comments on commit f14d78e

Please sign in to comment.