Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@Enjection
Enjection committed Feb 20, 2024
1 parent 0f04b8a commit ce282a8
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 48 deletions.
24 changes: 23 additions & 1 deletion ydb/core/config/init/init.h
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
#pragma once

#include <ydb/core/base/event_filter.h>
#include <ydb/core/cms/console/config_item_info.h>
#include <ydb/core/protos/config.pb.h>
#include <ydb/library/actors/core/interconnect.h>
#include <ydb/core/base/event_filter.h>
#include <ydb/public/lib/deprecated/kicli/kicli.h>

#include <library/cpp/getopt/small/last_getopt_opts.h>

Expand Down Expand Up @@ -93,6 +94,27 @@ class INodeBrokerClient {

// ===

struct TDynConfigSettings {
ui32 NodeId;
TString DomainName;
TString TenantName;
TString FQDNHostName;
TString NodeType;
TString StaffApiUserToken;
};

class IDynConfigClient {
public:
virtual ~IDynConfigClient() {}
virtual TMaybe<NKikimr::NClient::TConfigurationResult> GetConfig(
const TGrpcSslSettings& gs,
const TVector<TString>& addrs,
const TDynConfigSettings& settings,
const IEnv& env) const = 0;
};

// ===

class IInitialConfigurator {
public:
virtual ~IInitialConfigurator() {};
Expand Down
117 changes: 70 additions & 47 deletions ydb/core/config/init/init_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ struct TConfigFields {
return nodeId;
}

NActors::TNodeLocation CreateNodeLocation() const { // FIXME
NActors::TNodeLocation CreateNodeLocation() const {
NActorsInterconnect::TNodeLocation location;
location.SetDataCenter(DataCenter ? DataCenter.GetRef() : TString(""));
location.SetRack(Rack);
Expand All @@ -565,7 +565,7 @@ struct TConfigFields {
return loc;
}

void ApplyLogSettings(NKikimrConfig::TAppConfig& appConfig, IConfigUpdateTracer& ConfigUpdateTracer) {
void ApplyLogSettings(NKikimrConfig::TAppConfig& appConfig, IConfigUpdateTracer& ConfigUpdateTracer) const {
if (SysLogServiceTag && !appConfig.GetLogConfig().GetSysLogService()) {
appConfig.MutableLogConfig()->SetSysLogService(SysLogServiceTag.GetRef());
TRACE_CONFIG_CHANGE_INPLACE_T(LogConfig, UpdateExplicitly);
Expand Down Expand Up @@ -727,7 +727,7 @@ struct TMbusConfigFields {
}
}

void InitMessageBusConfig(NKikimrConfig::TAppConfig& appConfig) {
void InitMessageBusConfig(NKikimrConfig::TAppConfig& appConfig) const {
auto messageBusConfig = appConfig.MutableMessageBusConfig();
messageBusConfig->SetStartBusProxy(Start);
messageBusConfig->SetBusProxyPort(BusProxyPort);
Expand All @@ -738,7 +738,6 @@ struct TMbusConfigFields {

auto sessionConfig = messageBusConfig->MutableProxyBusSessionConfig();

// TODO use macro from messagebus header file
sessionConfig->SetName(ProxyBusSessionConfig.Name);
sessionConfig->SetNumRetries(ProxyBusSessionConfig.NumRetries);
sessionConfig->SetRetryInterval(ProxyBusSessionConfig.RetryInterval);
Expand Down Expand Up @@ -774,7 +773,31 @@ struct TMbusConfigFields {

// =====

static ui32 NextValidKind(ui32 kind) {
TString DeduceNodeDomain(const NConfig::TConfigFields& cf, const NKikimrConfig::TAppConfig& appConfig) {
if (cf.NodeDomain) {
return cf.NodeDomain;
}

if (appConfig.GetDomainsConfig().DomainSize() == 1) {
return appConfig.GetDomainsConfig().GetDomain(0).GetName();
}

if (appConfig.GetTenantPoolConfig().SlotsSize() == 1) {
auto &slot = appConfig.GetTenantPoolConfig().GetSlots(0);
if (slot.GetDomainName()) {
return slot.GetDomainName();
}

auto &tenantName = slot.GetTenantName();
if (IsStartWithSlash(tenantName)) {
return ToString(ExtractDomain(tenantName));
}
}

return "";
}

ui32 NextValidKind(ui32 kind) {
do {
++kind;
if (kind != NKikimrConsole::TConfigItem::Auto && NKikimrConsole::TConfigItem::EKind_IsValid(kind)) {
Expand All @@ -784,7 +807,7 @@ static ui32 NextValidKind(ui32 kind) {
return kind;
}

static bool HasCorrespondingManagedKind(ui32 kind, const NKikimrConfig::TAppConfig& appConfig) {
bool HasCorrespondingManagedKind(ui32 kind, const NKikimrConfig::TAppConfig& appConfig) {
return (kind == NKikimrConsole::TConfigItem::NameserviceConfigItem && appConfig.HasNameserviceConfig()) ||
(kind == NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem && appConfig.HasNetClassifierDistributableConfig()) ||
(kind == NKikimrConsole::TConfigItem::NamedConfigsItem && appConfig.NamedConfigsSize());
Expand Down Expand Up @@ -987,7 +1010,7 @@ class TInitialConfiguratorImpl

void FillData(const NConfig::TConfigFields& cf) {
if (cf.TenantName && ScopeId.IsEmpty()) {
const TString myDomain = DeduceNodeDomain(cf);
const TString myDomain = DeduceNodeDomain(cf, AppConfig);
for (const auto& domain : AppConfig.GetDomainsConfig().GetDomain()) {
if (domain.GetName() == myDomain) {
ScopeId = TKikimrScopeId(0, domain.GetDomainId());
Expand All @@ -1011,7 +1034,7 @@ class TInitialConfiguratorImpl
auto &slot = *AppConfig.MutableTenantPoolConfig()->AddSlots();
slot.SetId("static-slot");
slot.SetIsDynamic(false);
TString tenantName = cf.TenantName ? cf.TenantName.GetRef() : CanonizePath(DeduceNodeDomain(cf));
TString tenantName = cf.TenantName ? cf.TenantName.GetRef() : CanonizePath(DeduceNodeDomain(cf, AppConfig));
slot.SetTenantName(tenantName);
return tenantName;
}
Expand Down Expand Up @@ -1082,26 +1105,6 @@ class TInitialConfiguratorImpl
ConfigFields.SetupInterconnectConfigDefaults(icConfig, ConfigUpdateTracer);
};

TString DeduceNodeDomain(const NConfig::TConfigFields& cf) const {
if (cf.NodeDomain) {
return cf.NodeDomain;
}
if (AppConfig.GetDomainsConfig().DomainSize() == 1) {
return AppConfig.GetDomainsConfig().GetDomain(0).GetName();
}
if (AppConfig.GetTenantPoolConfig().SlotsSize() == 1) {
auto &slot = AppConfig.GetTenantPoolConfig().GetSlots(0);
if (slot.GetDomainName()) {
return slot.GetDomainName();
}
auto &tenantName = slot.GetTenantName();
if (IsStartWithSlash(tenantName)) {
return ToString(ExtractDomain(tenantName));
}
}
return "";
}

void RegisterDynamicNode(NConfig::TConfigFields& cf) {
TVector<TString> addrs;

Expand All @@ -1115,7 +1118,7 @@ class TInitialConfiguratorImpl
ythrow yexception() << "List of Node Broker end-points is empty";
}

TString domainName = DeduceNodeDomain(cf);
TString domainName = DeduceNodeDomain(cf, AppConfig);

if (!cf.NodeHost) {
cf.NodeHost = Env.FQDNHostName();
Expand Down Expand Up @@ -1155,11 +1158,19 @@ class TInitialConfiguratorImpl
}
}

struct TDynConfigSettings {
ui32 NodeId;
TString DomainName;
TString TenantName;
TString FQDNHostName;
TString NodeType;
TString StaffApiUserToken;
};

bool TryToLoadConfigForDynamicNodeFromCMS(
const NConfig::TConfigFields& cf,
const TString &addr,
const TString &domainName,
const TGrpcSslSettings& gs,
const TString &addr,
const TDynConfigSettings& settings,
const IEnv& env,
TMaybe<NKikimr::NClient::TConfigurationResult>& res,
TString &error) const
Expand All @@ -1172,12 +1183,12 @@ class TInitialConfiguratorImpl

Cout << "Trying to get configs from " << addr << Endl;

auto result = configurator.SyncGetNodeConfig(NodeId,
env.FQDNHostName(),
cf.TenantName.GetRef(),
cf.NodeType.GetRef(),
domainName,
AppConfig.GetAuthConfig().GetStaffApiUserToken(),
auto result = configurator.SyncGetNodeConfig(settings.NodeId,
settings.FQDNHostName,
settings.TenantName,
settings.NodeType,
settings.DomainName,
settings.StaffApiUserToken,
true,
1);

Expand All @@ -1194,30 +1205,30 @@ class TInitialConfiguratorImpl
return true;
}

TMaybe<NKikimr::NClient::TConfigurationResult> LoadConfigForDynamicNode(const NConfig::TConfigFields& cf) const {
TMaybe<NKikimr::NClient::TConfigurationResult> LoadConfigForDynamicNode(
const TGrpcSslSettings& gs,
const TVector<TString>& addrs,
const TDynConfigSettings& settings,
const IEnv& env) const
{
TMaybe<NKikimr::NClient::TConfigurationResult> res;
bool success = false;
TString error;

TVector<TString> addrs;
cf.FillClusterEndpoints(AppConfig, addrs);

const TString domainName = DeduceNodeDomain(cf);

SetRandomSeed(TInstant::Now().MicroSeconds());
int minAttempts = 10;
int attempts = 0;
while (!success && attempts < minAttempts) {
for (auto addr : addrs) {
success = TryToLoadConfigForDynamicNodeFromCMS(cf, addr, domainName, cf.GrpcSslSettings, Env, res, error);
success = TryToLoadConfigForDynamicNodeFromCMS(gs, addr, settings, env, res, error);
++attempts;
if (success) {
break;
}
}
// Randomized backoff
if (!success) {
Env.Sleep(TDuration::MilliSeconds(500 + RandomNumber<ui64>(1000)));
env.Sleep(TDuration::MilliSeconds(500 + RandomNumber<ui64>(1000)));
}
}

Expand Down Expand Up @@ -1245,7 +1256,19 @@ class TInitialConfiguratorImpl
return;
}

TMaybe<NKikimr::NClient::TConfigurationResult> result = LoadConfigForDynamicNode(ConfigFields);
TVector<TString> addrs;
ConfigFields.FillClusterEndpoints(AppConfig, addrs);

TDynConfigSettings settings {
NodeId,
DeduceNodeDomain(ConfigFields, AppConfig),
ConfigFields.TenantName.GetRef(),
Env.FQDNHostName(),
ConfigFields.NodeType.GetRef(),
AppConfig.GetAuthConfig().GetStaffApiUserToken(),
};

TMaybe<NKikimr::NClient::TConfigurationResult> result = LoadConfigForDynamicNode(ConfigFields.GrpcSslSettings, addrs, settings, Env);

if (!result) {
return;
Expand Down

0 comments on commit ce282a8

Please sign in to comment.