Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@Enjection
Enjection committed Feb 20, 2024
1 parent ce282a8 commit 13b3706
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 84 deletions.
80 changes: 80 additions & 0 deletions ydb/core/config/init/init.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,82 @@ class TDefaultNodeBrokerClient
}
};

class TDefaultDynConfigClient
: public IDynConfigClient
{
static bool TryToLoadConfigForDynamicNodeFromCMS(
const TGrpcSslSettings& gs,
const TString &addr,
const TDynConfigSettings& settings,
const IEnv& env,
TMaybe<NKikimr::NClient::TConfigurationResult>& res,
TString &error)
{
NClient::TKikimr kikimr(GetKikimr(
gs,
addr,
env));
auto configurator = kikimr.GetNodeConfigurator();

Cout << "Trying to get configs from " << addr << Endl;

auto result = configurator.SyncGetNodeConfig(settings.NodeId,
settings.FQDNHostName,
settings.TenantName,
settings.NodeType,
settings.DomainName,
settings.StaffApiUserToken,
true,
1);

if (!result.IsSuccess()) {
error = result.GetErrorMessage();
Cerr << "Configuration error: " << error << Endl;
return false;
}

Cout << "Success." << Endl;

res = result;

return true;
}
public:
TMaybe<NKikimr::NClient::TConfigurationResult> GetConfig(
const TGrpcSslSettings& gs,
const TVector<TString>& addrs,
const TDynConfigSettings& settings,
const IEnv& env) const override
{
TMaybe<NKikimr::NClient::TConfigurationResult> res;
bool success = false;
TString error;

SetRandomSeed(TInstant::Now().MicroSeconds());
int minAttempts = 10;
int attempts = 0;
while (!success && attempts < minAttempts) {
for (auto addr : addrs) {
success = TryToLoadConfigForDynamicNodeFromCMS(gs, addr, settings, env, res, error);
++attempts;
if (success) {
break;
}
}
// Randomized backoff
if (!success) {
env.Sleep(TDuration::MilliSeconds(500 + RandomNumber<ui64>(1000)));
}
}

if (!success) {
Cerr << "WARNING: couldn't load config from CMS: " << error << Endl;
}

return res;
}
};

std::unique_ptr<IEnv> MakeDefaultEnv() {
return std::make_unique<TDefaultEnv>();
}
Expand All @@ -418,6 +494,10 @@ std::unique_ptr<INodeBrokerClient> MakeDefaultNodeBrokerClient() {
return std::make_unique<TDefaultNodeBrokerClient>();
}

std::unique_ptr<IDynConfigClient> MakeDefaultDynConfigClient() {
return std::make_unique<TDefaultDynConfigClient>();
}

void CopyNodeLocation(NActorsInterconnect::TNodeLocation* dst, const NYdb::NDiscovery::TNodeLocation& src) {
if (src.DataCenterNum) {
dst->SetDataCenterNum(src.DataCenterNum.value());
Expand Down
4 changes: 4 additions & 0 deletions ydb/core/config/init/init.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,15 @@ std::unique_ptr<IEnv> MakeDefaultEnv();
std::unique_ptr<IErrorCollector> MakeDefaultErrorCollector();
std::unique_ptr<IMemLogInitializer> MakeDefaultMemLogInitializer();
std::unique_ptr<INodeBrokerClient> MakeDefaultNodeBrokerClient();
std::unique_ptr<IDynConfigClient> MakeDefaultDynConfigClient();

std::unique_ptr<IInitialConfigurator> MakeDefaultInitialConfigurator(
NConfig::IErrorCollector& errorCollector,
NConfig::IProtoConfigFileProvider& protoConfigFileProvider,
NConfig::IConfigUpdateTracer& configUpdateTracer,
NConfig::IMemLogInitializer& memLogInit,
NConfig::INodeBrokerClient& nodeBrokerClient,
NConfig::IDynConfigClient& DynConfigClient,
NConfig::IEnv& env);

class TInitialConfigurator {
Expand All @@ -146,13 +148,15 @@ class TInitialConfigurator {
NConfig::IConfigUpdateTracer& configUpdateTracer,
NConfig::IMemLogInitializer& memLogInit,
NConfig::INodeBrokerClient& nodeBrokerClient,
NConfig::IDynConfigClient& dynConfigClient,
NConfig::IEnv& env)
: Impl(MakeDefaultInitialConfigurator(
errorCollector,
protoConfigFileProvider,
configUpdateTracer,
memLogInit,
nodeBrokerClient,
dynConfigClient,
env))
{}

Expand Down
97 changes: 13 additions & 84 deletions ydb/core/config/init/init_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
#include <ydb/library/actors/core/log_iface.h>
#include <ydb/library/yaml_config/yaml_config.h>
#include <ydb/library/yaml_config/yaml_config_parser.h>
#include <ydb/public/lib/deprecated/kicli/kicli.h>
#include <ydb/public/lib/ydb_cli/common/common.h>
#include <ydb/public/sdk/cpp/client/ydb_discovery/discovery.h>
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
Expand Down Expand Up @@ -899,6 +898,7 @@ class TInitialConfiguratorImpl
NConfig::IConfigUpdateTracer& ConfigUpdateTracer;
NConfig::IMemLogInitializer& MemLogInit;
NConfig::INodeBrokerClient& NodeBrokerClient;
NConfig::IDynConfigClient& DynConfigClient;
NConfig::IEnv& Env;

public:
Expand All @@ -908,12 +908,14 @@ class TInitialConfiguratorImpl
NConfig::IConfigUpdateTracer& configUpdateTracer,
NConfig::IMemLogInitializer& memLogInit,
NConfig::INodeBrokerClient& nodeBrokerClient,
NConfig::IDynConfigClient& dynConfigClient,
NConfig::IEnv& env)
: ErrorCollector(errorCollector)
, ProtoConfigFileProvider(protoConfigFileProvider)
, ConfigUpdateTracer(configUpdateTracer)
, MemLogInit(memLogInit)
, NodeBrokerClient(nodeBrokerClient)
, DynConfigClient(dynConfigClient)
, Env(env)
{}

Expand Down Expand Up @@ -1158,87 +1160,6 @@ class TInitialConfiguratorImpl
}
}

struct TDynConfigSettings {
ui32 NodeId;
TString DomainName;
TString TenantName;
TString FQDNHostName;
TString NodeType;
TString StaffApiUserToken;
};

bool TryToLoadConfigForDynamicNodeFromCMS(
const TGrpcSslSettings& gs,
const TString &addr,
const TDynConfigSettings& settings,
const IEnv& env,
TMaybe<NKikimr::NClient::TConfigurationResult>& res,
TString &error) const
{
NClient::TKikimr kikimr(GetKikimr(
gs,
addr,
env));
auto configurator = kikimr.GetNodeConfigurator();

Cout << "Trying to get configs from " << addr << Endl;

auto result = configurator.SyncGetNodeConfig(settings.NodeId,
settings.FQDNHostName,
settings.TenantName,
settings.NodeType,
settings.DomainName,
settings.StaffApiUserToken,
true,
1);

if (!result.IsSuccess()) {
error = result.GetErrorMessage();
Cerr << "Configuration error: " << error << Endl;
return false;
}

Cout << "Success." << Endl;

res = result;

return true;
}

TMaybe<NKikimr::NClient::TConfigurationResult> LoadConfigForDynamicNode(
const TGrpcSslSettings& gs,
const TVector<TString>& addrs,
const TDynConfigSettings& settings,
const IEnv& env) const
{
TMaybe<NKikimr::NClient::TConfigurationResult> res;
bool success = false;
TString error;

SetRandomSeed(TInstant::Now().MicroSeconds());
int minAttempts = 10;
int attempts = 0;
while (!success && attempts < minAttempts) {
for (auto addr : addrs) {
success = TryToLoadConfigForDynamicNodeFromCMS(gs, addr, settings, env, res, error);
++attempts;
if (success) {
break;
}
}
// Randomized backoff
if (!success) {
env.Sleep(TDuration::MilliSeconds(500 + RandomNumber<ui64>(1000)));
}
}

if (!success) {
Cerr << "WARNING: couldn't load config from CMS: " << error << Endl;
}

return res;
}

void InitStaticNode() {
ConfigFields.ValidateStaticNodeConfig();

Expand Down Expand Up @@ -1268,7 +1189,7 @@ class TInitialConfiguratorImpl
AppConfig.GetAuthConfig().GetStaffApiUserToken(),
};

TMaybe<NKikimr::NClient::TConfigurationResult> result = LoadConfigForDynamicNode(ConfigFields.GrpcSslSettings, addrs, settings, Env);
TMaybe<NKikimr::NClient::TConfigurationResult> result = DynConfigClient.GetConfig(ConfigFields.GrpcSslSettings, addrs, settings, Env);

if (!result) {
return;
Expand Down Expand Up @@ -1304,9 +1225,17 @@ std::unique_ptr<IInitialConfigurator> MakeDefaultInitialConfigurator(
NConfig::IConfigUpdateTracer& configUpdateTracer,
NConfig::IMemLogInitializer& memLogInit,
NConfig::INodeBrokerClient& nodeBrokerClient,
NConfig::IDynConfigClient& dynConfigClient,
NConfig::IEnv& env)
{
return std::make_unique<TInitialConfiguratorImpl>(errorCollector, protoConfigFileProvider, configUpdateTracer, memLogInit, nodeBrokerClient, env);
return std::make_unique<TInitialConfiguratorImpl>(
errorCollector,
protoConfigFileProvider,
configUpdateTracer,
memLogInit,
nodeBrokerClient,
dynConfigClient,
env);
}

} // namespace NKikimr::NConfig
3 changes: 3 additions & 0 deletions ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@ class TClientCommandServer : public TClientCommand {
, ConfigUpdateTracer(NConfig::MakeDefaultConfigUpdateTracer())
, MemLogInit(NConfig::MakeDefaultMemLogInitializer())
, NodeBrokerClient(NConfig::MakeDefaultNodeBrokerClient())
, DynConfigClient(NConfig::MakeDefaultDynConfigClient())
, Env(NConfig::MakeDefaultEnv())
, InitCfg(
*ErrorCollector,
*ProtoConfigFileProvider,
*ConfigUpdateTracer,
*MemLogInit,
*NodeBrokerClient,
*DynConfigClient,
*Env)
{}

Expand All @@ -44,6 +46,7 @@ class TClientCommandServer : public TClientCommand {
std::unique_ptr<NConfig::IConfigUpdateTracer> ConfigUpdateTracer;
std::unique_ptr<NConfig::IMemLogInitializer> MemLogInit;
std::unique_ptr<NConfig::INodeBrokerClient> NodeBrokerClient;
std::unique_ptr<NConfig::IDynConfigClient> DynConfigClient;
std::unique_ptr<NConfig::IEnv> Env;

NConfig::TInitialConfigurator InitCfg;
Expand Down

0 comments on commit 13b3706

Please sign in to comment.