Skip to content

Commit

Permalink
YDB FQ: turning gateways_config.proto into a file without external de…
Browse files Browse the repository at this point in the history 
…pendencies
  • Loading branch information
@vitalyisaev2
vitalyisaev2 committed Dec 16, 2024
1 parent b55e608 commit 0c4f6d4
Show file tree
Hide file tree
Showing 50 changed files with 225 additions and 362 deletions.
40 changes: 20 additions & 20 deletions ydb/core/fq/libs/actors/clusters_from_connections.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include "clusters_from_connections.h"

#include <yql/essentials/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h>
#include <yql/essentials/providers/common/proto/gateways_config.pb.h>
#include <ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.h>
#include <yql/essentials/utils/url_builder.h>
#include <ydb/library/actors/http/http.h>
Expand Down Expand Up @@ -112,7 +112,7 @@ void FillGenericClusterConfigBase(
TGenericClusterConfig& clusterCfg,
const TConnection& connection,
const TString& connectionName,
NConnector::NApi::EDataSourceKind dataSourceKind,
NYql::EGenericDataSourceKind dataSourceKind,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures
) {
Expand All @@ -128,21 +128,21 @@ void FillGenericClusterConfigBase(
// In YQv1 we just hardcode the appropriate protocols here.
// In YQv2 protocol can be configured via `CREATE EXTERNAL DATA SOURCE` params.
switch (dataSourceKind) {
case NYql::NConnector::NApi::CLICKHOUSE:
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::NConnector::NApi::EProtocol::NATIVE : NYql::NConnector::NApi::EProtocol::HTTP);
case NYql::EGenericDataSourceKind::CLICKHOUSE:
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::EGenericProtocol::NATIVE : NYql::EGenericProtocol::HTTP);
break;
case NYql::NConnector::NApi::GREENPLUM:
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
case NYql::EGenericDataSourceKind::GREENPLUM:
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
break;
case NYql::NConnector::NApi::MYSQL:
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
case NYql::EGenericDataSourceKind::MYSQL:
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
break;
case NYql::NConnector::NApi::POSTGRESQL:
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
case NYql::EGenericDataSourceKind::POSTGRESQL:
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
break;
default:
ythrow yexception() << "Unexpected data source kind: '"
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
<< NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'";
}

ValidateGenericClusterConfig(clusterCfg, "NFq::FillGenericClusterFromConfig");
Expand All @@ -154,7 +154,7 @@ void FillGenericClusterConfig(
TGenericClusterConfig& clusterCfg,
const TConnection& connection,
const TString& connectionName,
NConnector::NApi::EDataSourceKind dataSourceKind,
NYql::EGenericDataSourceKind dataSourceKind,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures
) {
Expand All @@ -167,7 +167,7 @@ void FillGenericClusterConfig<FederatedQuery::PostgreSQLCluster>(
TGenericClusterConfig& clusterCfg,
const FederatedQuery::PostgreSQLCluster& connection,
const TString& connectionName,
NConnector::NApi::EDataSourceKind dataSourceKind,
NYql::EGenericDataSourceKind dataSourceKind,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures
){
Expand Down Expand Up @@ -227,8 +227,8 @@ void AddClustersFromConnections(
case FederatedQuery::ConnectionSetting::kYdbDatabase: {
const auto& db = conn.content().setting().ydb_database();
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::YDB);
clusterCfg->SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
clusterCfg->SetKind(NYql::EGenericDataSourceKind::YDB);
clusterCfg->SetProtocol(NYql::EGenericProtocol::NATIVE);
clusterCfg->SetName(connectionName);
clusterCfg->SetDatabaseId(db.database_id());
clusterCfg->SetUseSsl(!common.GetDisableSslForGenericDataSources());
Expand All @@ -242,7 +242,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().clickhouse_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::CLICKHOUSE,
NYql::EGenericDataSourceKind::CLICKHOUSE,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand Down Expand Up @@ -275,7 +275,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().postgresql_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::POSTGRESQL,
NYql::EGenericDataSourceKind::POSTGRESQL,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand All @@ -287,7 +287,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().greenplum_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::GREENPLUM,
NYql::EGenericDataSourceKind::GREENPLUM,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand All @@ -299,7 +299,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().mysql_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::MYSQL,
NYql::EGenericDataSourceKind::MYSQL,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand All @@ -308,7 +308,7 @@ void AddClustersFromConnections(
case FederatedQuery::ConnectionSetting::kLogging: {
const auto& connection = conn.content().setting().logging();
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::LOGGING);
clusterCfg->SetKind(NYql::EGenericDataSourceKind::LOGGING);
clusterCfg->SetName(connectionName);
clusterCfg->mutable_datasourceoptions()->insert({"folder_id", connection.folder_id()});
FillClusterAuth(*clusterCfg, connection.auth(), authToken, accountIdSignatures);
Expand Down
14 changes: 7 additions & 7 deletions ydb/core/fq/libs/actors/database_resolver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ using TParser = std::function<TDatabaseDescription(
NJson::TJsonValue& body,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
)>;
using TParsers = THashMap<NYql::EDatabaseType, TParser>;

Expand Down Expand Up @@ -292,7 +292,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
.SetErrorTtl(TDuration::Minutes(1))
.SetMaxSize(1000000))
{
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) {
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NYql::EGenericProtocol) {
bool secure = false;
TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust();
TString prefix("/?database=");
Expand Down Expand Up @@ -333,7 +333,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol)
NYql::EGenericProtocol protocol)
{
auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol);
// TODO: Take explicit field from MVP
Expand All @@ -349,7 +349,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
Expand Down Expand Up @@ -380,7 +380,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
Expand Down Expand Up @@ -427,7 +427,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TString aliveHost;
Expand Down Expand Up @@ -465,7 +465,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
Expand Down
34 changes: 17 additions & 17 deletions ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

void Test(
NYql::EDatabaseType databaseType,
NYql::NConnector::NApi::EProtocol protocol,
NYql::EGenericProtocol protocol,
const TString& getUrl,
const TString& status,
const TString& responseBody,
Expand Down Expand Up @@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(Ydb_Serverless) {
Test(
NYql::EDatabaseType::Ydb,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand Down Expand Up @@ -218,7 +218,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::Ydb,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"",
"",
Expand All @@ -237,7 +237,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(Ydb_Dedicated) {
Test(
NYql::EDatabaseType::Ydb,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand All @@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(DataStreams_Serverless) {
Test(
NYql::EDatabaseType::DataStreams,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand All @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(DataStreams_Dedicated) {
Test(
NYql::EDatabaseType::DataStreams,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand All @@ -302,7 +302,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(ClickHouseNative) {
Test(
NYql::EDatabaseType::ClickHouse,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -336,7 +336,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(ClickHouseHttp) {
Test(
NYql::EDatabaseType::ClickHouse,
NYql::NConnector::NApi::EProtocol::HTTP,
NYql::EGenericProtocol::HTTP,
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -381,7 +381,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::ClickHouse,
NYql::NConnector::NApi::EProtocol::HTTP,
NYql::EGenericProtocol::HTTP,
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"403",
R"(
Expand All @@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(PostgreSQL) {
Test(
NYql::EDatabaseType::PostgreSQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -454,7 +454,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::PostgreSQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"403",
R"(
Expand All @@ -478,7 +478,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(Greenplum_MasterNode) {
Test(
NYql::EDatabaseType::Greenplum,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
"200",
R"({
Expand Down Expand Up @@ -520,7 +520,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::Greenplum,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
"403",
R"(
Expand All @@ -542,7 +542,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(MySQL) {
Test(
NYql::EDatabaseType::MySQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -590,7 +590,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::MySQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"403",
R"(
Expand Down Expand Up @@ -624,7 +624,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
};
Test(
NYql::EDatabaseType::DataStreams,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"403",
R"(
Expand All @@ -642,7 +642,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

NYql::TDatabaseAuth databaseAuth;
databaseAuth.UseTls = true;
databaseAuth.Protocol = NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED;
databaseAuth.Protocol = NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED;

TString databaseId1{"etn021us5r9rhld1vgb1"};
TString databaseId2{"etn021us5r9rhld1vgb2"};
Expand Down
1 change: 0 additions & 1 deletion ydb/core/fq/libs/actors/ya.make
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ PEERDIR(
ydb/library/yql/providers/dq/provider
ydb/library/yql/providers/dq/provider/exec
ydb/library/yql/providers/dq/worker_manager/interface
ydb/library/yql/providers/generic/connector/api/common
ydb/library/yql/providers/generic/connector/libcpp
ydb/library/yql/providers/generic/provider
ydb/library/yql/providers/pq/cm_client
Expand Down
Loading

0 comments on commit 0c4f6d4

Please sign in to comment.