Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge from main #12612

Merged
merged 7 commits into from
Dec 17, 2024
Merged

Merge from main #12612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fb0afe4
YDB FQ: turning gateways_config.proto into a file without external de…
vitalyisaev2 Dec 11, 2024
df2595b
YDB FQ: turning gateways_config.proto into a file without external de…
vitalyisaev2 Dec 11, 2024
a684503
Manual fixes to make Generic provider compile
vitalyisaev2 Dec 13, 2024
2ff65c1
Avoid extra changes
vitalyisaev2 Dec 13, 2024
08b8d47
Fix ya.make files
vitalyisaev2 Dec 16, 2024
fcb71dd
YDB FQ: fix tests after type renaming (#12589)
vitalyisaev2 Dec 13, 2024
d72c390
Fix PostgreSQL tests
vitalyisaev2 Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 22 additions & 16 deletions ydb/core/fq/libs/actors/clusters_from_connections.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include "clusters_from_connections.h"

#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h>
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
#include <ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.h>
#include <ydb/library/yql/utils/url_builder.h>
#include <ydb/library/actors/http/http.h>
Expand Down Expand Up @@ -109,7 +109,7 @@ void FillGenericClusterConfigBase(
TGenericClusterConfig& clusterCfg,
const TConnection& connection,
const TString& connectionName,
NConnector::NApi::EDataSourceKind dataSourceKind,
NYql::EGenericDataSourceKind dataSourceKind,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures
) {
Expand All @@ -125,15 +125,21 @@ void FillGenericClusterConfigBase(
// In YQv1 we just hardcode desired protocols here.
// In YQv2 protocol can be configured via `CREATE EXTERNAL DATA SOURCE` params.
switch (dataSourceKind) {
case NYql::NConnector::NApi::CLICKHOUSE:
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::NConnector::NApi::EProtocol::NATIVE : NYql::NConnector::NApi::EProtocol::HTTP);
case NYql::EGenericDataSourceKind::CLICKHOUSE:
clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::EGenericProtocol::NATIVE : NYql::EGenericProtocol::HTTP);
break;
case NYql::NConnector::NApi::POSTGRESQL:
clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
case NYql::EGenericDataSourceKind::GREENPLUM:
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
break;
case NYql::EGenericDataSourceKind::MYSQL:
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
break;
case NYql::EGenericDataSourceKind::POSTGRESQL:
clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE);
break;
default:
ythrow yexception() << "Unexpected data source kind: '"
<< NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'";
<< NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'";
}

ValidateGenericClusterConfig(clusterCfg, "NFq::FillGenericClusterFromConfig");
Expand All @@ -145,7 +151,7 @@ void FillGenericClusterConfig(
TGenericClusterConfig& clusterCfg,
const TConnection& connection,
const TString& connectionName,
NConnector::NApi::EDataSourceKind dataSourceKind,
NYql::EGenericDataSourceKind dataSourceKind,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures
) {
Expand All @@ -158,7 +164,7 @@ void FillGenericClusterConfig<FederatedQuery::PostgreSQLCluster>(
TGenericClusterConfig& clusterCfg,
const FederatedQuery::PostgreSQLCluster& connection,
const TString& connectionName,
NConnector::NApi::EDataSourceKind dataSourceKind,
NYql::EGenericDataSourceKind dataSourceKind,
const TString& authToken,
const THashMap<TString, TString>& accountIdSignatures
){
Expand Down Expand Up @@ -217,8 +223,8 @@ void AddClustersFromConnections(
case FederatedQuery::ConnectionSetting::kYdbDatabase: {
const auto& db = conn.content().setting().ydb_database();
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::YDB);
clusterCfg->SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE);
clusterCfg->SetKind(NYql::EGenericDataSourceKind::YDB);
clusterCfg->SetProtocol(NYql::EGenericProtocol::NATIVE);
clusterCfg->SetName(connectionName);
clusterCfg->SetDatabaseId(db.database_id());
clusterCfg->SetUseSsl(!common.GetDisableSslForGenericDataSources());
Expand All @@ -232,7 +238,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().clickhouse_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::CLICKHOUSE,
NYql::EGenericDataSourceKind::CLICKHOUSE,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand Down Expand Up @@ -265,7 +271,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().postgresql_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::POSTGRESQL,
NYql::EGenericDataSourceKind::POSTGRESQL,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand All @@ -277,7 +283,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().greenplum_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::GREENPLUM,
NYql::EGenericDataSourceKind::GREENPLUM,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand All @@ -289,7 +295,7 @@ void AddClustersFromConnections(
*gatewaysConfig.MutableGeneric()->AddClusterMapping(),
conn.content().setting().mysql_cluster(),
connectionName,
NYql::NConnector::NApi::EDataSourceKind::MYSQL,
NYql::EGenericDataSourceKind::MYSQL,
authToken,
accountIdSignatures);
clusters.emplace(connectionName, GenericProviderName);
Expand All @@ -298,7 +304,7 @@ void AddClustersFromConnections(
case FederatedQuery::ConnectionSetting::kLogging: {
const auto& connection = conn.content().setting().logging();
auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping();
clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::LOGGING);
clusterCfg->SetKind(NYql::EGenericDataSourceKind::LOGGING);
clusterCfg->SetName(connectionName);
clusterCfg->mutable_datasourceoptions()->insert({"folder_id", connection.folder_id()});
FillClusterAuth(*clusterCfg, connection.auth(), authToken, accountIdSignatures);
Expand Down
14 changes: 7 additions & 7 deletions ydb/core/fq/libs/actors/database_resolver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ using TParser = std::function<TDatabaseDescription(
NJson::TJsonValue& body,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
)>;
using TParsers = THashMap<NYql::EDatabaseType, TParser>;

Expand Down Expand Up @@ -292,7 +292,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
.SetErrorTtl(TDuration::Minutes(1))
.SetMaxSize(1000000))
{
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) {
auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NYql::EGenericProtocol) {
bool secure = false;
TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust();
TString prefix("/?database=");
Expand Down Expand Up @@ -333,7 +333,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol)
NYql::EGenericProtocol protocol)
{
auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol);
// TODO: Take explicit field from MVP
Expand All @@ -349,7 +349,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
Expand Down Expand Up @@ -380,7 +380,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
Expand Down Expand Up @@ -427,7 +427,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TString aliveHost;
Expand Down Expand Up @@ -465,7 +465,7 @@ class TDatabaseResolver: public TActor<TDatabaseResolver>
NJson::TJsonValue& databaseInfo,
const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
bool useTls,
NConnector::NApi::EProtocol protocol
NYql::EGenericProtocol protocol
) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
Expand Down
34 changes: 17 additions & 17 deletions ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

void Test(
NYql::EDatabaseType databaseType,
NYql::NConnector::NApi::EProtocol protocol,
NYql::EGenericProtocol protocol,
const TString& getUrl,
const TString& status,
const TString& responseBody,
Expand Down Expand Up @@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(Ydb_Serverless) {
Test(
NYql::EDatabaseType::Ydb,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand Down Expand Up @@ -218,7 +218,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::Ydb,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"",
"",
Expand All @@ -237,7 +237,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(Ydb_Dedicated) {
Test(
NYql::EDatabaseType::Ydb,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand All @@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(DataStreams_Serverless) {
Test(
NYql::EDatabaseType::DataStreams,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand All @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(DataStreams_Dedicated) {
Test(
NYql::EDatabaseType::DataStreams,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"200",
R"(
Expand All @@ -302,7 +302,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(ClickHouseNative) {
Test(
NYql::EDatabaseType::ClickHouse,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -336,7 +336,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(ClickHouseHttp) {
Test(
NYql::EDatabaseType::ClickHouse,
NYql::NConnector::NApi::EProtocol::HTTP,
NYql::EGenericProtocol::HTTP,
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -381,7 +381,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::ClickHouse,
NYql::NConnector::NApi::EProtocol::HTTP,
NYql::EGenericProtocol::HTTP,
"https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"403",
R"(
Expand All @@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(PostgreSQL) {
Test(
NYql::EDatabaseType::PostgreSQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -454,7 +454,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::PostgreSQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"403",
R"(
Expand All @@ -478,7 +478,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(Greenplum_MasterNode) {
Test(
NYql::EDatabaseType::Greenplum,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
"200",
R"({
Expand Down Expand Up @@ -520,7 +520,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::Greenplum,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts",
"403",
R"(
Expand All @@ -542,7 +542,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
Y_UNIT_TEST(MySQL) {
Test(
NYql::EDatabaseType::MySQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"200",
R"({
Expand Down Expand Up @@ -590,7 +590,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

Test(
NYql::EDatabaseType::MySQL,
NYql::NConnector::NApi::EProtocol::NATIVE,
NYql::EGenericProtocol::NATIVE,
"https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts",
"403",
R"(
Expand Down Expand Up @@ -624,7 +624,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {
};
Test(
NYql::EDatabaseType::DataStreams,
NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED,
NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED,
"https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh",
"403",
R"(
Expand All @@ -642,7 +642,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) {

NYql::TDatabaseAuth databaseAuth;
databaseAuth.UseTls = true;
databaseAuth.Protocol = NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED;
databaseAuth.Protocol = NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED;

TString databaseId1{"etn021us5r9rhld1vgb1"};
TString databaseId2{"etn021us5r9rhld1vgb2"};
Expand Down
1 change: 0 additions & 1 deletion ydb/core/fq/libs/actors/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ PEERDIR(
ydb/library/yql/providers/dq/provider
ydb/library/yql/providers/dq/provider/exec
ydb/library/yql/providers/dq/worker_manager/interface
ydb/library/yql/providers/generic/connector/api/common
ydb/library/yql/providers/generic/connector/libcpp
ydb/library/yql/providers/generic/provider
ydb/library/yql/providers/pq/cm_client
Expand Down
6 changes: 5 additions & 1 deletion ydb/core/fq/libs/config/protos/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ PEERDIR(
ydb/library/yql/providers/s3/proto
)

EXCLUDE_TAGS(GO_PROTO)
ONLY_TAGS(
CPP_PROTO
PY_PROTO
PY3_PROTO
)

END()
Loading
Loading