From 9b690f98c8a90ba31a911a659a6a7d1c92206f8c Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Fri, 13 Dec 2024 11:14:01 +0300 Subject: [PATCH] YDB FQ: fix tests after type renaming (#12589) --- .../connector/tests/common_test_cases/base.py | 58 +++++------ .../select_missing_database.py | 10 +- .../common_test_cases/select_missing_table.py | 6 +- .../select_positive_common.py | 98 +++++++++---------- .../tests/datasource/clickhouse/collection.py | 10 +- .../tests/datasource/clickhouse/conftest.py | 6 +- .../datasource/clickhouse/docker-compose.yml | 2 +- .../datasource/clickhouse/select_datetime.py | 10 +- .../datasource/clickhouse/select_positive.py | 24 ++--- .../tests/datasource/clickhouse/test.py | 6 +- .../datasource/ms_sql_server/collection.py | 10 +- .../datasource/ms_sql_server/conftest.py | 6 +- .../ms_sql_server/docker-compose.yml | 2 +- .../ms_sql_server/select_datetime.py | 10 +- .../ms_sql_server/select_positive.py | 22 ++--- .../tests/datasource/ms_sql_server/test.py | 6 +- .../tests/datasource/mysql/collection.py | 10 +- .../tests/datasource/mysql/conftest.py | 4 +- .../tests/datasource/mysql/docker-compose.yml | 2 +- .../tests/datasource/mysql/select_datetime.py | 10 +- .../tests/datasource/mysql/select_positive.py | 26 ++--- .../connector/tests/datasource/mysql/test.py | 6 +- .../tests/datasource/oracle/collection.py | 8 +- .../tests/datasource/oracle/conftest.py | 4 +- .../datasource/oracle/docker-compose.yml | 2 +- .../select_datetime_with_service_name.py | 10 +- .../select_positive_with_service_name.py | 34 +++---- .../connector/tests/datasource/oracle/test.py | 6 +- .../tests/datasource/postgresql/collection.py | 10 +- .../tests/datasource/postgresql/conftest.py | 6 +- .../datasource/postgresql/docker-compose.yml | 2 +- .../datasource/postgresql/select_datetime.py | 10 +- .../datasource/postgresql/select_positive.py | 28 +++--- .../postgresql/select_positive_with_schema.py | 6 +- .../tests/datasource/postgresql/test.py | 4 +- .../tests/datasource/ydb/collection.py | 8 +- .../tests/datasource/ydb/docker-compose.yml | 2 +- .../tests/datasource/ydb/select_positive.py | 34 +++---- .../connector/tests/datasource/ydb/test.py | 6 +- .../generic/connector/tests/join/conftest.py | 4 +- .../connector/tests/join/docker-compose.yml | 2 +- .../generic/connector/tests/join/scenario.py | 6 +- .../generic/connector/tests/join/test.py | 4 +- .../generic/connector/tests/join/test_case.py | 26 ++--- .../connector/tests/utils/data_source_kind.py | 8 +- .../generic/connector/tests/utils/database.py | 44 ++++----- .../connector/tests/utils/docker_compose.py | 14 +-- .../connector/tests/utils/one_time_waiter.py | 4 +- .../connector/tests/utils/run/dqrun.py | 8 +- .../connector/tests/utils/run/kqprun.py | 8 +- .../tests/utils/scenario/postgresql.py | 2 +- .../generic/connector/tests/utils/schema.py | 20 ++-- .../generic/connector/tests/utils/settings.py | 48 ++++----- .../fq/generic/analytics/docker-compose.yml | 2 +- .../fq/generic/streaming/docker-compose.yml | 2 +- 55 files changed, 372 insertions(+), 354 deletions(-) diff --git a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py index 6292077391c4..ce3f9e37306d 100644 --- a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py +++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py @@ -4,7 +4,7 @@ from typing import Dict import functools -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.library.yql.providers.generic.connector.tests.utils.database import Database from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings @@ -13,26 +13,26 @@ @dataclass class BaseTestCase: name_: str - data_source_kind: EDataSourceKind.ValueType + data_source_kind: EGenericDataSourceKind.ValueType pragmas: Dict[str, str] - protocol: EProtocol + protocol: EGenericProtocol @property def name(self) -> str: match self.data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: # ClickHouse has two kinds of network protocols: NATIVE and HTTP, # so we append protocol name to the test case name - return f'{self.name_}_{EProtocol.Name(self.protocol)}' - case EDataSourceKind.MS_SQL_SERVER: + return f'{self.name_}_{EGenericProtocol.Name(self.protocol)}' + case EGenericDataSourceKind.MS_SQL_SERVER: return self.name_ - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return self.name_ - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return self.name_ - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return self.name_ - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return self.name_ case _: raise Exception(f'invalid data source: {self.data_source_kind}') @@ -45,17 +45,17 @@ def database(self) -> Database: ''' # FIXME: do not hardcode databases here match self.data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return Database(self.name, self.data_source_kind) - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return Database("master", self.data_source_kind) - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return Database("db", self.data_source_kind) - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return Database(self.name, self.data_source_kind) - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return Database(self.name, self.data_source_kind) - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return Database("local", self.data_source_kind) @functools.cached_property @@ -65,17 +65,17 @@ def table_name(self) -> str: so we provide a random table name instead where necessary. ''' match self.data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return self.name_ # without protocol - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return self.name - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return self.name - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return self.name - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return 't' + make_random_string(8) - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return self.name case _: raise Exception(f'invalid data source: {self.data_source_kind}') @@ -90,34 +90,34 @@ def pragmas_sql_string(self) -> str: @property def generic_settings(self) -> GenericSettings: match self.data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return GenericSettings( date_time_format=EDateTimeFormat.YQL_FORMAT, clickhouse_clusters=[ - GenericSettings.ClickHouseCluster(database=self.database.name, protocol=EProtocol.NATIVE) + GenericSettings.ClickHouseCluster(database=self.database.name, protocol=EGenericProtocol.NATIVE) ], ) - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return GenericSettings( date_time_format=EDateTimeFormat.YQL_FORMAT, ms_sql_server_clusters=[GenericSettings.MsSQLServerCluster(database=self.database.name)], ) - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return GenericSettings( date_time_format=EDateTimeFormat.YQL_FORMAT, mysql_clusters=[GenericSettings.MySQLCluster(database=self.database.name)], ) - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return GenericSettings( date_time_format=EDateTimeFormat.YQL_FORMAT, oracle_clusters=[GenericSettings.OracleCluster(database=self.database.name, service_name=None)], ) - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return GenericSettings( date_time_format=EDateTimeFormat.YQL_FORMAT, postgresql_clusters=[GenericSettings.PostgreSQLCluster(database=self.database.name, schema=None)], ) - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return GenericSettings( date_time_format=EDateTimeFormat.YQL_FORMAT, ydb_clusters=[GenericSettings.YdbCluster(database=self.database.name)], diff --git a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py index da33e6401036..c94735e59db4 100644 --- a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py +++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings @@ -16,14 +16,14 @@ def generic_settings(self) -> GenericSettings: gs = super().generic_settings # Overload setting for MySQL database - if self.data_source_kind == EDataSourceKind.MYSQL: + if self.data_source_kind == EGenericDataSourceKind.MYSQL: for cluster in gs.mysql_clusters: cluster.database = "missing_database" for cluster in gs.oracle_clusters: if self.service_name is not None: cluster.service_name = self.service_name - if self.data_source_kind == EDataSourceKind.MS_SQL_SERVER: + if self.data_source_kind == EGenericDataSourceKind.MS_SQL_SERVER: for cluster in gs.ms_sql_server_clusters: cluster.database = "missing_database" @@ -36,12 +36,12 @@ class Factory: def __init__(self, ss: Settings): self.ss = ss - def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]: + def make_test_cases(self, data_source_kind: EGenericDataSourceKind) -> List[TestCase]: return [ TestCase( name_="missing_database", data_source_kind=data_source_kind, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), service_name=self.ss.oracle.service_name if self.ss.oracle is not None else None, ) diff --git a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py index 3d24f0e3f9ce..9426d0d27dee 100644 --- a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py +++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py @@ -3,7 +3,7 @@ from typing import Sequence from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings @@ -31,7 +31,7 @@ class Factory: def __init__(self, ss: Settings): self.ss = ss - def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]: + def make_test_cases(self, data_source_kind: EGenericDataSourceKind) -> List[TestCase]: test_cases = [] test_case_name = 'missing_table' @@ -39,7 +39,7 @@ def make_test_cases(self, data_source_kind: EDataSourceKind) -> List[TestCase]: test_case = TestCase( name_=test_case_name, data_source_kind=data_source_kind, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), service_name=self.ss.oracle.service_name if self.ss.oracle is not None else None, ) diff --git a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py index 99da19b19aae..f9cc66c2b797 100644 --- a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py +++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, replace from typing import Sequence, Optional -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings @@ -95,11 +95,11 @@ def _column_selection(self) -> Sequence[TestCase]: SelectWhat.asterisk(column_list=schema.columns), data_in, ( - EDataSourceKind.CLICKHOUSE, - EDataSourceKind.POSTGRESQL, - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.POSTGRESQL, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ), ), # SELECT COL1 FROM table @@ -110,11 +110,11 @@ def _column_selection(self) -> Sequence[TestCase]: [10], ], ( - EDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.CLICKHOUSE, # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1) - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ), ), # SELECT col1 FROM table @@ -124,7 +124,7 @@ def _column_selection(self) -> Sequence[TestCase]: [1], [10], ], - (EDataSourceKind.POSTGRESQL,), # works because of implicit cast to lowercase (COL1 -> col1) + (EGenericDataSourceKind.POSTGRESQL,), # works because of implicit cast to lowercase (COL1 -> col1) ), # SELECT col2 FROM table ( @@ -134,11 +134,11 @@ def _column_selection(self) -> Sequence[TestCase]: [20], ], ( - EDataSourceKind.CLICKHOUSE, - EDataSourceKind.POSTGRESQL, - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.POSTGRESQL, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ), ), # SELECT col2, COL1 FROM table @@ -149,11 +149,11 @@ def _column_selection(self) -> Sequence[TestCase]: [20, 10], ], ( - EDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.CLICKHOUSE, # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1) - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ), ), # SELECT col2, col1 FROM table @@ -163,7 +163,7 @@ def _column_selection(self) -> Sequence[TestCase]: [2, 1], [20, 10], ], - (EDataSourceKind.POSTGRESQL,), # works because of implicit cast to lowercase (COL1 -> col1) + (EGenericDataSourceKind.POSTGRESQL,), # works because of implicit cast to lowercase (COL1 -> col1) ), # Simple math computation: # SELECT COL1 + col2 AS col3 FROM table @@ -174,11 +174,11 @@ def _column_selection(self) -> Sequence[TestCase]: [30], ], ( - EDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.CLICKHOUSE, # NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1) - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ), ), # Select the same column multiple times with different aliases @@ -196,11 +196,11 @@ def _column_selection(self) -> Sequence[TestCase]: [20, 20, 20, 20, 20], ], ( - EDataSourceKind.CLICKHOUSE, - EDataSourceKind.POSTGRESQL, - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.POSTGRESQL, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ), ), ) @@ -215,7 +215,7 @@ def _column_selection(self) -> Sequence[TestCase]: test_case = TestCase( data_in=data_in, data_source_kind=data_source_kind, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, select_what=select_what, select_where=None, schema=schema, @@ -260,8 +260,8 @@ def _large_table(self) -> Sequence[TestCase]: ) data_source_kinds = ( - EDataSourceKind.CLICKHOUSE, - EDataSourceKind.POSTGRESQL, + EGenericDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.POSTGRESQL, ) test_case_name = 'large' @@ -269,11 +269,11 @@ def _large_table(self) -> Sequence[TestCase]: for data_source_kind in data_source_kinds: match data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: tc = TestCase( name_=test_case_name, data_source_kind=data_source_kind, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, data_in=None, data_out_=[[999999]], # We put 1M of rows in the large table select_what=SelectWhat(SelectWhat.Item(name='MAX(col_00_int32)', kind='expr')), @@ -282,18 +282,18 @@ def _large_table(self) -> Sequence[TestCase]: pragmas=dict(), ) - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: # Assuming that request will look something like: # `SELECT * FROM table WHERE id = (SELECT MAX(id) FROM table)` # We expect last line to be the answer data_in = generate_table_data(schema=schema, bytes_soft_limit=table_size) data_out = [data_in[-1]] - data_source_kinds = [EDataSourceKind.CLICKHOUSE, EDataSourceKind.POSTGRESQL] + data_source_kinds = [EGenericDataSourceKind.CLICKHOUSE, EGenericDataSourceKind.POSTGRESQL] tc = TestCase( name_=test_case_name, data_source_kind=data_source_kind, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, data_in=data_in, data_out_=data_out, select_what=SelectWhat.asterisk(schema.columns), @@ -311,31 +311,31 @@ def _large_table(self) -> Sequence[TestCase]: return test_cases - def make_test_cases(self, data_source_kind: EDataSourceKind) -> Sequence[TestCase]: + def make_test_cases(self, data_source_kind: EGenericDataSourceKind) -> Sequence[TestCase]: protocols = { - EDataSourceKind.CLICKHOUSE: [EProtocol.NATIVE, EProtocol.HTTP], - EDataSourceKind.POSTGRESQL: [EProtocol.NATIVE], - EDataSourceKind.YDB: [EProtocol.NATIVE], - EDataSourceKind.MYSQL: [EProtocol.NATIVE], - EDataSourceKind.MS_SQL_SERVER: [EProtocol.NATIVE], + EGenericDataSourceKind.CLICKHOUSE: [EGenericProtocol.NATIVE, EGenericProtocol.HTTP], + EGenericDataSourceKind.POSTGRESQL: [EGenericProtocol.NATIVE], + EGenericDataSourceKind.YDB: [EGenericProtocol.NATIVE], + EGenericDataSourceKind.MYSQL: [EGenericProtocol.NATIVE], + EGenericDataSourceKind.MS_SQL_SERVER: [EGenericProtocol.NATIVE], } base_test_cases = None if data_source_kind in [ - EDataSourceKind.YDB, - EDataSourceKind.MYSQL, - EDataSourceKind.MS_SQL_SERVER, + EGenericDataSourceKind.YDB, + EGenericDataSourceKind.MYSQL, + EGenericDataSourceKind.MS_SQL_SERVER, ]: base_test_cases = self._column_selection() - elif data_source_kind in [EDataSourceKind.CLICKHOUSE, EDataSourceKind.POSTGRESQL]: + elif data_source_kind in [EGenericDataSourceKind.CLICKHOUSE, EGenericDataSourceKind.POSTGRESQL]: base_test_cases = list( itertools.chain( self._column_selection(), self._large_table(), ) ) - elif data_source_kind == EDataSourceKind.ORACLE: + elif data_source_kind == EGenericDataSourceKind.ORACLE: raise 'Common test cases are not supported by Oracle due to the lack of Int32 columns' else: raise f'Unexpected data source kind: {data_source_kind}' diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py index b317efd3b5ef..48bc1c1af596 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py @@ -1,6 +1,6 @@ from typing import Sequence, Mapping -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind # test cases import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database @@ -17,10 +17,12 @@ class Collection(object): def __init__(self, ss: Settings): self._test_cases = { - 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EDataSourceKind.CLICKHOUSE), - 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.CLICKHOUSE), + 'select_missing_database': select_missing_database.Factory(ss).make_test_cases( + EGenericDataSourceKind.CLICKHOUSE + ), + 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EGenericDataSourceKind.CLICKHOUSE), 'select_positive': select_positive.Factory().make_test_cases() - + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.CLICKHOUSE), + + select_positive_common.Factory(ss).make_test_cases(EGenericDataSourceKind.CLICKHOUSE), 'select_datetime': select_datetime.Factory().make_test_cases(), } diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py index b2869f8d1f5f..2788b5c4017b 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py @@ -3,7 +3,7 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse") @@ -11,4 +11,6 @@ @pytest.fixture def settings() -> Settings: - return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.CLICKHOUSE]) + return Settings.from_env( + docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.CLICKHOUSE] + ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml index 5a0c45f76f75..cd94b3249893 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml @@ -15,7 +15,7 @@ services: - ./init:/docker-entrypoint-initdb.d fq-connector-go: container_name: fq-tests-ch-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py index 1057054a4b92..5e9dcbcc5e52 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py @@ -2,7 +2,7 @@ import datetime from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.public.api.protos.ydb_value_pb2 import Type @@ -120,8 +120,8 @@ def _make_test_yql(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(schema.columns), select_where=None, - data_source_kind=EDataSourceKind.CLICKHOUSE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, + protocol=EGenericProtocol.NATIVE, schema=schema, pragmas=dict(), check_output_schema=True, @@ -177,12 +177,12 @@ def _make_test_string(self) -> TestCase: return TestCase( name_=test_case_name, date_time_format=EDateTimeFormat.STRING_FORMAT, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, data_in=None, data_out_=data_out, select_what=SelectWhat.asterisk(schema.columns), select_where=None, - data_source_kind=EDataSourceKind.CLICKHOUSE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, schema=schema, pragmas=dict(), check_output_schema=True, diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py index da75133e3432..244a4e93add7 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py @@ -3,7 +3,7 @@ from dataclasses import replace from typing import Sequence, Final -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse @@ -169,8 +169,8 @@ def _primitive_types_non_nullable(self) -> Sequence[TestCase]: datetime.datetime(2023, 3, 21, 11, 21, 31, 456000), ], ], - data_source_kind=EDataSourceKind.CLICKHOUSE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -271,8 +271,8 @@ def _primitive_types_nullable(self) -> Sequence[TestCase]: datetime.datetime(2023, 3, 21, 11, 21, 31, 456000), ], ], - data_source_kind=EDataSourceKind.CLICKHOUSE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -315,8 +315,8 @@ def _constant(self) -> Sequence[TestCase]: 42, ], ], - data_source_kind=EDataSourceKind.CLICKHOUSE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -350,8 +350,8 @@ def _counts(self) -> Sequence[TestCase]: 4, ], ], - protocol=EProtocol.NATIVE, - data_source_kind=EDataSourceKind.CLICKHOUSE, + protocol=EGenericProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, pragmas=dict(), check_output_schema=False, # because the aggregate's value has other type ) @@ -378,7 +378,7 @@ def _pushdown(self) -> TestCase: ['one'], ] - data_source_kind = EDataSourceKind.CLICKHOUSE + data_source_kind = EGenericDataSourceKind.CLICKHOUSE test_case_name = 'pushdown' return [ @@ -390,7 +390,7 @@ def _pushdown(self) -> TestCase: select_what=SelectWhat(SelectWhat.Item(name='col_01_string')), select_where=SelectWhere('col_00_int32 = 1'), data_source_kind=data_source_kind, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, schema=schema, # TODO: implement schema checkswhen selecting only one column check_output_schema=False, @@ -398,7 +398,7 @@ def _pushdown(self) -> TestCase: ] def make_test_cases(self) -> Sequence[TestCase]: - protocols = [EProtocol.NATIVE, EProtocol.HTTP] + protocols = [EGenericProtocol.NATIVE, EGenericProtocol.HTTP] base_test_cases = list( itertools.chain( diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py index ab690a43f2d7..4380bf270bf8 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner import ydb.library.yql.providers.generic.connector.tests.utils.scenario.clickhouse as scenario @@ -14,7 +14,7 @@ import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common one_time_waiter = OneTimeWaiter( - data_source_kind=EDataSourceKind.CLICKHOUSE, + data_source_kind=EGenericDataSourceKind.CLICKHOUSE, docker_compose_file_path=str(docker_compose_dir / 'docker-compose.yml'), expected_tables=[ "column_selection_A_b_C_d_E", @@ -36,7 +36,7 @@ # Global collection of test cases dependent on environment tc_collection = Collection( - Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.CLICKHOUSE]) + Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.CLICKHOUSE]) ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/collection.py index 19868608b43f..1643f2355483 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/collection.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/collection.py @@ -1,6 +1,6 @@ from typing import Sequence, Mapping -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common @@ -18,11 +18,13 @@ class Collection(object): def __init__(self, ss: Settings): self._test_cases = { 'select_missing_database': select_missing_database.Factory(ss).make_test_cases( - EDataSourceKind.MS_SQL_SERVER + EGenericDataSourceKind.MS_SQL_SERVER + ), + 'select_missing_table': select_missing_table.Factory(ss).make_test_cases( + EGenericDataSourceKind.MS_SQL_SERVER ), - 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.MS_SQL_SERVER), 'select_positive': select_positive.Factory().make_test_cases() - + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.MS_SQL_SERVER), + + select_positive_common.Factory(ss).make_test_cases(EGenericDataSourceKind.MS_SQL_SERVER), 'select_datetime': select_datetime.Factory().make_test_cases(), } diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/conftest.py index c229347d688b..90b079354d5f 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/conftest.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/conftest.py @@ -4,7 +4,7 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings @@ -13,4 +13,6 @@ @pytest.fixture def settings() -> Settings: - return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MS_SQL_SERVER]) + return Settings.from_env( + docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.MS_SQL_SERVER] + ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml index aa7213e1f539..9a78ab8851c9 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mssql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_datetime.py index 21db4ef79157..125f4ebad100 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_datetime.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_datetime.py @@ -2,7 +2,7 @@ import datetime from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.public.api.protos.ydb_value_pb2 import Type @@ -96,8 +96,8 @@ def _make_test_yql(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(self._schema.columns), select_where=None, - data_source_kind=EDataSourceKind.MS_SQL_SERVER, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, + protocol=EGenericProtocol.NATIVE, schema=self._schema, pragmas=dict(), ) @@ -134,8 +134,8 @@ def _make_test_string(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(self._schema.columns), select_where=None, - data_source_kind=EDataSourceKind.MS_SQL_SERVER, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, + protocol=EGenericProtocol.NATIVE, schema=self._schema, pragmas=dict(), ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_positive.py index ce729a234ab7..d00811dfdfd7 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_positive.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/select_positive.py @@ -2,7 +2,7 @@ import itertools from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type import ydb.library.yql.providers.generic.connector.tests.utils.types.ms_sql_server as ms_sql_server @@ -212,8 +212,8 @@ def _primitive_types(self) -> Sequence[TestCase]: datetime.datetime(2023, 3, 21, 11, 21, 31, 0), ], ], - data_source_kind=EDataSourceKind.MS_SQL_SERVER, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -254,8 +254,8 @@ def _constant(self) -> Sequence[TestCase]: 42, ], ], - data_source_kind=EDataSourceKind.MS_SQL_SERVER, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -289,8 +289,8 @@ def _count_rows(self) -> Sequence[TestCase]: 3, ], ], - data_source_kind=EDataSourceKind.MS_SQL_SERVER, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -324,11 +324,11 @@ def _pushdown(self) -> TestCase: name_=test_case_name, data_in=None, data_out_=[[4, None, None]], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat.asterisk(schema.columns), select_where=SelectWhere('col_00_id = 4'), - data_source_kind=EDataSourceKind.MS_SQL_SERVER, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, schema=schema, ), TestCase( @@ -337,11 +337,11 @@ def _pushdown(self) -> TestCase: data_out_=[ ['b'], ], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat(SelectWhat.Item(name='col_02_text')), select_where=SelectWhere('col_00_id = col_01_integer'), - data_source_kind=EDataSourceKind.MS_SQL_SERVER, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, schema=schema, ), ] diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/test.py index 57ac2806b79b..d313915cf4a2 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.one_time_waiter import OneTimeWaiter from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner @@ -19,11 +19,11 @@ # Global collection of test cases dependent on environment tc_collection = Collection( - Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MS_SQL_SERVER]) + Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.MS_SQL_SERVER]) ) one_time_waiter = OneTimeWaiter( - data_source_kind=EDataSourceKind.MS_SQL_SERVER, + data_source_kind=EGenericDataSourceKind.MS_SQL_SERVER, docker_compose_file_path=str(docker_compose_dir / 'docker-compose.yml'), expected_tables=[ 'column_selection_A_b_C_d_E', diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/collection.py index 2f6d5334368f..97c4381dce20 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/collection.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/collection.py @@ -1,6 +1,6 @@ from typing import Sequence, Mapping -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common @@ -17,10 +17,12 @@ class Collection(object): def __init__(self, ss: Settings): self._test_cases = { - 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EDataSourceKind.MYSQL), - 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.MYSQL), + 'select_missing_database': select_missing_database.Factory(ss).make_test_cases( + EGenericDataSourceKind.MYSQL + ), + 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EGenericDataSourceKind.MYSQL), 'select_positive': select_positive.Factory().make_test_cases() - + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.MYSQL), + + select_positive_common.Factory(ss).make_test_cases(EGenericDataSourceKind.MYSQL), 'select_datetime': select_datetime.Factory().make_test_cases(), } diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/conftest.py index 0724b709dbde..4fed6c744d2b 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/conftest.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/conftest.py @@ -4,7 +4,7 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings @@ -13,4 +13,4 @@ @pytest.fixture def settings() -> Settings: - return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MYSQL]) + return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.MYSQL]) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml index 55e01b2cd08c..aecb80c7b28b 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mysql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_datetime.py index 8cad456b187a..1c210b276c40 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_datetime.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_datetime.py @@ -2,7 +2,7 @@ import datetime from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.public.api.protos.ydb_value_pb2 import Type @@ -90,8 +90,8 @@ def _make_test_yql(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(self._schema.columns), select_where=None, - data_source_kind=EDataSourceKind.MYSQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MYSQL, + protocol=EGenericProtocol.NATIVE, schema=self._schema, pragmas=dict(), ) @@ -121,8 +121,8 @@ def _make_test_string(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(self._schema.columns), select_where=None, - data_source_kind=EDataSourceKind.MYSQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MYSQL, + protocol=EGenericProtocol.NATIVE, schema=self._schema, pragmas=dict(), ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_positive.py index 96278c1ae29e..a08c823927d1 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_positive.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/select_positive.py @@ -2,7 +2,7 @@ import itertools from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type import ydb.library.yql.providers.generic.connector.tests.utils.types.mysql as mysql @@ -285,8 +285,8 @@ def _primitive_types(self) -> Sequence[TestCase]: '{ "TODO" : "unicode" }', ], ], - data_source_kind=EDataSourceKind.MYSQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MYSQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -327,8 +327,8 @@ def _constant(self) -> Sequence[TestCase]: 42, ], ], - data_source_kind=EDataSourceKind.MYSQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MYSQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -362,8 +362,8 @@ def _count_rows(self) -> Sequence[TestCase]: 3, ], ], - data_source_kind=EDataSourceKind.MYSQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.MYSQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -397,11 +397,11 @@ def _pushdown(self) -> TestCase: name_=test_case_name, data_in=None, data_out_=[[4, None, None]], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat.asterisk(schema.columns), select_where=SelectWhere('col_00_id = 4'), - data_source_kind=EDataSourceKind.MYSQL, + data_source_kind=EGenericDataSourceKind.MYSQL, schema=schema, ), TestCase( @@ -410,11 +410,11 @@ def _pushdown(self) -> TestCase: data_out_=[ ['b'], ], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat(SelectWhat.Item(name='col_02_text')), select_where=SelectWhere('col_00_id = col_01_integer'), - data_source_kind=EDataSourceKind.MYSQL, + data_source_kind=EGenericDataSourceKind.MYSQL, schema=schema, ), ] @@ -444,10 +444,10 @@ def _json(self) -> TestCase: [None], [None], ], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, select_what=SelectWhat(SelectWhat.Item(name='JSON_QUERY(col_01_json, "$.friends[0]")', kind='expr')), select_where=None, - data_source_kind=EDataSourceKind.MYSQL, + data_source_kind=EGenericDataSourceKind.MYSQL, pragmas=dict(), schema=schema, ), diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/test.py index 78203acf53fe..b1340face7bb 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.one_time_waiter import OneTimeWaiter from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner @@ -17,7 +17,7 @@ LOGGER = make_logger(__name__) one_time_waiter = OneTimeWaiter( - data_source_kind=EDataSourceKind.MYSQL, + data_source_kind=EGenericDataSourceKind.MYSQL, docker_compose_file_path=str(docker_compose_dir / 'docker-compose.yml'), expected_tables=[ 'column_selection_A_b_C_d_E', @@ -39,7 +39,7 @@ # Global collection of test cases dependent on environment tc_collection = Collection( - Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.MYSQL]) + Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.MYSQL]) ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/collection.py index cf014c90af84..6f59eb08ed29 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/collection.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/collection.py @@ -1,6 +1,6 @@ from typing import Sequence, Mapping -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind # import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table @@ -19,10 +19,10 @@ class Collection(object): def __init__(self, ss: Settings): self._test_cases = { - # 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EDataSourceKind.ORACLE), # TODO YQ-3413 - 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.ORACLE), + # 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EGenericDataSourceKind.ORACLE), # TODO YQ-3413 + 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EGenericDataSourceKind.ORACLE), 'select_positive': select_positive_with_service_name.Factory(ss).make_test_cases(), - # + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.ORACLE), # TODO does not work because of cast to uppercase and lack of Int32 column type + # + select_positive_common.Factory(ss).make_test_cases(EGenericDataSourceKind.ORACLE), # TODO does not work because of cast to uppercase and lack of Int32 column type 'select_datetime': select_datetime_with_service_name.Factory(ss).make_test_cases(), } diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/conftest.py index be427e6e3003..405a314b4fb9 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/conftest.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/conftest.py @@ -4,7 +4,7 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings @@ -13,4 +13,4 @@ @pytest.fixture def settings() -> Settings: - return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.ORACLE]) + return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.ORACLE]) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml index 0ad174526862..a663e34d6d21 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-oracle-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_datetime_with_service_name.py b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_datetime_with_service_name.py index cf0cca4ec14f..084bc5462aaa 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_datetime_with_service_name.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_datetime_with_service_name.py @@ -3,7 +3,7 @@ from typing import Sequence from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.public.api.protos.ydb_value_pb2 import Type @@ -91,8 +91,8 @@ def _make_test_yql(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(self._schema.columns), select_where=None, - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, schema=self._schema, pragmas=dict(), service_name=self.ss.oracle.service_name, @@ -121,8 +121,8 @@ def _make_test_string(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(self._schema.columns), select_where=None, - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, schema=self._schema, pragmas=dict(), service_name=self.ss.oracle.service_name, diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_positive_with_service_name.py b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_positive_with_service_name.py index 54b4a9be3383..dfdfca783946 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_positive_with_service_name.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/select_positive_with_service_name.py @@ -4,7 +4,7 @@ from typing import Sequence, Optional from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings @@ -250,8 +250,8 @@ def _primitive_types(self) -> Sequence[TestCase]: '{ "TODO" : "unicode" }', ], ], - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, service_name=self.ss.oracle.service_name, @@ -296,8 +296,8 @@ def _longraw(self) -> Sequence[TestCase]: None, ], ], - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, service_name=self.ss.oracle.service_name, @@ -342,8 +342,8 @@ def _long_table(self) -> Sequence[TestCase]: None, ], ], - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, service_name=self.ss.oracle.service_name, @@ -385,8 +385,8 @@ def _constant(self) -> Sequence[TestCase]: 42, ], ], - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), service_name=self.ss.oracle.service_name, ) @@ -421,8 +421,8 @@ def _count_rows(self) -> Sequence[TestCase]: 3, ], ], - data_source_kind=EDataSourceKind.ORACLE, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.ORACLE, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), service_name=self.ss.oracle.service_name, ) @@ -457,11 +457,11 @@ def _pushdown(self) -> TestCase: name_=test_case_name, data_in=None, data_out_=[[4, None, None]], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat.asterisk(schema.columns), select_where=SelectWhere('COL_00_ID = 4'), - data_source_kind=EDataSourceKind.ORACLE, + data_source_kind=EGenericDataSourceKind.ORACLE, schema=schema, service_name=self.ss.oracle.service_name, ), @@ -471,11 +471,11 @@ def _pushdown(self) -> TestCase: data_out_=[ ['b'], ], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat(SelectWhat.Item(name='COL_02_TEXT')), select_where=SelectWhere('COL_00_ID = COL_01_INTEGER'), - data_source_kind=EDataSourceKind.ORACLE, + data_source_kind=EGenericDataSourceKind.ORACLE, schema=schema, service_name=self.ss.oracle.service_name, ), @@ -506,10 +506,10 @@ def _json(self) -> TestCase: [None], [None], ], - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, select_what=SelectWhat(SelectWhat.Item(name='JSON_QUERY(COL_01_JSON, "$.friends[0]")', kind='expr')), select_where=None, - data_source_kind=EDataSourceKind.ORACLE, + data_source_kind=EGenericDataSourceKind.ORACLE, pragmas=dict(), schema=schema, service_name=self.ss.oracle.service_name, diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/test.py index 8c57b8bce1a9..9ee77d1dc4a5 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.one_time_waiter import OneTimeWaiter from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner @@ -20,11 +20,11 @@ # Global collection of test cases dependent on environment tc_collection = Collection( - Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.ORACLE]) + Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.ORACLE]) ) one_time_waiter = OneTimeWaiter( - data_source_kind=EDataSourceKind.ORACLE, + data_source_kind=EGenericDataSourceKind.ORACLE, docker_compose_file_path=str(docker_compose_dir / 'docker-compose.yml'), expected_tables=[ 'column_selection_A_b_C_d_E', diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py index 3f077ca089ff..353871b7d3c7 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py @@ -1,6 +1,6 @@ from typing import Sequence, Mapping -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common @@ -16,10 +16,12 @@ class Collection(object): def __init__(self, ss: Settings): self._test_cases = { - 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EDataSourceKind.POSTGRESQL), - 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.POSTGRESQL), + 'select_missing_database': select_missing_database.Factory(ss).make_test_cases( + EGenericDataSourceKind.POSTGRESQL + ), + 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EGenericDataSourceKind.POSTGRESQL), 'select_positive': select_positive.Factory().make_test_cases() - + select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.POSTGRESQL), + + select_positive_common.Factory(ss).make_test_cases(EGenericDataSourceKind.POSTGRESQL), 'select_positive_with_schema': select_positive_with_schema.Factory().make_test_cases(), 'select_datetime': select_datetime.Factory().make_test_cases(), } diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py index e2e36087059d..7d3f0c096603 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py @@ -4,7 +4,7 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client @@ -14,7 +14,9 @@ @pytest.fixture def settings() -> Settings: - return Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.POSTGRESQL]) + return Settings.from_env( + docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.POSTGRESQL] + ) @pytest.fixture diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml index 8a17be178649..42e8a7908c50 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-pg-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py index 7c396bd5013a..522bf9cecc92 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py @@ -2,7 +2,7 @@ import datetime from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.public.api.protos.ydb_value_pb2 import Type @@ -92,8 +92,8 @@ def _make_test_yql_postgresql(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(schema.columns), select_where=None, - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, schema=schema, pragmas=dict(), ) @@ -150,8 +150,8 @@ def _make_test_string_postgresql(self) -> TestCase: data_out_=data_out, select_what=SelectWhat.asterisk(schema.columns), select_where=None, - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, schema=schema, pragmas=dict(), ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py index c42ad0152b43..8db8a254cd29 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py @@ -2,7 +2,7 @@ import itertools from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type import ydb.library.yql.providers.generic.connector.tests.utils.types.postgresql as postgresql @@ -254,8 +254,8 @@ def _primitive_types(self) -> Sequence[TestCase]: ], ], data_out_=None, - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -295,8 +295,8 @@ def _upper_case_column(self) -> Sequence[TestCase]: 3, ], ], - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -346,8 +346,8 @@ def _constant(self) -> Sequence[TestCase]: 42, ], ], - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -391,8 +391,8 @@ def _count(self) -> Sequence[TestCase]: 3, ], ], - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -438,7 +438,7 @@ def _pushdown(self) -> TestCase: ['two'], ] - data_source_kind = EDataSourceKind.POSTGRESQL + data_source_kind = EGenericDataSourceKind.POSTGRESQL test_case_name = 'pushdown' @@ -447,7 +447,7 @@ def _pushdown(self) -> TestCase: name_=test_case_name, data_in=data_in, data_out_=data_out_1, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat(SelectWhat.Item(name='col_string')), select_where=SelectWhere('col_int32 = 1'), @@ -458,7 +458,7 @@ def _pushdown(self) -> TestCase: name_=test_case_name, data_in=data_in, data_out_=data_out_2, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat(SelectWhat.Item(name='col_string')), select_where=SelectWhere('col_int32 = col_int64'), @@ -490,7 +490,7 @@ def _json(self) -> TestCase: [None], ] - data_source_kind = EDataSourceKind.POSTGRESQL + data_source_kind = EGenericDataSourceKind.POSTGRESQL test_case_name = 'json' @@ -499,7 +499,7 @@ def _json(self) -> TestCase: name_=test_case_name, data_in=data_in, data_out_=data_out_1, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, select_what=SelectWhat(SelectWhat.Item(name='JSON_QUERY(col_json, "$.friends[0]")', kind='expr')), select_where=None, data_source_kind=data_source_kind, diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py index b2cb9bf457cf..338dc14159ff 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py @@ -5,7 +5,7 @@ from string import ascii_lowercase, digits -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings @@ -67,8 +67,8 @@ def _select_with_pg_schema(self) -> Sequence[TestCase]: test_case = TestCase( name_=test_case_name, data_in=[[1, 2], [10, 20]], - data_source_kind=EDataSourceKind.POSTGRESQL, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.POSTGRESQL, + protocol=EGenericProtocol.NATIVE, select_what=select_what, schema=schema, data_out_=[[1, 2], [10, 20]], diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py index a46e79609c34..c94f36b5797e 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client @@ -16,7 +16,7 @@ # Global collection of test cases dependent on environment tc_collection = Collection( - Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.POSTGRESQL]) + Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.POSTGRESQL]) ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py index cb0da9fe7b23..23a8e64c1745 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py @@ -1,6 +1,6 @@ from typing import Sequence, Mapping -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table @@ -15,9 +15,9 @@ class Collection(object): def __init__(self, ss: Settings): self._test_cases = { - 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EDataSourceKind.YDB), - 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EDataSourceKind.YDB), - 'select_positive': select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.YDB) + 'select_missing_database': select_missing_database.Factory(ss).make_test_cases(EGenericDataSourceKind.YDB), + 'select_missing_table': select_missing_table.Factory(ss).make_test_cases(EGenericDataSourceKind.YDB), + 'select_positive': select_positive_common.Factory(ss).make_test_cases(EGenericDataSourceKind.YDB) + select_positive.Factory().make_test_cases(), } diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml index 72680763e599..a2091af3ed92 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml @@ -8,7 +8,7 @@ services: dnsmasq; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: fq-tests-ydb-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/select_positive.py index 7d23cecce8d4..af096e1de4a8 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/select_positive.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/select_positive.py @@ -2,7 +2,7 @@ import itertools from typing import Sequence -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.public.api.protos.ydb_value_pb2 import Type import ydb.library.yql.providers.generic.connector.tests.utils.types.ydb as types_ydb @@ -167,8 +167,8 @@ def _primitive_types(self) -> Sequence[TestCase]: '{ "TODO" : "unicode" }', ], ], - data_source_kind=EDataSourceKind.YDB, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.YDB, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -341,8 +341,8 @@ def _optional_types(self) -> Sequence[TestCase]: None, ], ], - data_source_kind=EDataSourceKind.YDB, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.YDB, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -383,8 +383,8 @@ def _constant(self) -> Sequence[TestCase]: 42, ], ], - data_source_kind=EDataSourceKind.YDB, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.YDB, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), ) @@ -418,8 +418,8 @@ def _count(self) -> Sequence[TestCase]: 4, ], ], - protocol=EProtocol.NATIVE, - data_source_kind=EDataSourceKind.YDB, + protocol=EGenericProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.YDB, pragmas=dict(), ) @@ -453,8 +453,8 @@ def _pushdown(self) -> TestCase: pragmas=dict({'generic.UsePredicatePushdown': 'true'}), select_what=SelectWhat(SelectWhat.Item(name='col_01_string')), select_where=SelectWhere('col_00_id = 1'), - data_source_kind=EDataSourceKind.YDB, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.YDB, + protocol=EGenericProtocol.NATIVE, schema=schema, ) ] @@ -492,8 +492,8 @@ def _unsupported_types(self) -> Sequence[TestCase]: 2, ], ], - data_source_kind=EDataSourceKind.YDB, - protocol=EProtocol.NATIVE, + data_source_kind=EGenericDataSourceKind.YDB, + protocol=EGenericProtocol.NATIVE, pragmas=dict(), check_output_schema=True, ) @@ -539,7 +539,7 @@ def _json(self) -> Sequence[TestCase]: [None], ] - data_source_kind = EDataSourceKind.YDB + data_source_kind = EGenericDataSourceKind.YDB test_case_name = 'json' @@ -548,7 +548,7 @@ def _json(self) -> Sequence[TestCase]: name_=test_case_name, data_in=data_in, data_out_=data_out_1, - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, select_what=SelectWhat(SelectWhat.Item(name='JSON_QUERY(col_01_json, "$.friends[0]")', kind='expr')), select_where=None, data_source_kind=data_source_kind, @@ -585,9 +585,9 @@ def _json_document(self) -> TestCase: data_out_=data_out, select_where=None, select_what=SelectWhat(SelectWhat.Item(name='col_01_data')), - data_source_kind=EDataSourceKind.YDB, + data_source_kind=EGenericDataSourceKind.YDB, pragmas=dict(), - protocol=EProtocol.NATIVE, + protocol=EGenericProtocol.NATIVE, schema=schema, # check_output_schema=True, ) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py index 843b885b6906..05aea2d55286 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner from ydb.library.yql.providers.generic.connector.tests.utils.one_time_waiter import OneTimeWaiter @@ -14,7 +14,7 @@ from collection import Collection one_time_waiter = OneTimeWaiter( - data_source_kind=EDataSourceKind.YDB, + data_source_kind=EGenericDataSourceKind.YDB, docker_compose_file_path=str(docker_compose_dir / 'docker-compose.yml'), expected_tables=[ "column_selection_A_b_C_d_E", @@ -35,7 +35,7 @@ ], ) -settings = Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.YDB]) +settings = Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EGenericDataSourceKind.YDB]) tc_collection = Collection(settings) diff --git a/ydb/library/yql/providers/generic/connector/tests/join/conftest.py b/ydb/library/yql/providers/generic/connector/tests/join/conftest.py index a2d90087ea6e..a896d70f9660 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/conftest.py +++ b/ydb/library/yql/providers/generic/connector/tests/join/conftest.py @@ -4,7 +4,7 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client as PostgreSQLClient from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings @@ -16,7 +16,7 @@ def settings() -> Settings: return Settings.from_env( docker_compose_dir=docker_compose_dir, - data_source_kinds=[EDataSourceKind.POSTGRESQL, EDataSourceKind.CLICKHOUSE], + data_source_kinds=[EGenericDataSourceKind.POSTGRESQL, EGenericDataSourceKind.CLICKHOUSE], ) diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml index 7ab666f7df14..c9c920712828 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml @@ -15,7 +15,7 @@ services: - ./init/clickhouse:/docker-entrypoint-initdb.d fq-connector-go: container_name: fq-tests-join-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/join/scenario.py b/ydb/library/yql/providers/generic/connector/tests/join/scenario.py index 156ceb8cabd6..32929e2e997f 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/scenario.py +++ b/ydb/library/yql/providers/generic/connector/tests/join/scenario.py @@ -1,4 +1,4 @@ -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.comparator import assert_data_outs_equal from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings @@ -22,10 +22,10 @@ def join( # prepare tables for data_source in test_case.data_sources: match data_source.kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: # do nothing as tables are initialized via init scripts continue - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: postgresql_scenario.prepare_table( test_name=test_name, client=postgresql_client, diff --git a/ydb/library/yql/providers/generic/connector/tests/join/test.py b/ydb/library/yql/providers/generic/connector/tests/join/test.py index 7623c10922ff..c163982860a4 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/test.py +++ b/ydb/library/yql/providers/generic/connector/tests/join/test.py @@ -1,6 +1,6 @@ import pytest -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner @@ -13,7 +13,7 @@ tc_collection = Collection( Settings.from_env( docker_compose_dir=conftest.docker_compose_dir, - data_source_kinds=[EDataSourceKind.CLICKHOUSE, EDataSourceKind.POSTGRESQL], + data_source_kinds=[EGenericDataSourceKind.CLICKHOUSE, EGenericDataSourceKind.POSTGRESQL], ) ) diff --git a/ydb/library/yql/providers/generic/connector/tests/join/test_case.py b/ydb/library/yql/providers/generic/connector/tests/join/test_case.py index ca0d0a0474ed..038b62665908 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/test_case.py +++ b/ydb/library/yql/providers/generic/connector/tests/join/test_case.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import Sequence, Final -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.public.api.protos.ydb_value_pb2 import Type @@ -36,7 +36,7 @@ def select_what(self) -> SelectWhat: class DataSource: database: Database table: Table - kind: EDataSourceKind.ValueType = EDataSourceKind.DATA_SOURCE_KIND_UNSPECIFIED + kind: EGenericDataSourceKind.ValueType = EGenericDataSourceKind.DATA_SOURCE_KIND_UNSPECIFIED @property def alias(self) -> str: @@ -70,12 +70,14 @@ def generic_settings(self) -> GenericSettings: for data_source in self.data_sources: match data_source.kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: clickhouse_clusters.add( - GenericSettings.ClickHouseCluster(database=data_source.database.name, protocol=EProtocol.NATIVE) + GenericSettings.ClickHouseCluster( + database=data_source.database.name, protocol=EGenericProtocol.NATIVE + ) ) - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: postgresql_clusters.add( GenericSettings.PostgreSQLCluster(database=data_source.database.name, schema=None) ) @@ -170,9 +172,9 @@ def __make_simple_test_cases(self) -> Sequence[TestCase]: data_out = list(map(lambda x: list(itertools.chain(*x)), zip(*(t.data_in for t in tables)))) - data_sources: Sequence[EDataSourceKind] = ( - EDataSourceKind.CLICKHOUSE, - EDataSourceKind.POSTGRESQL, + data_sources: Sequence[EGenericDataSourceKind] = ( + EGenericDataSourceKind.CLICKHOUSE, + EGenericDataSourceKind.POSTGRESQL, ) # For each test case we create a unique set of datasources; @@ -247,13 +249,13 @@ def __make_inner_join_test_case(self) -> Sequence[TestCase]: test_case_data_sources = [ DataSource( - kind=EDataSourceKind.CLICKHOUSE, - database=Database(kind=EDataSourceKind.CLICKHOUSE, name=test_case_name), + kind=EGenericDataSourceKind.CLICKHOUSE, + database=Database(kind=EGenericDataSourceKind.CLICKHOUSE, name=test_case_name), table=ch_table, ), DataSource( - kind=EDataSourceKind.POSTGRESQL, - database=Database(kind=EDataSourceKind.POSTGRESQL, name=test_case_name), + kind=EGenericDataSourceKind.POSTGRESQL, + database=Database(kind=EGenericDataSourceKind.POSTGRESQL, name=test_case_name), table=pg_table, ), ] diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/data_source_kind.py b/ydb/library/yql/providers/generic/connector/tests/utils/data_source_kind.py index a186fdd87ab0..2098dbb4f113 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/data_source_kind.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/data_source_kind.py @@ -1,11 +1,11 @@ -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind -def data_source_kind_alias(kind: EDataSourceKind.ValueType) -> str: +def data_source_kind_alias(kind: EGenericDataSourceKind.ValueType) -> str: match (kind): - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return "ch" - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return "pg" case _: raise Exception(f'invalid data source: {kind}') diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/database.py b/ydb/library/yql/providers/generic/connector/tests/utils/database.py index 82b2f50139a7..ccbea9570f50 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/database.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/database.py @@ -1,34 +1,34 @@ from dataclasses import dataclass -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind @dataclass class Database: name: str - def __init__(self, name: str, kind: EDataSourceKind.ValueType): + def __init__(self, name: str, kind: EGenericDataSourceKind.ValueType): self.kind = kind match kind: - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: # PostgreSQL implicitly converts all identifiers to lowercase, # so we'd better make it first on our own self.name = name[:63].lower() - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: # We use preinitialized database when working with ClickHouse. self.name = "db" - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: # For this kind of database this name is provided by the external logic self.name = name - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: # For this kind of database this name is provided by the external logic self.name = name - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: # Oracle is not sensitive for identifiers until they are inclosed in quota marks, # therefore, we'd better use uppercase for ease of testing self.name = name[:127].upper() # TODO: is it needed? max length of Oracle table name is 128 bytes/chars - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: # We use preinitialized database when working with YDB. self.name = "local" case _: @@ -36,14 +36,14 @@ def __init__(self, name: str, kind: EDataSourceKind.ValueType): def exists(self) -> str: match self.kind: - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return f"SELECT 1 FROM pg_database WHERE datname = '{self.name}'" case _: raise Exception(f'invalid data source: {self.kind}') def create(self) -> str: match self.kind: - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return f"CREATE DATABASE {self.name}" case _: raise Exception(f'invalid data source: {self.kind}') @@ -53,34 +53,34 @@ def sql_table_name(self, table_name: str) -> str: def missing_database_msg(self) -> str: match self.kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return f"Database {self.name} doesn't exist" - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return f'database "{self.name}" does not exist' - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: raise Exception("Fix me first in YQ-3315") - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return 'Cannot open database' - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return 'Unknown database' - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: raise Exception("Fix me first in YQ-3413") case _: raise Exception(f'invalid data source: {self.kind}') def missing_table_msg(self) -> str: match self.kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return 'table does not exist' - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return 'table does not exist' - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return 'issues = [{\'Path not found\'}])' - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return 'table does not exist' - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return 'table does not exist' - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return 'table does not exist' case _: raise Exception(f'invalid data source: {self.kind}') diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py b/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py index 19d308744e20..dd2acea25897 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py @@ -9,7 +9,7 @@ import yatest.common -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger LOGGER = make_logger(__name__) @@ -111,17 +111,17 @@ def get_internal_ip(self, service_name: str) -> str: def get_container_name(self, service_name: str) -> str: return self.docker_compose_yml_data['services'][service_name]['container_name'] - def list_tables(self, dataSourceKind: EDataSourceKind) -> Sequence[str]: + def list_tables(self, dataSourceKind: EGenericDataSourceKind) -> Sequence[str]: match dataSourceKind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return self.list_clickhouse_tables() - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return self._list_ydb_tables() - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return self._list_mysql_tables() - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return self._list_ms_sql_server_tables() - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return self._list_oracle_tables() case _: raise ValueError("invalid data source kind: {dataSourceKind}") diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/one_time_waiter.py b/ydb/library/yql/providers/generic/connector/tests/utils/one_time_waiter.py index 3659b095beca..082ecba1325a 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/one_time_waiter.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/one_time_waiter.py @@ -4,7 +4,7 @@ import yatest.common -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger from ydb.library.yql.providers.generic.connector.tests.utils.docker_compose import DockerComposeHelper @@ -17,7 +17,7 @@ class OneTimeWaiter: def __init__( self, docker_compose_file_path: str, - data_source_kind: EDataSourceKind, + data_source_kind: EGenericDataSourceKind, expected_tables: Sequence[str], ): docker_compose_file_abs_path = yatest.common.source_path(docker_compose_file_path) diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py index 9bdea4d763ec..5c95f875e3b2 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py @@ -6,7 +6,7 @@ from yt import yson -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat import ydb.library.yql.providers.generic.connector.tests.utils.artifacts as artifacts @@ -76,10 +76,10 @@ class GatewaysConfRenderer: {% for cluster in generic_settings.clickhouse_clusters %} -{% if cluster.protocol == EProtocol.NATIVE %} +{% if cluster.protocol == EGenericProtocol.NATIVE %} {% set CLICKHOUSE_PORT = settings.clickhouse.native_port_internal %} {% set CLICKHOUSE_PROTOCOL = NATIVE %} -{% elif cluster.protocol == EProtocol.HTTP %} +{% elif cluster.protocol == EGenericProtocol.HTTP %} {% set CLICKHOUSE_PORT = settings.clickhouse.http_port_internal %} {% set CLICKHOUSE_PROTOCOL = HTTP %} {% endif %} @@ -244,7 +244,7 @@ def __init__(self): self.template = jinja2.Environment(loader=jinja2.BaseLoader, undefined=jinja2.DebugUndefined).from_string( self._template ) - self.template.globals['EProtocol'] = EProtocol + self.template.globals['EGenericProtocol'] = EGenericProtocol self.template.globals['EDateTimeFormat'] = EDateTimeFormat def render(self, file_path: Path, settings: Settings, generic_settings: GenericSettings) -> None: diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py index 36ee776144df..f1c5f3bb2464 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py @@ -5,7 +5,7 @@ import jinja2 -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat import ydb.library.yql.providers.generic.connector.tests.utils.artifacts as artifacts @@ -63,10 +63,10 @@ class SchemeRenderer: {% for cluster in generic_settings.clickhouse_clusters %} -{% if cluster.protocol == EProtocol.NATIVE %} +{% if cluster.protocol == EGenericProtocol.NATIVE %} {% set CLICKHOUSE_PORT = settings.clickhouse.native_port_internal %} {% set CLICKHOUSE_PROTOCOL = NATIVE %} -{% elif cluster.protocol == EProtocol.HTTP %} +{% elif cluster.protocol == EGenericProtocol.HTTP %} {% set CLICKHOUSE_PORT = settings.clickhouse.http_port_internal %} {% set CLICKHOUSE_PROTOCOL = HTTP %} {% endif %} @@ -165,7 +165,7 @@ def __init__(self): self.template = jinja2.Environment(loader=jinja2.BaseLoader, undefined=jinja2.DebugUndefined).from_string( self.template_ ) - self.template.globals['EProtocol'] = EProtocol + self.template.globals['EGenericProtocol'] = EGenericProtocol def render(self, file_path: Path, settings: Settings, generic_settings: GenericSettings) -> None: content = self.template.render(dict(settings=settings, generic_settings=generic_settings)) diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py index 91e0522bdcfc..dbc555ba1800 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py @@ -1,6 +1,6 @@ from typing import Sequence -import ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 as data_source_pb2 +import yql.essentials.providers.common.proto.gateways_config_pb2 as data_source_pb2 import ydb.library.yql.providers.generic.connector.tests.utils.artifacts as artifacts from ydb.library.yql.providers.generic.connector.tests.utils.comparator import assert_data_outs_equal diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/schema.py b/ydb/library/yql/providers/generic/connector/tests/utils/schema.py index d931f31730a1..cbbb7abb4e7a 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/schema.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/schema.py @@ -5,7 +5,7 @@ from yt import yson from yt.yson.yson_types import YsonEntity import ydb.public.api.protos.ydb_value_pb2 as ydb_value -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind from ydb.public.api.protos.ydb_value_pb2 import Type, OptionalType import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse @@ -27,20 +27,20 @@ class DataSourceType: pg: postgresql.Type = None ydb: Ydb.Type = None - def pick(self, kind: EDataSourceKind.ValueType) -> str: + def pick(self, kind: EGenericDataSourceKind.ValueType) -> str: target = None match kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: target = self.ch - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: target = self.ms - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: target = self.my - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: target = self.ora - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: target = self.pg - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: target = self.ydb case _: raise Exception(f'invalid data source: {kind}') @@ -192,7 +192,7 @@ def __cast_primitive_type(self, primitive_type_id: ydb_value.Type.PrimitiveTypeI case _: raise Exception(f"invalid type '{primitive_type_id}' for value '{value}'") - def format_for_data_source(self, kind: EDataSourceKind.ValueType) -> str: + def format_for_data_source(self, kind: EGenericDataSourceKind.ValueType) -> str: return f'{self.name} {self.data_source_type.pick(kind)}' @@ -349,7 +349,7 @@ def from_yson(cls, src: YsonList): def from_json(cls, src: Dict): return cls(columns=ColumnList(*map(Column.from_json, src))) - def yql_column_list(self, kind: EDataSourceKind.ValueType) -> str: + def yql_column_list(self, kind: EGenericDataSourceKind.ValueType) -> str: return ", ".join(map(lambda col: col.format_for_data_source(kind), self.columns)) def select_every_column(self) -> SelectWhat: diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/settings.py b/ydb/library/yql/providers/generic/connector/tests/utils/settings.py index db76d0927539..7d639bfe5e84 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/settings.py +++ b/ydb/library/yql/providers/generic/connector/tests/utils/settings.py @@ -4,7 +4,7 @@ import yatest.common -from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol +from yql.essentials.providers.common.proto.gateways_config_pb2 import EGenericDataSourceKind, EGenericProtocol from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat from ydb.library.yql.providers.generic.connector.tests.utils.docker_compose import DockerComposeHelper @@ -100,7 +100,9 @@ class Ydb: ydb: Ydb @classmethod - def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[EDataSourceKind]) -> 'Settings': + def from_env( + cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[EGenericDataSourceKind] + ) -> 'Settings': docker_compose_file_relative_path = str(docker_compose_dir / 'docker-compose.yml') docker_compose_file_abs_path = yatest.common.source_path(docker_compose_file_relative_path) endpoint_determiner = DockerComposeHelper(docker_compose_file_abs_path) @@ -109,7 +111,7 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ for data_source_kind in data_source_kinds: match data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: data_sources[data_source_kind] = cls.ClickHouse( cluster_name='clickhouse_integration_test', host_external='0.0.0.0', @@ -125,7 +127,7 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ password='password', protocol='native', ) - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: data_sources[data_source_kind] = cls.MsSQLServer( cluster_name='ms_sql_server_integration_test', host_external='0.0.0.0', @@ -139,7 +141,7 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ username='sa', password='Qwerty12345!', ) - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: data_sources[data_source_kind] = cls.MySQL( cluster_name='mysql_integration_test', host_external='0.0.0.0', @@ -153,7 +155,7 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ username='root', password='password', ) - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: data_sources[data_source_kind] = cls.Oracle( cluster_name='oracle_integration_test', host_external='0.0.0.0', @@ -168,7 +170,7 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ password='password', service_name="FREE", ) - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: data_sources[data_source_kind] = cls.PostgreSQL( cluster_name='postgresql_integration_test', host_external='0.0.0.0', @@ -182,7 +184,7 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ username='user', password='password', ) - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: data_sources[data_source_kind] = cls.Ydb( cluster_name='ydb_integration_test', host_internal=endpoint_determiner.get_container_name('ydb'), @@ -195,36 +197,36 @@ def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[ raise Exception(f'invalid data source: {data_source_kind}') return cls( - clickhouse=data_sources.get(EDataSourceKind.CLICKHOUSE), + clickhouse=data_sources.get(EGenericDataSourceKind.CLICKHOUSE), connector=cls.Connector( grpc_host='localhost', grpc_port=endpoint_determiner.get_external_port('fq-connector-go', 2130), paging_bytes_per_page=4 * 1024 * 1024, paging_prefetch_queue_capacity=2, ), - ms_sql_server=data_sources.get(EDataSourceKind.MS_SQL_SERVER), - mysql=data_sources.get(EDataSourceKind.MYSQL), - oracle=data_sources.get(EDataSourceKind.ORACLE), - postgresql=data_sources.get(EDataSourceKind.POSTGRESQL), - ydb=data_sources.get(EDataSourceKind.YDB), + ms_sql_server=data_sources.get(EGenericDataSourceKind.MS_SQL_SERVER), + mysql=data_sources.get(EGenericDataSourceKind.MYSQL), + oracle=data_sources.get(EGenericDataSourceKind.ORACLE), + postgresql=data_sources.get(EGenericDataSourceKind.POSTGRESQL), + ydb=data_sources.get(EGenericDataSourceKind.YDB), ) - def get_cluster_name(self, data_source_kind: EDataSourceKind) -> str: + def get_cluster_name(self, data_source_kind: EGenericDataSourceKind) -> str: match data_source_kind: - case EDataSourceKind.CLICKHOUSE: + case EGenericDataSourceKind.CLICKHOUSE: return self.clickhouse.cluster_name - case EDataSourceKind.MYSQL: + case EGenericDataSourceKind.MYSQL: return self.mysql.cluster_name - case EDataSourceKind.ORACLE: + case EGenericDataSourceKind.ORACLE: return self.oracle.cluster_name - case EDataSourceKind.MS_SQL_SERVER: + case EGenericDataSourceKind.MS_SQL_SERVER: return self.ms_sql_server.cluster_name - case EDataSourceKind.POSTGRESQL: + case EGenericDataSourceKind.POSTGRESQL: return self.postgresql.cluster_name - case EDataSourceKind.YDB: + case EGenericDataSourceKind.YDB: return self.ydb.cluster_name case _: - raise Exception(f'invalid data source: {EDataSourceKind.Name(data_source_kind)}') + raise Exception(f'invalid data source: {EGenericDataSourceKind.Name(data_source_kind)}') @dataclass @@ -237,7 +239,7 @@ def __hash__(self) -> int: return hash(self.database) + hash(self.protocol) database: str - protocol: EProtocol + protocol: EGenericProtocol clickhouse_clusters: Sequence[ClickHouseCluster] = field(default_factory=list) diff --git a/ydb/tests/fq/generic/analytics/docker-compose.yml b/ydb/tests/fq/generic/analytics/docker-compose.yml index b71cb5fee9c9..6288502d252e 100644 --- a/ydb/tests/fq/generic/analytics/docker-compose.yml +++ b/ydb/tests/fq/generic/analytics/docker-compose.yml @@ -19,7 +19,7 @@ services: echo \"$$(dig tests-fq-generic-analytics-ydb +short) tests-fq-generic-analytics-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-analytics-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - "2130" volumes: diff --git a/ydb/tests/fq/generic/streaming/docker-compose.yml b/ydb/tests/fq/generic/streaming/docker-compose.yml index d6ea9aa7b69b..4bdfb1219436 100644 --- a/ydb/tests/fq/generic/streaming/docker-compose.yml +++ b/ydb/tests/fq/generic/streaming/docker-compose.yml @@ -5,7 +5,7 @@ services: echo \"$$(dig tests-fq-generic-streaming-ydb +short) tests-fq-generic-streaming-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-streaming-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.12-rc.2@sha256:84bb0b19f16f354b8a9ef7a020ee80f3ba7dc28db92f7007f235241153025b8a + image: ghcr.io/ydb-platform/fq-connector-go:v0.6.0-rc.1@sha256:4f74bd11e696218053f48622d1d065567d103906c187b3a24ea4e56f886c6c60 ports: - "2130" volumes: