From 4d5e2b7f35f396eccb3550af07b415d6cdfe9993 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 18 Apr 2024 13:19:53 +0000 Subject: [PATCH 1/7] Doesn't work --- ydb/tests/fq/generic/docker-compose.yml | 25 ++++++++-- .../generic/utils}/endpoint_determiner.py | 0 ydb/tests/fq/generic/utils/settings.py | 4 +- ydb/tests/fq/generic/utils/ya.make | 5 ++ ydb/tests/fq/generic/ya.make | 50 ++++++++++++------- ydb/tests/fq/generic/ydb/01_basic.sh | 49 ++++++++++++++++++ ydb/tests/fq/generic/ydb/init_ydb | 12 +++++ .../tools/docker_compose_helpers/ya.make | 11 ---- ydb/tests/tools/ya.make | 1 - 9 files changed, 121 insertions(+), 36 deletions(-) rename ydb/tests/{tools/docker_compose_helpers => fq/generic/utils}/endpoint_determiner.py (100%) create mode 100755 ydb/tests/fq/generic/ydb/01_basic.sh create mode 100755 ydb/tests/fq/generic/ydb/init_ydb delete mode 100644 ydb/tests/tools/docker_compose_helpers/ya.make diff --git a/ydb/tests/fq/generic/docker-compose.yml b/ydb/tests/fq/generic/docker-compose.yml index 5bfdd752ce71..4beb7a13424d 100644 --- a/ydb/tests/fq/generic/docker-compose.yml +++ b/ydb/tests/fq/generic/docker-compose.yml @@ -3,7 +3,7 @@ services: postgresql: image: "postgres:15-bullseye@sha256:3411b9f2e5239cd7867f34fcf22fe964230f7d447a71d63c283e3593d3f84085" # to be able to run tests by different users on the same machine we set prefix to ${USER} - container_name: ${USER}_ydb_tests_fq_generic_postgresql + container_name: tests-fq-generic-postgresql environment: POSTGRES_DB: db POSTGRES_USER: user @@ -13,7 +13,7 @@ services: command: -p 6432 clickhouse: image: "clickhouse/clickhouse-server:23-alpine@sha256:b078c1cd294632afa2aeba3530e7ba2e568513da23304354f455a25fab575c06" - container_name: ${USER}_ydb_tests_fq_generic_clickhouse + container_name: tests-fq-generic-clickhouse environment: CLICKHOUSE_DB: db CLICKHOUSE_USER: user @@ -21,8 +21,23 @@ services: CLICKHOUSE_PASSWORD: password volumes: - ./clickhouse:/docker-entrypoint-initdb.d - connector: - image: "ghcr.io/ydb-platform/fq-connector-go:v0.1.1-rc.2@sha256:e5c2d86bce9cb43420eed0ed534afe760fb90ad41229dbbf34af28023b219af3" - container_name: ${USER}_ydb_tests_fq_generic_connector + ydb: + image: ghcr.io/ydb-platform/local-ydb:latest@sha256:9045e00afec1923dc3277564c7b2f829087c2115f45f18e1d38b80bb89f98be6 + container_name: tests-fq-generic-ydb + hostname: tests-fq-generic-ydb + environment: + YDB_DEFAULT_LOG_LEVEL: DEBUG + POSTGRES_USER: user + POSTGRES_PASSWORD: password + volumes: + - ./ydb/init_ydb:/init_ydb + - ./ydb/01_basic.sh:/01_basic.sh + fq-connector-go: + image: "ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6" + container_name: tests-fq-generic-fq-connector-go ports: - '50051' + command: > + sh -c " + echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts; + /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" diff --git a/ydb/tests/tools/docker_compose_helpers/endpoint_determiner.py b/ydb/tests/fq/generic/utils/endpoint_determiner.py similarity index 100% rename from ydb/tests/tools/docker_compose_helpers/endpoint_determiner.py rename to ydb/tests/fq/generic/utils/endpoint_determiner.py diff --git a/ydb/tests/fq/generic/utils/settings.py b/ydb/tests/fq/generic/utils/settings.py index c822b07672fa..ddcd387a7063 100644 --- a/ydb/tests/fq/generic/utils/settings.py +++ b/ydb/tests/fq/generic/utils/settings.py @@ -4,7 +4,7 @@ import yatest.common -from ydb.tests.tools.docker_compose_helpers.endpoint_determiner import EndpointDeterminer +from ydb.tests.fq.generic.utils.endpoint_determiner import EndpointDeterminer @dataclass @@ -54,7 +54,7 @@ def from_env(cls) -> 'Settings': s = cls( connector=cls.Connector( grpc_host='localhost', - grpc_port=endpoint_determiner.get_port('connector', 50051), + grpc_port=endpoint_determiner.get_port('fq-connector-go', 50051), ), mdb_mock=cls.MdbMock( endpoint=environ['MDB_MOCK_ENDPOINT'], diff --git a/ydb/tests/fq/generic/utils/ya.make b/ydb/tests/fq/generic/utils/ya.make index 62d01a35a38a..5e3283aeb5c6 100644 --- a/ydb/tests/fq/generic/utils/ya.make +++ b/ydb/tests/fq/generic/utils/ya.make @@ -3,7 +3,12 @@ PY3_LIBRARY() STYLE_PYTHON() PY_SRCS( + endpoint_determiner.py settings.py ) +PEERDIR( + library/python/testing/yatest_common +) + END() diff --git a/ydb/tests/fq/generic/ya.make b/ydb/tests/fq/generic/ya.make index 3756dc05b184..3f36aa7a003b 100644 --- a/ydb/tests/fq/generic/ya.make +++ b/ydb/tests/fq/generic/ya.make @@ -5,31 +5,48 @@ PY3TEST() STYLE_PYTHON() NO_CHECK_IMPORTS() -TAG( - ya:external - ya:force_sandbox - ya:fat -) - -REQUIREMENTS( - container:4467981730 - cpu:all - dns:dns64 -) - INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/mdb_mock/recipe.inc) INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/token_accessor_mock/recipe.inc) INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/fq_runner/ydb_runner_with_datastreams.inc) -INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) -# Including of docker_compose/recipe.inc automatically converts these tests into LARGE, -# which makes it impossible to run them during precommit checks on Github CI. -# Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928. +IF (AUTOCHECK) + # Temporarily disable these tests due to infrastructure incompatibility + SKIP_TEST("DEVTOOLSUPPORT-44637") + + # Split tests to chunks only when they're running on different machines with distbuild, + # otherwise this directive will slow down local test execution. + # Look through DEVTOOLSSUPPORT-39642 for more information. + FORK_SUBTESTS() + + # TAG and REQUIREMENTS are copied from: https://docs.yandex-team.ru/devtools/test/environment#docker-compose + TAG( + ya:external + ya:force_sandbox + ya:fat + ) + + REQUIREMENTS( + cpu:all + container:4467981730 + dns:dns64 + ) +ENDIF() + +INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) + # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, + # which makes it impossible to run them during precommit checks on Github CI. + # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928. SIZE(MEDIUM) SET(TEST_TAGS_VALUE) SET(TEST_REQUIREMENTS_VALUE) + + # This requirement forces tests to be launched consequently, + # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. + # See DEVTOOLSSUPPORT-44103, YA-1759 for details. + TAG(ya:not_autocheck) + REQUIREMENTS(cpu:all) ENDIF() PEERDIR( @@ -39,7 +56,6 @@ PEERDIR( library/python/testing/yatest_common library/recipes/common ydb/tests/tools/fq_runner - ydb/tests/tools/docker_compose_helpers ydb/public/api/protos contrib/python/pytest diff --git a/ydb/tests/fq/generic/ydb/01_basic.sh b/ydb/tests/fq/generic/ydb/01_basic.sh new file mode 100755 index 000000000000..6794d283b33d --- /dev/null +++ b/ydb/tests/fq/generic/ydb/01_basic.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -ex + +/ydb -p ${PROFILE} yql -s ' + CREATE TABLE column_selection_A_b_C_d_E_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + COMMIT; + INSERT INTO column_selection_A_b_C_d_E_NATIVE (COL1, col2) VALUES + (1, 2), + (10, 20); + COMMIT; + + CREATE TABLE column_selection_COL1_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + COMMIT; + INSERT INTO column_selection_COL1_NATIVE (COL1, col2) VALUES + (1, 2), + (10, 20); + COMMIT; + + CREATE TABLE column_selection_asterisk_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + COMMIT; + INSERT INTO column_selection_asterisk_NATIVE (COL1, col2) VALUES + (1, 2), + (10, 20); + COMMIT; + + CREATE TABLE column_selection_col2_COL1_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + COMMIT; + INSERT INTO column_selection_col2_COL1_NATIVE (COL1, col2) VALUES + (1, 2), + (10, 20); + COMMIT; + + CREATE TABLE column_selection_col2_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + COMMIT; + INSERT INTO column_selection_col2_NATIVE (COL1, col2) VALUES + (1, 2), + (10, 20); + COMMIT; + + CREATE TABLE column_selection_col3_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + COMMIT; + INSERT INTO column_selection_col3_NATIVE (COL1, col2) VALUES + (1, 2), + (10, 20); + COMMIT; + ' + +echo $(date +"%T.%6N") "SUCCESS" diff --git a/ydb/tests/fq/generic/ydb/init_ydb b/ydb/tests/fq/generic/ydb/init_ydb new file mode 100755 index 000000000000..d2a8cbdddfd3 --- /dev/null +++ b/ydb/tests/fq/generic/ydb/init_ydb @@ -0,0 +1,12 @@ +#!/bin/bash + +set -ex + +export PROFILE=tests-ydb-client-$(LC_ALL=C tr -dc A-Za-z0-9 Date: Thu, 18 Apr 2024 13:48:45 +0000 Subject: [PATCH 2/7] Works --- ydb/tests/fq/generic/docker-compose.yml | 2 +- ydb/tests/fq/generic/utils/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/tests/fq/generic/docker-compose.yml b/ydb/tests/fq/generic/docker-compose.yml index 4beb7a13424d..d4176ac88dbc 100644 --- a/ydb/tests/fq/generic/docker-compose.yml +++ b/ydb/tests/fq/generic/docker-compose.yml @@ -36,7 +36,7 @@ services: image: "ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6" container_name: tests-fq-generic-fq-connector-go ports: - - '50051' + - '2130' command: > sh -c " echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts; diff --git a/ydb/tests/fq/generic/utils/settings.py b/ydb/tests/fq/generic/utils/settings.py index ddcd387a7063..69760eed0f52 100644 --- a/ydb/tests/fq/generic/utils/settings.py +++ b/ydb/tests/fq/generic/utils/settings.py @@ -54,7 +54,7 @@ def from_env(cls) -> 'Settings': s = cls( connector=cls.Connector( grpc_host='localhost', - grpc_port=endpoint_determiner.get_port('fq-connector-go', 50051), + grpc_port=endpoint_determiner.get_port('fq-connector-go', 2130), ), mdb_mock=cls.MdbMock( endpoint=environ['MDB_MOCK_ENDPOINT'], From 05b62858d363086a0e0c01335e4667bf6f7f24e9 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 18 Apr 2024 15:39:56 +0000 Subject: [PATCH 3/7] YDB MVP Mock implemented --- ydb/tests/fq/generic/conftest.py | 2 + ydb/tests/fq/generic/test_ydb.py | 47 +++++++++++++++ ydb/tests/fq/generic/utils/settings.py | 22 +++++++ ydb/tests/fq/generic/ya.make | 4 +- ydb/tests/fq/generic/ydb/01_basic.sh | 46 ++++---------- ydb/tests/tools/fq_runner/fq_client.py | 15 +++++ ydb/tests/tools/fq_runner/kikimr_utils.py | 16 +++++ ydb/tests/tools/ydb_mvp_mock/__main__.py | 73 +++++++++++++++++++++++ ydb/tests/tools/ydb_mvp_mock/recipe.inc | 2 + ydb/tests/tools/ydb_mvp_mock/ya.make | 22 +++++++ 10 files changed, 212 insertions(+), 37 deletions(-) create mode 100644 ydb/tests/fq/generic/test_ydb.py create mode 100644 ydb/tests/tools/ydb_mvp_mock/__main__.py create mode 100644 ydb/tests/tools/ydb_mvp_mock/recipe.inc create mode 100644 ydb/tests/tools/ydb_mvp_mock/ya.make diff --git a/ydb/tests/fq/generic/conftest.py b/ydb/tests/fq/generic/conftest.py index e21444a3efc2..4c19855d762d 100644 --- a/ydb/tests/fq/generic/conftest.py +++ b/ydb/tests/fq/generic/conftest.py @@ -6,6 +6,7 @@ from ydb.tests.tools.fq_runner.kikimr_utils import YQv2Extension from ydb.tests.tools.fq_runner.kikimr_utils import TokenAccessorExtension from ydb.tests.tools.fq_runner.kikimr_utils import MDBExtension +from ydb.tests.tools.fq_runner.kikimr_utils import YdbMvpExtension from ydb.tests.tools.fq_runner.kikimr_utils import start_kikimr from utils.settings import Settings @@ -22,6 +23,7 @@ def kikimr(request: pytest.FixtureRequest, settings: Settings, yq_version: str): ConnectorExtension(settings.connector.grpc_host, settings.connector.grpc_port, False), TokenAccessorExtension(settings.token_accessor_mock.endpoint, settings.token_accessor_mock.hmac_secret_file), MDBExtension(settings.mdb_mock.endpoint), + YdbMvpExtension(settings.ydb_mvp_mock.endpoint), YQv2Extension(yq_version), ] with start_kikimr(request, kikimr_extensions) as kikimr: diff --git a/ydb/tests/fq/generic/test_ydb.py b/ydb/tests/fq/generic/test_ydb.py new file mode 100644 index 000000000000..1189f487b5e9 --- /dev/null +++ b/ydb/tests/fq/generic/test_ydb.py @@ -0,0 +1,47 @@ +import logging +import pytest + +import ydb.public.api.protos.draft.fq_pb2 as fq +import ydb.public.api.protos.ydb_value_pb2 as ydb +from ydb.tests.tools.fq_runner.kikimr_utils import yq_v2 + +from ydb.tests.tools.fq_runner.fq_client import FederatedQueryClient +from utils.settings import Settings + + +class TestYdb: + @yq_v2 + @pytest.mark.parametrize("fq_client", [{"folder_id": "my_folder"}], indirect=True) + def test_simple(self, fq_client: FederatedQueryClient, settings: Settings): + table_name = 'simple_table' + conn_name = f'conn_{table_name}' + query_name = f'query_{table_name}' + + fq_client.create_ydb_connection( + name=conn_name, + database_name=settings.ydb.dbname, + database_id='/local', + login=settings.ydb.username, + password=settings.ydb.password, + ) + + sql = fR''' + SELECT * + FROM {conn_name}.{table_name}; + ''' + + query_id = fq_client.create_query(query_name, sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + fq_client.wait_query_status(query_id, fq.QueryMeta.COMPLETED) + + data = fq_client.get_result_data(query_id) + result_set = data.result.result_set + logging.debug(str(result_set)) + assert len(result_set.columns) == 1 + assert result_set.columns[0].name == "number" + assert result_set.columns[0].type == ydb.Type( + optional_type=ydb.OptionalType(item=ydb.Type(type_id=ydb.Type.INT32)) + ) + assert len(result_set.rows) == 3 + assert result_set.rows[0].items[0].int32_value == 1 + assert result_set.rows[1].items[0].int32_value == 2 + assert result_set.rows[2].items[0].int32_value == 3 diff --git a/ydb/tests/fq/generic/utils/settings.py b/ydb/tests/fq/generic/utils/settings.py index 69760eed0f52..ec430695102d 100644 --- a/ydb/tests/fq/generic/utils/settings.py +++ b/ydb/tests/fq/generic/utils/settings.py @@ -22,6 +22,12 @@ class MdbMock: mdb_mock: MdbMock + @dataclass + class YdbMvpMock: + endpoint: str + + ydb_mvp_mock: YdbMvpMock + @dataclass class TokenAccessorMock: endpoint: str @@ -46,6 +52,14 @@ class PostgreSQL: postgresql: PostgreSQL + @dataclass + class Ydb: + dbname: str + username: str + password: str + + ydb: Ydb + @classmethod def from_env(cls) -> 'Settings': docker_compose_file = yatest.common.source_path('ydb/tests/fq/generic/docker-compose.yml') @@ -59,6 +73,9 @@ def from_env(cls) -> 'Settings': mdb_mock=cls.MdbMock( endpoint=environ['MDB_MOCK_ENDPOINT'], ), + ydb_mvp_mock=cls.YdbMvpMock( + endpoint=environ['YDB_MVP_MOCK_ENDPOINT'], + ), token_accessor_mock=cls.TokenAccessorMock( endpoint=environ['TOKEN_ACCESSOR_MOCK_ENDPOINT'], hmac_secret_file=environ['TOKEN_ACCESSOR_HMAC_SECRET_FILE'], @@ -74,6 +91,11 @@ def from_env(cls) -> 'Settings': username='user', password='password', ), + ydb=cls.Ydb( + dbname='local', + username='user', + password='password' + ) ) return s diff --git a/ydb/tests/fq/generic/ya.make b/ydb/tests/fq/generic/ya.make index 3f36aa7a003b..c62744cb34ce 100644 --- a/ydb/tests/fq/generic/ya.make +++ b/ydb/tests/fq/generic/ya.make @@ -5,9 +5,10 @@ PY3TEST() STYLE_PYTHON() NO_CHECK_IMPORTS() +INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/fq_runner/ydb_runner_with_datastreams.inc) INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/mdb_mock/recipe.inc) INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/token_accessor_mock/recipe.inc) -INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/fq_runner/ydb_runner_with_datastreams.inc) +INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/ydb_mvp_mock/recipe.inc) IF (AUTOCHECK) # Temporarily disable these tests due to infrastructure incompatibility @@ -66,6 +67,7 @@ TEST_SRCS( test_clickhouse.py test_join.py test_postgresql.py + test_ydb.py ) END() diff --git a/ydb/tests/fq/generic/ydb/01_basic.sh b/ydb/tests/fq/generic/ydb/01_basic.sh index 6794d283b33d..ac06c575b5f7 100755 --- a/ydb/tests/fq/generic/ydb/01_basic.sh +++ b/ydb/tests/fq/generic/ydb/01_basic.sh @@ -3,46 +3,20 @@ set -ex /ydb -p ${PROFILE} yql -s ' - CREATE TABLE column_selection_A_b_C_d_E_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + CREATE TABLE simple_table (number Int32, PRIMARY KEY (number)); COMMIT; - INSERT INTO column_selection_A_b_C_d_E_NATIVE (COL1, col2) VALUES - (1, 2), - (10, 20); + INSERT INTO simple_table (number) VALUES + (1), + (2), + (3); COMMIT; - CREATE TABLE column_selection_COL1_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); + CREATE TABLE join_table (number Int32, data UTF8 PRIMARY KEY (number)); COMMIT; - INSERT INTO column_selection_COL1_NATIVE (COL1, col2) VALUES - (1, 2), - (10, 20); - COMMIT; - - CREATE TABLE column_selection_asterisk_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); - COMMIT; - INSERT INTO column_selection_asterisk_NATIVE (COL1, col2) VALUES - (1, 2), - (10, 20); - COMMIT; - - CREATE TABLE column_selection_col2_COL1_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); - COMMIT; - INSERT INTO column_selection_col2_COL1_NATIVE (COL1, col2) VALUES - (1, 2), - (10, 20); - COMMIT; - - CREATE TABLE column_selection_col2_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); - COMMIT; - INSERT INTO column_selection_col2_NATIVE (COL1, col2) VALUES - (1, 2), - (10, 20); - COMMIT; - - CREATE TABLE column_selection_col3_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1)); - COMMIT; - INSERT INTO column_selection_col3_NATIVE (COL1, col2) VALUES - (1, 2), - (10, 20); + INSERT INTO join_table (number) VALUES + (1, 'ydb10'), + (2, 'ydb20'), + (3, 'ydb30'); COMMIT; ' diff --git a/ydb/tests/tools/fq_runner/fq_client.py b/ydb/tests/tools/fq_runner/fq_client.py index 28c3a67c13d7..f58dd3704ab6 100644 --- a/ydb/tests/tools/fq_runner/fq_client.py +++ b/ydb/tests/tools/fq_runner/fq_client.py @@ -453,6 +453,21 @@ def create_clickhouse_connection(self, name, database_name, database_id, login, request.content.acl.visibility = visibility return self.create_connection(request, check_issues) + def create_ydb_connection(self, name, database_name, database_id, login, password, + secure=False, visibility=fq.Acl.Visibility.PRIVATE, auth_method=AuthMethod.service_account('sa'), check_issues=True): + request = fq.CreateConnectionRequest() + request.content.name = name + ydb = request.content.setting.ydb_database + #ydb.database_name = database_name + ydb.database_id = database_id + ydb.secure = secure + #ydb.login = login + #ydb.password = password + + ydb.auth.CopyFrom(auth_method) + request.content.acl.visibility = visibility + return self.create_connection(request, check_issues) + @retry.retry_intrusive def list_connections(self, visibility, name_substring=None, limit=100, check_issues=True, page_token=""): request = fq.ListConnectionsRequest() diff --git a/ydb/tests/tools/fq_runner/kikimr_utils.py b/ydb/tests/tools/fq_runner/kikimr_utils.py index 1b1b598f8e5c..93fe071278d4 100644 --- a/ydb/tests/tools/fq_runner/kikimr_utils.py +++ b/ydb/tests/tools/fq_runner/kikimr_utils.py @@ -309,6 +309,22 @@ def apply_to_kikimr(self, request, kikimr): kikimr.compute_plane.fq_config['common']['mdb_gateway'] = self.endpoint # v2 kikimr.compute_plane.fq_config['gateways']['generic']['mdb_gateway'] = self.endpoint # v1 +class YdbMvpExtension(ExtensionPoint): + + def __init__(self, endpoint: str, use_ssl=False): + MDBExtension.__init__.__annotations__ = { + 'endpoint': str, + 'use_ssl': bool + } + super().__init__() + self.endpoint = endpoint + self.use_ssl = use_ssl + + def is_applicable(self, request): + return True + + def apply_to_kikimr(self, request, kikimr): + kikimr.compute_plane.qs_config['generic']['ydb_mvp_endpoint'] = self.endpoint class TokenAccessorExtension(ExtensionPoint): diff --git a/ydb/tests/tools/ydb_mvp_mock/__main__.py b/ydb/tests/tools/ydb_mvp_mock/__main__.py new file mode 100644 index 000000000000..08bc8ecc4aeb --- /dev/null +++ b/ydb/tests/tools/ydb_mvp_mock/__main__.py @@ -0,0 +1,73 @@ +import os +import logging +from typing import Final + +import json +from aiohttp import web + +import yatest.common as yat +from library.python.testing.recipe import declare_recipe, set_env +from library.recipes.common import find_free_ports, start_daemon + +logger = logging.getLogger('ydb_mvp_mock.recipe') + +YDB_MVP_MOCK_PID_FILE: Final = 'recipe.ydb_mvp_mock.pid' +YDB_HOSTNAME: Final = 'tests-fq-generic-ydb' +YDB_PORT: Final = 2136 + +async def ydb_handler(request): + databaseId = request.rel_url.query['databaseId'] + + return web.Response(body=json.dumps( + { + "endpoint": f"grpc://{YDB_HOSTNAME}:{YDB_PORT}/?database={databaseId}" + } + )) + +def serve(port: int): + app = web.Application() + app.add_routes([web.get('/database', ydb_handler)]) + web.run_app(app, port=port) + + +def start(argv): + logger.debug('Start arguments: %s', argv) + + host = "0.0.0.0" + port = find_free_ports(1)[0] + _update_environment(host=host, port=port) + + pid = os.fork() + if pid == 0: + logger.info('Starting ydb_mvp_mock server...') + serve(port=port) + else: + with open(YDB_MVP_MOCK_PID_FILE, "w") as f: + f.write(str(pid)) + + +def _update_environment(host: str, port: int): + variables = { + 'YDB_MVP_MOCK_ENDPOINT': f'http://{host}:{port}', + } + + for k, v in variables.items(): + set_env(k, v) + + +def stop(argv): + logger.debug('Start arguments: %s', argv) + logger.info('Terminating ydb_mvp_mock server...') + try: + with open(yat.work_path(YDB_MVP_MOCK_PID_FILE)) as fin: + pid = fin.read() + except IOError: + logger.error('Can not find server PID') + else: + logger.info('Terminate ydb_mvp_mock server PID: %s', pid) + os.kill(int(pid), 15) + logger.info('Server terminated.') + + +if __name__ == "__main__": + declare_recipe(start, stop) diff --git a/ydb/tests/tools/ydb_mvp_mock/recipe.inc b/ydb/tests/tools/ydb_mvp_mock/recipe.inc new file mode 100644 index 000000000000..97848ed64155 --- /dev/null +++ b/ydb/tests/tools/ydb_mvp_mock/recipe.inc @@ -0,0 +1,2 @@ +DEPENDS(ydb/tests/tools/ydb_mvp_mock) +USE_RECIPE(ydb/tests/tools/ydb_mvp_mock/recipe) diff --git a/ydb/tests/tools/ydb_mvp_mock/ya.make b/ydb/tests/tools/ydb_mvp_mock/ya.make new file mode 100644 index 000000000000..01f85a88dade --- /dev/null +++ b/ydb/tests/tools/ydb_mvp_mock/ya.make @@ -0,0 +1,22 @@ +OWNER( + vitalyisaev + g:yq +) + +PY3_PROGRAM(recipe) + +STYLE_PYTHON() + +PY_SRCS( + __main__.py +) + +PEERDIR( + library/python/testing/recipe + library/python/testing/yatest_common + library/recipes/common + + contrib/python/aiohttp +) + +END() From fe4f2da0122cfbd8ac48a434371a272dcebdacf8 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 18 Apr 2024 15:53:25 +0000 Subject: [PATCH 4/7] Simple test worked --- ydb/tests/fq/generic/ydb/01_basic.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/ydb/tests/fq/generic/ydb/01_basic.sh b/ydb/tests/fq/generic/ydb/01_basic.sh index ac06c575b5f7..3606e3d412c0 100755 --- a/ydb/tests/fq/generic/ydb/01_basic.sh +++ b/ydb/tests/fq/generic/ydb/01_basic.sh @@ -10,14 +10,19 @@ set -ex (2), (3); COMMIT; - - CREATE TABLE join_table (number Int32, data UTF8 PRIMARY KEY (number)); - COMMIT; - INSERT INTO join_table (number) VALUES - (1, 'ydb10'), - (2, 'ydb20'), - (3, 'ydb30'); - COMMIT; ' + # CREATE TABLE join_table (number Int32, data UTF8 PRIMARY KEY (number)); + # COMMIT; + # INSERT INTO join_table (number) VALUES + # (1, 'ydb10'), + # (2, 'ydb20'), + # (3, 'ydb30'); + # COMMIT; + +retVal=$? +if [ $retVal -ne 0 ]; then + echo $retVal + exit $retVal +fi echo $(date +"%T.%6N") "SUCCESS" From 7afe66bb0f016193643420b0f8deb98c878706dc Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 18 Apr 2024 16:38:45 +0000 Subject: [PATCH 5/7] Join test worked --- ydb/tests/fq/generic/test_join.py | 18 +++++++++++++--- ydb/tests/fq/generic/test_ydb.py | 5 +---- ydb/tests/fq/generic/ydb/01_basic.sh | 15 ++++++------- ydb/tests/tools/fq_runner/fq_client.py | 29 ++++++++++++-------------- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/ydb/tests/fq/generic/test_join.py b/ydb/tests/fq/generic/test_join.py index 183ca08234ce..cb8d3a7baaee 100644 --- a/ydb/tests/fq/generic/test_join.py +++ b/ydb/tests/fq/generic/test_join.py @@ -15,6 +15,7 @@ def test_simple(self, fq_client: FederatedQueryClient, settings: Settings): table_name = 'join_table' ch_conn_name = f'ch_conn_{table_name}' pg_conn_name = f'pg_conn_{table_name}' + ydb_conn_name = f'ydb_conn_{table_name}' query_name = f'query_{table_name}' fq_client.create_postgresql_connection( @@ -33,11 +34,18 @@ def test_simple(self, fq_client: FederatedQueryClient, settings: Settings): password=settings.clickhouse.password, ) + fq_client.create_ydb_connection( + name=ydb_conn_name, + database_id=settings.ydb.dbname, + ) + sql = fR''' - SELECT pg.data AS data_pg, ch.data AS data_ch + SELECT pg.data AS data_pg, ch.data AS data_ch, ydb.data AS data_ydb FROM {pg_conn_name}.{table_name} AS pg JOIN {ch_conn_name}.{table_name} AS ch - ON pg.id = ch.id; + ON pg.id = ch.id + JOIN {ydb_conn_name}.{table_name} AS ydb + ON pg.id = ydb.id; ''' query_id = fq_client.create_query(query_name, sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id @@ -46,13 +54,17 @@ def test_simple(self, fq_client: FederatedQueryClient, settings: Settings): data = fq_client.get_result_data(query_id) result_set = data.result.result_set logging.debug(str(result_set)) - assert len(result_set.columns) == 2 + assert len(result_set.columns) == 3 assert result_set.columns[0].name == "data_pg" assert result_set.columns[1].name == "data_ch" + assert result_set.columns[2].name == "data_ydb" assert len(result_set.rows) == 3 assert result_set.rows[0].items[0].bytes_value == b'pg10' assert result_set.rows[0].items[1].bytes_value == b'ch10' + assert result_set.rows[0].items[2].bytes_value == b'ydb10', result_set assert result_set.rows[1].items[0].bytes_value == b'pg20' assert result_set.rows[1].items[1].bytes_value == b'ch20' + assert result_set.rows[1].items[2].bytes_value == b'ydb20' assert result_set.rows[2].items[0].bytes_value == b'pg30' assert result_set.rows[2].items[1].bytes_value == b'ch30' + assert result_set.rows[2].items[2].bytes_value == b'ydb30' diff --git a/ydb/tests/fq/generic/test_ydb.py b/ydb/tests/fq/generic/test_ydb.py index 1189f487b5e9..0ec87e3d7c33 100644 --- a/ydb/tests/fq/generic/test_ydb.py +++ b/ydb/tests/fq/generic/test_ydb.py @@ -19,10 +19,7 @@ def test_simple(self, fq_client: FederatedQueryClient, settings: Settings): fq_client.create_ydb_connection( name=conn_name, - database_name=settings.ydb.dbname, - database_id='/local', - login=settings.ydb.username, - password=settings.ydb.password, + database_id=settings.ydb.dbname, ) sql = fR''' diff --git a/ydb/tests/fq/generic/ydb/01_basic.sh b/ydb/tests/fq/generic/ydb/01_basic.sh index 3606e3d412c0..2c94ade1c561 100755 --- a/ydb/tests/fq/generic/ydb/01_basic.sh +++ b/ydb/tests/fq/generic/ydb/01_basic.sh @@ -10,14 +10,15 @@ set -ex (2), (3); COMMIT; + + CREATE TABLE join_table (id Int32, data STRING, PRIMARY KEY (id)); + COMMIT; + INSERT INTO join_table (id, data) VALUES + (1, "ydb10"), + (2, "ydb20"), + (3, "ydb30"); + COMMIT; ' - # CREATE TABLE join_table (number Int32, data UTF8 PRIMARY KEY (number)); - # COMMIT; - # INSERT INTO join_table (number) VALUES - # (1, 'ydb10'), - # (2, 'ydb20'), - # (3, 'ydb30'); - # COMMIT; retVal=$? if [ $retVal -ne 0 ]; then diff --git a/ydb/tests/tools/fq_runner/fq_client.py b/ydb/tests/tools/fq_runner/fq_client.py index f58dd3704ab6..817c912e4a47 100644 --- a/ydb/tests/tools/fq_runner/fq_client.py +++ b/ydb/tests/tools/fq_runner/fq_client.py @@ -389,18 +389,18 @@ def create_connection(self, request, check_issues=True): response.operation.result.Unpack(result) return FederatedQueryClient.Response(response.operation.issues, result, check_issues) - @retry.retry_intrusive - def create_ydb_connection(self, name, database, endpoint, visibility=fq.Acl.Visibility.PRIVATE, - auth_method=AuthMethod.no_auth(), check_issues=True): - request = fq.CreateConnectionRequest() - request.content.name = name - ydb = request.content.setting.ydb_database - ydb.database = database - ydb.endpoint = endpoint - - ydb.auth.CopyFrom(auth_method) - request.content.acl.visibility = visibility - return self.create_connection(request, check_issues) + # @retry.retry_intrusive + # def create_ydb_connection(self, name, database, endpoint, visibility=fq.Acl.Visibility.PRIVATE, + # auth_method=AuthMethod.no_auth(), check_issues=True): + # request = fq.CreateConnectionRequest() + # request.content.name = name + # ydb = request.content.setting.ydb_database + # ydb.database = database + # ydb.endpoint = endpoint + + # ydb.auth.CopyFrom(auth_method) + # request.content.acl.visibility = visibility + # return self.create_connection(request, check_issues) @retry.retry_intrusive def create_yds_connection(self, name, database=None, endpoint=None, database_id=None, @@ -453,16 +453,13 @@ def create_clickhouse_connection(self, name, database_name, database_id, login, request.content.acl.visibility = visibility return self.create_connection(request, check_issues) - def create_ydb_connection(self, name, database_name, database_id, login, password, + def create_ydb_connection(self, name, database_id, secure=False, visibility=fq.Acl.Visibility.PRIVATE, auth_method=AuthMethod.service_account('sa'), check_issues=True): request = fq.CreateConnectionRequest() request.content.name = name ydb = request.content.setting.ydb_database - #ydb.database_name = database_name ydb.database_id = database_id ydb.secure = secure - #ydb.login = login - #ydb.password = password ydb.auth.CopyFrom(auth_method) request.content.acl.visibility = visibility From dca7b6cb74bdf69e346f075c4105a4099cfed241 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 18 Apr 2024 16:42:14 +0000 Subject: [PATCH 6/7] Minor changes --- ydb/tests/tools/fq_runner/fq_client.py | 36 +++++++++----------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/ydb/tests/tools/fq_runner/fq_client.py b/ydb/tests/tools/fq_runner/fq_client.py index 817c912e4a47..6d9740cdc44b 100644 --- a/ydb/tests/tools/fq_runner/fq_client.py +++ b/ydb/tests/tools/fq_runner/fq_client.py @@ -389,18 +389,18 @@ def create_connection(self, request, check_issues=True): response.operation.result.Unpack(result) return FederatedQueryClient.Response(response.operation.issues, result, check_issues) - # @retry.retry_intrusive - # def create_ydb_connection(self, name, database, endpoint, visibility=fq.Acl.Visibility.PRIVATE, - # auth_method=AuthMethod.no_auth(), check_issues=True): - # request = fq.CreateConnectionRequest() - # request.content.name = name - # ydb = request.content.setting.ydb_database - # ydb.database = database - # ydb.endpoint = endpoint - - # ydb.auth.CopyFrom(auth_method) - # request.content.acl.visibility = visibility - # return self.create_connection(request, check_issues) + @retry.retry_intrusive + def create_ydb_connection(self, name, database_id, + secure=False, visibility=fq.Acl.Visibility.PRIVATE, auth_method=AuthMethod.service_account('sa'), check_issues=True): + request = fq.CreateConnectionRequest() + request.content.name = name + ydb = request.content.setting.ydb_database + ydb.database_id = database_id + ydb.secure = secure + + ydb.auth.CopyFrom(auth_method) + request.content.acl.visibility = visibility + return self.create_connection(request, check_issues) @retry.retry_intrusive def create_yds_connection(self, name, database=None, endpoint=None, database_id=None, @@ -453,18 +453,6 @@ def create_clickhouse_connection(self, name, database_name, database_id, login, request.content.acl.visibility = visibility return self.create_connection(request, check_issues) - def create_ydb_connection(self, name, database_id, - secure=False, visibility=fq.Acl.Visibility.PRIVATE, auth_method=AuthMethod.service_account('sa'), check_issues=True): - request = fq.CreateConnectionRequest() - request.content.name = name - ydb = request.content.setting.ydb_database - ydb.database_id = database_id - ydb.secure = secure - - ydb.auth.CopyFrom(auth_method) - request.content.acl.visibility = visibility - return self.create_connection(request, check_issues) - @retry.retry_intrusive def list_connections(self, visibility, name_substring=None, limit=100, check_issues=True, page_token=""): request = fq.ListConnectionsRequest() From 3b511680301825dde922c2eb51e36357384644c3 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 18 Apr 2024 18:02:26 +0000 Subject: [PATCH 7/7] Fix style --- ydb/tests/fq/generic/utils/settings.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ydb/tests/fq/generic/utils/settings.py b/ydb/tests/fq/generic/utils/settings.py index ec430695102d..19debc78b174 100644 --- a/ydb/tests/fq/generic/utils/settings.py +++ b/ydb/tests/fq/generic/utils/settings.py @@ -91,11 +91,7 @@ def from_env(cls) -> 'Settings': username='user', password='password', ), - ydb=cls.Ydb( - dbname='local', - username='user', - password='password' - ) + ydb=cls.Ydb(dbname='local', username='user', password='password'), ) return s