From a57e612ff6cc192e4d5f94db0c0273ee78af82b4 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 11:05:07 +0000 Subject: [PATCH 1/3] YDB FQ: set up filtering options for YQL Generic Read Actor and Lookup Actor --- .../yql/providers/generic/actors/yql_generic_lookup_actor.cpp | 1 + .../yql/providers/generic/actors/yql_generic_read_actor.cpp | 1 + .../connector/tests/datasource/clickhouse/docker-compose.yml | 2 +- .../connector/tests/datasource/ms_sql_server/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/mysql/docker-compose.yml | 2 +- .../connector/tests/datasource/oracle/docker-compose.yml | 2 +- .../connector/tests/datasource/postgresql/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/ydb/docker-compose.yml | 2 +- .../providers/generic/connector/tests/join/docker-compose.yml | 2 +- ydb/tests/fq/generic/analytics/docker-compose.yml | 2 +- ydb/tests/fq/generic/streaming/docker-compose.yml | 2 +- 11 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index eba5f8a1f207..90db928a8f2b 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -168,6 +168,7 @@ namespace NYql::NDq { *readRequest.add_splits() = split; readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING); + readRequest.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) { YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; auto result = ExtractFromConstFuture(asyncResult); diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index c74f7a9b4b4a..031eac3a069b 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -199,6 +199,7 @@ namespace NYql::NDq { // Prepare request NConnector::NApi::TReadSplitsRequest request; request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); + request.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); request.mutable_splits()->Reserve(Splits_.size()); for (const auto& split : Splits_) { diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml index a459f1239505..1ae3fd60aa54 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml @@ -16,7 +16,7 @@ services: - /var fq-connector-go: container_name: fq-tests-ch-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml index b65e7c70dc06..43f328927139 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mssql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml index 8ba878edf6b3..4971bacac70b 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mysql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml index a2ecbd99242d..f4a815b74448 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-oracle-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml index 6d64be978bc7..53562fc7fd1c 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-pg-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml index b97b90b21151..b2922cb72451 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml @@ -8,7 +8,7 @@ services: dnsmasq; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: fq-tests-ydb-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml index 6f8acd4ace53..992d594daaf6 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-join-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - 2130 volumes: diff --git a/ydb/tests/fq/generic/analytics/docker-compose.yml b/ydb/tests/fq/generic/analytics/docker-compose.yml index ceb5ea309884..f702857495a2 100644 --- a/ydb/tests/fq/generic/analytics/docker-compose.yml +++ b/ydb/tests/fq/generic/analytics/docker-compose.yml @@ -19,7 +19,7 @@ services: echo \"$$(dig tests-fq-generic-analytics-ydb +short) tests-fq-generic-analytics-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-analytics-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - "2130" greenplum: diff --git a/ydb/tests/fq/generic/streaming/docker-compose.yml b/ydb/tests/fq/generic/streaming/docker-compose.yml index c77865cccb98..2108a4b67504 100644 --- a/ydb/tests/fq/generic/streaming/docker-compose.yml +++ b/ydb/tests/fq/generic/streaming/docker-compose.yml @@ -5,7 +5,7 @@ services: echo \"$$(dig tests-fq-generic-streaming-ydb +short) tests-fq-generic-streaming-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-streaming-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.10@sha256:82ab2332b049113058488581fbf0328639fe192aacf30bbf106e40f6bd297554 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498 ports: - "2130" ydb: From 9d9d0abf33b4c86f4c754d3c81f47c2faaa196f6 Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 11:09:12 +0000 Subject: [PATCH 2/3] Minor changes --- .../yql/providers/generic/actors/yql_generic_read_actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index 031eac3a069b..f3b32eb2d27a 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -199,7 +199,7 @@ namespace NYql::NDq { // Prepare request NConnector::NApi::TReadSplitsRequest request; request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING); - request.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); + request.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL); request.mutable_splits()->Reserve(Splits_.size()); for (const auto& split : Splits_) { From 12bae762271b454165484b98b26c45cb9268a0de Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Mon, 7 Oct 2024 11:18:00 +0000 Subject: [PATCH 3/3] Minor changes --- .../yql/providers/generic/actors/yql_generic_lookup_actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index 90db928a8f2b..25c809c63d79 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -168,7 +168,7 @@ namespace NYql::NDq { *readRequest.add_splits() = split; readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING); - readRequest.set_filtering(::NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); + readRequest.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) { YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; auto result = ExtractFromConstFuture(asyncResult);