From f854433772dc3b37b8a60d7c364579cb9899d753 Mon Sep 17 00:00:00 2001 From: Aleksei Pozdniakov Date: Tue, 15 Oct 2024 16:54:30 +0000 Subject: [PATCH] add world to external source settings --- .../expr_nodes/yql_clickhouse_expr_nodes.json | 7 +++--- .../yql_clickhouse_datasource_type_ann.cpp | 6 ++++- .../yql_clickhouse_dq_integration.cpp | 1 + .../expr_nodes/yql_generic_expr_nodes.json | 11 ++++---- .../yql_generic_datasource_type_ann.cpp | 6 ++++- .../provider/yql_generic_dq_integration.cpp | 1 + .../pq/expr_nodes/yql_pq_expr_nodes.json | 11 ++++---- .../provider/yql_pq_datasource_type_ann.cpp | 7 +++++- .../pq/provider/yql_pq_dq_integration.cpp | 1 + .../s3/expr_nodes/yql_s3_expr_nodes.json | 23 +++++++++-------- .../provider/yql_s3_datasource_type_ann.cpp | 12 +++++++-- .../s3/provider/yql_s3_dq_integration.cpp | 2 ++ .../expr_nodes/yql_solomon_expr_nodes.json | 25 ++++++++++--------- .../yql_solomon_datasource_type_ann.cpp | 6 ++++- .../provider/yql_solomon_dq_integration.cpp | 1 + .../ydb/expr_nodes/yql_ydb_expr_nodes.json | 7 +++--- .../provider/yql_ydb_datasource_type_ann.cpp | 6 ++++- .../ydb/provider/yql_ydb_dq_integration.cpp | 1 + 18 files changed, 88 insertions(+), 46 deletions(-) diff --git a/ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.json b/ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.json index a3378a39810d..358d2e112b8b 100644 --- a/ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.json +++ b/ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.json @@ -41,9 +41,10 @@ "Base": "TCallable", "Match": {"Type": "Callable", "Name": "ClSourceSettings"}, "Children": [ - {"Index": 0, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Token", "Type": "TCoSecureParam"}, - {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"} + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"}, + {"Index": 3, "Name": "Columns", "Type": "TCoAtomList"} ] } ] diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp index 23e0e3d1b1e8..1c52e0764dfc 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource_type_ann.cpp @@ -25,7 +25,11 @@ class TClickHouseDataSourceTypeAnnotationTransformer : public TVisitorTransforme } TStatus HandleSourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureArgsCount(*input, 3U, ctx)) { + if (!EnsureArgsCount(*input, 4, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TClSourceSettings::idx_World), ctx)) { return TStatus::Error; } diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp index 0f997f1e4495..f9d370bdacbd 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp @@ -52,6 +52,7 @@ class TClickHouseDqIntegration: public TDqIntegrationBase { return Build(ctx, read->Pos()) .Input() + .World(clReadTable.World()) .Table(clReadTable.Table()) .Token() .Name().Build(token) diff --git a/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json b/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json index 88e63d15833d..90652bc3037e 100644 --- a/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json +++ b/ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.json @@ -41,11 +41,12 @@ "Base": "TCallable", "Match": {"Type": "Callable", "Name": "GenSourceSettings"}, "Children": [ - {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"}, - {"Index": 3, "Name": "Columns", "Type": "TCoAtomList"}, - {"Index": 4, "Name": "FilterPredicate", "Type": "TCoLambda"} + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Token", "Type": "TCoSecureParam"}, + {"Index": 4, "Name": "Columns", "Type": "TCoAtomList"}, + {"Index": 5, "Name": "FilterPredicate", "Type": "TCoLambda"} ] } ] diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp index 791c9ecb7f8a..78d89b89d8ab 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_datasource_type_ann.cpp @@ -49,7 +49,11 @@ namespace NYql { } TStatus HandleSourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureArgsCount(*input, 5, ctx)) { + if (!EnsureArgsCount(*input, 6, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TGenSourceSettings::idx_World), ctx)) { return TStatus::Error; } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp index 777cf1b6c7c3..88001bf7a9e6 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp @@ -87,6 +87,7 @@ namespace NYql { // clang-format off return Build(ctx, read->Pos()) .Input() + .World(genReadTable.World()) .Cluster(genReadTable.DataSource().Cluster()) .Table(genReadTable.Table()) .Token() diff --git a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json index 0b178695aaeb..d70cc37688f4 100644 --- a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json +++ b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json @@ -67,11 +67,12 @@ "Base": "TCallable", "Match": {"Type": "Callable", "Name": "DqPqTopicSource"}, "Children": [ - {"Index": 0, "Name": "Topic", "Type": "TPqTopic"}, - {"Index": 1, "Name": "Columns", "Type": "TExprBase"}, - {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"}, - {"Index": 3, "Name": "Token", "Type": "TCoSecureParam"}, - {"Index": 4, "Name": "FilterPredicate", "Type": "TCoLambda"} + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "Topic", "Type": "TPqTopic"}, + {"Index": 2, "Name": "Columns", "Type": "TExprBase"}, + {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}, + {"Index": 4, "Name": "Token", "Type": "TCoSecureParam"}, + {"Index": 5, "Name": "FilterPredicate", "Type": "TCoLambda"} ] }, { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp index ad404949ed03..2e2a458d72b6 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp @@ -132,11 +132,16 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { } TStatus HandleDqTopicSource(TExprBase input, TExprContext& ctx) { - if (!EnsureArgsCount(input.Ref(), 5, ctx)) { + if (!EnsureArgsCount(input.Ref(), 6, ctx)) { return TStatus::Error; } TDqPqTopicSource topicSource = input.Cast(); + + if (!EnsureWorldType(topicSource.World().Ref(), ctx)) { + return TStatus::Error; + } + TPqTopic topic = topicSource.Topic(); if (!EnsureCallable(topic.Ref(), ctx)) { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp index 3a305edaf59d..530bda256dc0 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp @@ -147,6 +147,7 @@ class TPqDqIntegration: public TDqIntegrationBase { return Build(ctx, read->Pos()) .Input() + .World(pqReadTopic.World()) .Topic(pqReadTopic.Topic()) .Columns(std::move(columnNames)) .Settings(BuildTopicReadSettings(clusterName, dqSettings, read->Pos(), format, ctx)) diff --git a/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json b/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json index 631e07dd3912..f7121ceaf650 100644 --- a/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json +++ b/ydb/library/yql/providers/s3/expr_nodes/yql_s3_expr_nodes.json @@ -45,10 +45,11 @@ "Base": "TCallable", "Match": {"Type": "CallableBase"}, "Children": [ - {"Index": 0, "Name": "Paths", "Type": "TS3Paths"}, - {"Index": 1, "Name": "Token", "Type": "TCoSecureParam"}, - {"Index": 2, "Name": "RowsLimitHint", "Type": "TCoAtom"}, - {"Index": 3, "Name": "Path", "Type": "TCoAtom"} + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "Paths", "Type": "TS3Paths"}, + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"}, + {"Index": 3, "Name": "RowsLimitHint", "Type": "TCoAtom"}, + {"Index": 4, "Name": "Path", "Type": "TCoAtom"} ] }, { @@ -56,9 +57,9 @@ "Base": "TS3SourceSettingsBase", "Match": {"Type": "Callable", "Name": "S3SourceSettings"}, "Children": [ - {"Index": 4, "Name": "SizeLimit", "Type": "TExprBase", "Optional": true}, - {"Index": 5, "Name": "PathPattern", "Type": "TExprBase", "Optional": true}, - {"Index": 6, "Name": "PathPatternVariant", "Type": "TExprBase", "Optional": true} + {"Index": 5, "Name": "SizeLimit", "Type": "TExprBase", "Optional": true}, + {"Index": 6, "Name": "PathPattern", "Type": "TExprBase", "Optional": true}, + {"Index": 7, "Name": "PathPatternVariant", "Type": "TExprBase", "Optional": true} ] }, { @@ -66,10 +67,10 @@ "Base": "TS3SourceSettingsBase", "Match": {"Type": "Callable", "Name": "S3ParseSettings"}, "Children": [ - {"Index": 4, "Name": "Format", "Type": "TCoAtom"}, - {"Index": 5, "Name": "RowType", "Type": "TExprBase"}, - {"Index": 6, "Name": "FilterPredicate", "Type": "TCoLambda"}, - {"Index": 7, "Name": "Settings", "Type": "TExprBase", "Optional": true} + {"Index": 5, "Name": "Format", "Type": "TCoAtom"}, + {"Index": 6, "Name": "RowType", "Type": "TExprBase"}, + {"Index": 7, "Name": "FilterPredicate", "Type": "TCoLambda"}, + {"Index": 8, "Name": "Settings", "Type": "TExprBase", "Optional": true} ] }, { diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp index 838d522b0b64..246c2292c5ed 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp @@ -303,7 +303,11 @@ class TS3DataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { } TStatus HandleS3SourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureMinArgsCount(*input, 4U, ctx)) { + if (!EnsureMinMaxArgsCount(*input, 5, 8, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TS3SourceSettings::idx_World), ctx)) { return TStatus::Error; } @@ -335,7 +339,11 @@ class TS3DataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { } TStatus HandleS3ParseSettings(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureMinMaxArgsCount(*input, 7U, 8U, ctx)) { + if (!EnsureMinMaxArgsCount(*input, 8, 9, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TS3ParseSettings::idx_World), ctx)) { return TStatus::Error; } diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp index 9cc5b024c6ed..1beda207ff51 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp @@ -286,6 +286,7 @@ class TS3DqIntegration: public TDqIntegrationBase { if (const auto useCoro = State_->Configuration->SourceCoroActor.Get(); (!useCoro || *useCoro) && format != "raw" && format != "json_list") { return Build(ctx, read->Pos()) .Input() + .World(s3ReadObject.World()) .Paths(s3ReadObject.Object().Paths()) .Token() .Name().Build(token) @@ -331,6 +332,7 @@ class TS3DqIntegration: public TDqIntegrationBase { auto emptyNode = Build(ctx, read->Pos()).Done().Ptr(); return Build(ctx, read->Pos()) .Input() + .World(s3ReadObject.World()) .Paths(s3ReadObject.Object().Paths()) .Token() .Name().Build(token) diff --git a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json index a4db8e885df2..74050a30cc33 100644 --- a/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json +++ b/ydb/library/yql/providers/solomon/expr_nodes/yql_solomon_expr_nodes.json @@ -48,18 +48,19 @@ "Base": "TCallable", "Match": {"Type": "Callable", "Name": "SoSourceSettings"}, "Children": [ - {"Index": 0, "Name": "Project", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Token", "Type": "TCoSecureParam"}, - {"Index": 2, "Name": "RowType", "Type": "TExprBase"}, - {"Index": 3, "Name": "SystemColumns", "Type": "TCoAtomList"}, - {"Index": 4, "Name": "LabelNames", "Type": "TCoAtomList"}, - {"Index": 5, "Name": "From", "Type": "TCoAtom"}, - {"Index": 6, "Name": "To", "Type": "TCoAtom"}, - {"Index": 7, "Name": "Program", "Type": "TCoAtom"}, - {"Index": 8, "Name": "DownsamplingDisabled", "Type": "TCoBool"}, - {"Index": 9, "Name": "DownsamplingAggregation", "Type": "TCoAtom"}, - {"Index": 10, "Name": "DownsamplingFill", "Type": "TCoAtom"}, - {"Index": 11, "Name": "DownsamplingGridSec", "Type": "TCoUint32"} + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "Project", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"}, + {"Index": 3, "Name": "RowType", "Type": "TExprBase"}, + {"Index": 4, "Name": "SystemColumns", "Type": "TCoAtomList"}, + {"Index": 5, "Name": "LabelNames", "Type": "TCoAtomList"}, + {"Index": 6, "Name": "From", "Type": "TCoAtom"}, + {"Index": 7, "Name": "To", "Type": "TCoAtom"}, + {"Index": 8, "Name": "Program", "Type": "TCoAtom"}, + {"Index": 9, "Name": "DownsamplingDisabled", "Type": "TCoBool"}, + {"Index": 10, "Name": "DownsamplingAggregation", "Type": "TCoAtom"}, + {"Index": 11, "Name": "DownsamplingFill", "Type": "TCoAtom"}, + {"Index": 12, "Name": "DownsamplingGridSec", "Type": "TCoUint32"} ] }, { diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp index 72fe1df5caa4..c09a053189bb 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasource_type_ann.cpp @@ -34,7 +34,11 @@ class TSolomonDataSourceTypeAnnotationTransformer : public TVisitorTransformerBa } TStatus HandleSoSourceSettings(const TExprNode::TPtr& input, TExprContext& ctx) { - if (!EnsureArgsCount(*input, 12U, ctx)) { + if (!EnsureArgsCount(*input, 13, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TSoSourceSettings::idx_World), ctx)) { return TStatus::Error; } diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 7cf47fa189b4..af4a567e6d2d 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -200,6 +200,7 @@ class TSolomonDqIntegration: public TDqIntegrationBase { return Build(ctx, read->Pos()) .Input() + .World(soReadObject.World()) .Project(soReadObject.Object().Project()) .Token() .Name().Build(token) diff --git a/ydb/library/yql/providers/ydb/expr_nodes/yql_ydb_expr_nodes.json b/ydb/library/yql/providers/ydb/expr_nodes/yql_ydb_expr_nodes.json index 534f097ec39d..62557c87f9a8 100644 --- a/ydb/library/yql/providers/ydb/expr_nodes/yql_ydb_expr_nodes.json +++ b/ydb/library/yql/providers/ydb/expr_nodes/yql_ydb_expr_nodes.json @@ -30,9 +30,10 @@ "Base": "TCallable", "Match": {"Type": "Callable", "Name": "YdbSourceSettings"}, "Children": [ - {"Index": 0, "Name": "Table", "Type": "TCoAtom"}, - {"Index": 1, "Name": "Token", "Type": "TCoSecureParam"}, - {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"} + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam"}, + {"Index": 3, "Name": "Columns", "Type": "TCoAtomList"} ] }, { diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource_type_ann.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource_type_ann.cpp index 466d4aa36c60..e074f7e9117c 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_datasource_type_ann.cpp @@ -29,7 +29,11 @@ class TYdbDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { TStatus HandleYdbSourceSettings(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 3U, ctx)) { + if (!EnsureArgsCount(*input, 4, ctx)) { + return TStatus::Error; + } + + if (!EnsureWorldType(*input->Child(TYdbSourceSettings::idx_World), ctx)) { return TStatus::Error; } diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp index 7ff7fc1c5cc7..9f165f0ba32c 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp @@ -101,6 +101,7 @@ class TYdbDqIntegration: public TDqIntegrationBase { return Build(ctx, read->Pos()) .Input() + .World(ydbReadTable.World()) .Table(ydbReadTable.Table()) .Token() .Name().Build(token)