From 502820a5f44da825a3b2a88e804f15a0d59a9475 Mon Sep 17 00:00:00 2001 From: Igor Munkin Date: Mon, 15 Apr 2024 19:28:12 +0500 Subject: [PATCH] YQL-18053: Add block implementation for Member callable (#3461) --- .../yql_opt_peephole_physical.cpp | 2 +- .../yql/core/type_ann/type_ann_blocks.cpp | 68 ++++++++++ .../yql/core/type_ann/type_ann_blocks.h | 1 + .../yql/core/type_ann/type_ann_core.cpp | 1 + .../minikql/comp_nodes/mkql_block_getelem.cpp | 119 ++++++++++++++++++ .../minikql/comp_nodes/mkql_block_getelem.h | 11 ++ .../minikql/comp_nodes/mkql_block_tuple.cpp | 86 ------------- .../yql/minikql/comp_nodes/mkql_block_tuple.h | 1 - .../yql/minikql/comp_nodes/mkql_factory.cpp | 2 + .../yql/minikql/comp_nodes/ya.make.inc | 1 + .../yql/minikql/mkql_program_builder.cpp | 18 +++ .../yql/minikql/mkql_program_builder.h | 1 + .../common/mkql/yql_provider_mkql.cpp | 6 + .../sql/dq_file/part11/canondata/result.json | 22 ++++ .../hybrid_file/part10/canondata/result.json | 14 +++ .../tests/sql/sql2yql/canondata/result.json | 14 +++ .../yql/tests/sql/suites/blocks/member.cfg | 1 + .../yql/tests/sql/suites/blocks/member.sql | 7 ++ .../part11/canondata/result.json | 28 +++++ 19 files changed, 315 insertions(+), 88 deletions(-) create mode 100644 ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.cpp create mode 100644 ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.h create mode 100644 ydb/library/yql/tests/sql/suites/blocks/member.cfg create mode 100644 ydb/library/yql/tests/sql/suites/blocks/member.sql diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp index 3f0da930973e..f6c2a2362458 100644 --- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -5468,7 +5468,7 @@ bool CollectBlockRewrites(const TMultiExprType* multiInputType, bool keepInputCo TExprNode::TListType funcArgs; std::string_view arrowFunctionName; - if (node->IsList() || node->IsCallable({"And", "Or", "Xor", "Not", "Coalesce", "Exists", "If", "Just", "Nth", "ToPg", "FromPg", "PgResolvedCall", "PgResolvedOp"})) + if (node->IsList() || node->IsCallable({"And", "Or", "Xor", "Not", "Coalesce", "Exists", "If", "Just", "Member", "Nth", "ToPg", "FromPg", "PgResolvedCall", "PgResolvedOp"})) { if (node->IsCallable() && !IsSupportedAsBlockType(node->Pos(), *node->GetTypeAnn(), ctx, types)) { return true; diff --git a/ydb/library/yql/core/type_ann/type_ann_blocks.cpp b/ydb/library/yql/core/type_ann/type_ann_blocks.cpp index 13ff78fb8f7b..537605305c2b 100644 --- a/ydb/library/yql/core/type_ann/type_ann_blocks.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_blocks.cpp @@ -1,4 +1,5 @@ #include "type_ann_blocks.h" +#include "type_ann_impl.h" #include "type_ann_list.h" #include "type_ann_wide.h" #include "type_ann_pg.h" @@ -429,6 +430,73 @@ IGraphTransformer::TStatus BlockAsTupleWrapper(const TExprNode::TPtr& input, TEx return IGraphTransformer::TStatus::Ok; } +IGraphTransformer::TStatus BlockMemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + Y_UNUSED(output); + if (!EnsureArgsCount(*input, 2, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto& child = input->Head(); + if (!EnsureBlockOrScalarType(child, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + bool isScalar; + const TTypeAnnotationNode* blockItemType = GetBlockItemType(*child.GetTypeAnn(), isScalar); + const TTypeAnnotationNode* resultType; + if (IsNull(*blockItemType)) { + resultType = blockItemType; + } else { + const TStructExprType* structType; + bool isOptional; + if (blockItemType->GetKind() == ETypeAnnotationKind::Optional) { + auto itemType = blockItemType->Cast()->GetItemType(); + if (!EnsureStructType(child.Pos(), *itemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + structType = itemType->Cast(); + isOptional = true; + } else { + if (!EnsureStructType(child.Pos(), *blockItemType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + structType = blockItemType->Cast(); + isOptional = false; + } + + if (!EnsureComputableType(input->Head().Pos(), *structType, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!EnsureAtom(input->Tail(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + auto memberName = input->Tail().Content(); + auto pos = FindOrReportMissingMember(memberName, input->Pos(), *structType, ctx.Expr); + if (!pos) { + return IGraphTransformer::TStatus::Error; + } + + resultType = structType->GetItems()[*pos]->GetItemType(); + if (isOptional && !resultType->IsOptionalOrNull()) { + resultType = ctx.Expr.MakeType(resultType); + } + } + + if (isScalar) { + resultType = ctx.Expr.MakeType(resultType); + } else { + resultType = ctx.Expr.MakeType(resultType); + } + + input->SetTypeAnn(resultType); + return IGraphTransformer::TStatus::Ok; +} + + IGraphTransformer::TStatus BlockNthWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 2, ctx.Expr)) { diff --git a/ydb/library/yql/core/type_ann/type_ann_blocks.h b/ydb/library/yql/core/type_ann/type_ann_blocks.h index 49ed4346478a..acd3883b156f 100644 --- a/ydb/library/yql/core/type_ann/type_ann_blocks.h +++ b/ydb/library/yql/core/type_ann/type_ann_blocks.h @@ -20,6 +20,7 @@ namespace NTypeAnnImpl { IGraphTransformer::TStatus BlockJustWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockAsTupleWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockNthWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); + IGraphTransformer::TStatus BlockMemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockToPgWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockFromPgWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus BlockFuncWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index 6592e8501c3e..cdaf82843d8d 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -12239,6 +12239,7 @@ template Functions["BlockIf"] = &BlockIfWrapper; Functions["BlockJust"] = &BlockJustWrapper; Functions["BlockAsTuple"] = &BlockAsTupleWrapper; + Functions["BlockMember"] = &BlockMemberWrapper; Functions["BlockNth"] = &BlockNthWrapper; Functions["BlockToPg"] = &BlockToPgWrapper; Functions["BlockFromPg"] = &BlockFromPgWrapper; diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.cpp new file mode 100644 index 000000000000..27e5e86be0ec --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.cpp @@ -0,0 +1,119 @@ +#include "mkql_block_getelem.h" + +#include +#include +#include + +namespace NKikimr { +namespace NMiniKQL { + +namespace { + +class TBlockGetElementExec { +public: + TBlockGetElementExec(const std::shared_ptr& returnArrowType, ui32 index, bool isOptional, bool needExternalOptional) + : ReturnArrowType(returnArrowType) + , Index(index) + , IsOptional(isOptional) + , NeedExternalOptional(needExternalOptional) + {} + + arrow::Status Exec(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) const { + arrow::Datum inputDatum = batch.values[0]; + if (inputDatum.is_scalar()) { + if (inputDatum.scalar()->is_valid) { + const auto& structScalar = arrow::internal::checked_cast(*inputDatum.scalar()); + *res = arrow::Datum(structScalar.value[Index]); + } else { + *res = arrow::Datum(arrow::MakeNullScalar(ReturnArrowType)); + } + } else { + const auto& array = inputDatum.array(); + auto child = array->child_data[Index]; + if (NeedExternalOptional) { + auto newArrayData = arrow::ArrayData::Make(ReturnArrowType, array->length, { array->buffers[0] }); + newArrayData->child_data.push_back(child); + *res = arrow::Datum(newArrayData); + } else if (!IsOptional || !array->buffers[0]) { + *res = arrow::Datum(child); + } else { + auto newArrayData = child->Copy(); + if (!newArrayData->buffers[0]) { + newArrayData->buffers[0] = array->buffers[0]; + } else { + auto buffer = AllocateBitmapWithReserve(array->length + array->offset, ctx->memory_pool()); + arrow::internal::BitmapAnd(child->GetValues(0, 0), child->offset, array->GetValues(0, 0), array->offset, array->length, array->offset, buffer->mutable_data()); + newArrayData->buffers[0] = buffer; + } + + newArrayData->SetNullCount(arrow::kUnknownNullCount); + *res = arrow::Datum(newArrayData); + } + } + + return arrow::Status::OK(); + } + +private: + const std::shared_ptr ReturnArrowType; + const ui32 Index; + const bool IsOptional; + const bool NeedExternalOptional; +}; + +std::shared_ptr MakeBlockGetElementKernel(const TVector& argTypes, TType* resultType, + ui32 index, bool isOptional, bool needExternalOptional) { + std::shared_ptr returnArrowType; + MKQL_ENSURE(ConvertArrowType(AS_TYPE(TBlockType, resultType)->GetItemType(), returnArrowType), "Unsupported arrow type"); + auto exec = std::make_shared(returnArrowType, index, isOptional, needExternalOptional); + auto kernel = std::make_shared(ConvertToInputTypes(argTypes), ConvertToOutputType(resultType), + [exec](arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { + return exec->Exec(ctx, batch, res); + }); + + kernel->null_handling = arrow::compute::NullHandling::COMPUTED_NO_PREALLOCATE; + return kernel; +} + +TType* GetElementType(const TStructType* structType, ui32 index) { + MKQL_ENSURE(index < structType->GetMembersCount(), "Bad member index"); + return structType->GetMemberType(index); +} + +TType* GetElementType(const TTupleType* tupleType, ui32 index) { + MKQL_ENSURE(index < tupleType->GetElementsCount(), "Bad tuple index"); + return tupleType->GetElementType(index); +} + +template +IComputationNode* WrapBlockGetElement(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + MKQL_ENSURE(callable.GetInputsCount() == 2, "Expected two args."); + auto inputObject = callable.GetInput(0); + auto blockType = AS_TYPE(TBlockType, inputObject.GetStaticType()); + bool isOptional; + auto objectType = AS_TYPE(ObjectType, UnpackOptional(blockType->GetItemType(), isOptional)); + auto indexData = AS_VALUE(TDataLiteral, callable.GetInput(1)); + auto index = indexData->AsValue().Get(); + auto childType = GetElementType(objectType, index); + bool needExternalOptional = isOptional && childType->IsVariant(); + + auto objectNode = LocateNode(ctx.NodeLocator, callable, 0); + + TComputationNodePtrVector argsNodes = { objectNode }; + TVector argsTypes = { blockType }; + auto kernel = MakeBlockGetElementKernel(argsTypes, callable.GetType()->GetReturnType(), index, isOptional, needExternalOptional); + return new TBlockFuncNode(ctx.Mutables, callable.GetType()->GetName(), std::move(argsNodes), argsTypes, *kernel, kernel); +} + +} // namespace + +IComputationNode* WrapBlockMember(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + return WrapBlockGetElement(callable, ctx); +} + +IComputationNode* WrapBlockNth(TCallable& callable, const TComputationNodeFactoryContext& ctx) { + return WrapBlockGetElement(callable, ctx); +} + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.h b/ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.h new file mode 100644 index 000000000000..e0e360684504 --- /dev/null +++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_getelem.h @@ -0,0 +1,11 @@ +#pragma once +#include + +namespace NKikimr { +namespace NMiniKQL { + +IComputationNode* WrapBlockMember(TCallable& callable, const TComputationNodeFactoryContext& ctx); +IComputationNode* WrapBlockNth(TCallable& callable, const TComputationNodeFactoryContext& ctx); + +} // namespace NMiniKQL +} // namespace NKikimr diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp index 494b6ad2229c..1d0328502a8d 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp @@ -66,58 +66,6 @@ class TBlockAsTupleExec { const std::shared_ptr ReturnArrowType; }; -class TBlockNthExec { -public: - TBlockNthExec(const std::shared_ptr& returnArrowType, ui32 index, bool isOptional, bool needExternalOptional) - : ReturnArrowType(returnArrowType) - , Index(index) - , IsOptional(isOptional) - , NeedExternalOptional(needExternalOptional) - {} - - arrow::Status Exec(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) const { - arrow::Datum inputDatum = batch.values[0]; - if (inputDatum.is_scalar()) { - if (inputDatum.scalar()->is_valid) { - const auto& structScalar = arrow::internal::checked_cast(*inputDatum.scalar()); - *res = arrow::Datum(structScalar.value[Index]); - } else { - *res = arrow::Datum(arrow::MakeNullScalar(ReturnArrowType)); - } - } else { - const auto& array = inputDatum.array(); - auto child = array->child_data[Index]; - if (NeedExternalOptional) { - auto newArrayData = arrow::ArrayData::Make(ReturnArrowType, array->length, { array->buffers[0] }); - newArrayData->child_data.push_back(child); - *res = arrow::Datum(newArrayData); - } else if (!IsOptional || !array->buffers[0]) { - *res = arrow::Datum(child); - } else { - auto newArrayData = child->Copy(); - if (!newArrayData->buffers[0]) { - newArrayData->buffers[0] = array->buffers[0]; - } else { - auto buffer = AllocateBitmapWithReserve(array->length + array->offset, ctx->memory_pool()); - arrow::internal::BitmapAnd(child->GetValues(0, 0), child->offset, array->GetValues(0, 0), array->offset, array->length, array->offset, buffer->mutable_data()); - newArrayData->buffers[0] = buffer; - } - - newArrayData->SetNullCount(arrow::kUnknownNullCount); - *res = arrow::Datum(newArrayData); - } - } - - return arrow::Status::OK(); - } - -private: - const std::shared_ptr ReturnArrowType; - const ui32 Index; - const bool IsOptional; - const bool NeedExternalOptional; -}; - std::shared_ptr MakeBlockAsTupleKernel(const TVector& argTypes, TType* resultType) { std::shared_ptr returnArrowType; MKQL_ENSURE(ConvertArrowType(AS_TYPE(TBlockType, resultType)->GetItemType(), returnArrowType), "Unsupported arrow type"); @@ -131,20 +79,6 @@ std::shared_ptr MakeBlockAsTupleKernel(const TVect return kernel; } -std::shared_ptr MakeBlockNthKernel(const TVector& argTypes, TType* resultType, ui32 index, - bool isOptional, bool needExternalOptional) { - std::shared_ptr returnArrowType; - MKQL_ENSURE(ConvertArrowType(AS_TYPE(TBlockType, resultType)->GetItemType(), returnArrowType), "Unsupported arrow type"); - auto exec = std::make_shared(returnArrowType, index, isOptional, needExternalOptional); - auto kernel = std::make_shared(ConvertToInputTypes(argTypes), ConvertToOutputType(resultType), - [exec](arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { - return exec->Exec(ctx, batch, res); - }); - - kernel->null_handling = arrow::compute::NullHandling::COMPUTED_NO_PREALLOCATE; - return kernel; -} - } // namespace IComputationNode* WrapBlockAsTuple(TCallable& callable, const TComputationNodeFactoryContext& ctx) { @@ -159,25 +93,5 @@ IComputationNode* WrapBlockAsTuple(TCallable& callable, const TComputationNodeFa return new TBlockFuncNode(ctx.Mutables, callable.GetType()->GetName(), std::move(argsNodes), argsTypes, *kernel, kernel); } -IComputationNode* WrapBlockNth(TCallable& callable, const TComputationNodeFactoryContext& ctx) { - MKQL_ENSURE(callable.GetInputsCount() == 2U, "Expected two args."); - auto input = callable.GetInput(0U); - auto blockType = AS_TYPE(TBlockType, input.GetStaticType()); - bool isOptional; - auto tupleType = AS_TYPE(TTupleType, UnpackOptional(blockType->GetItemType(), isOptional)); - auto indexData = AS_VALUE(TDataLiteral, callable.GetInput(1U)); - auto index = indexData->AsValue().Get(); - MKQL_ENSURE(index < tupleType->GetElementsCount(), "Bad tuple index"); - auto childType = tupleType->GetElementType(index); - bool needExternalOptional = isOptional && childType->IsVariant(); - - auto tuple = LocateNode(ctx.NodeLocator, callable, 0); - - TComputationNodePtrVector argsNodes = { tuple }; - TVector argsTypes = { blockType }; - auto kernel = MakeBlockNthKernel(argsTypes, callable.GetType()->GetReturnType(), index, isOptional, needExternalOptional); - return new TBlockFuncNode(ctx.Mutables, callable.GetType()->GetName(), std::move(argsNodes), argsTypes, *kernel, kernel); -} - } } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.h b/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.h index 06e60f6e288c..b8acb251d4ba 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.h +++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.h @@ -5,7 +5,6 @@ namespace NKikimr { namespace NMiniKQL { IComputationNode* WrapBlockAsTuple(TCallable& callable, const TComputationNodeFactoryContext& ctx); -IComputationNode* WrapBlockNth(TCallable& callable, const TComputationNodeFactoryContext& ctx); } } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp index bb5c8546b714..6b723fba98ee 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp @@ -9,6 +9,7 @@ #include "mkql_block_agg.h" #include "mkql_block_coalesce.h" #include "mkql_block_exists.h" +#include "mkql_block_getelem.h" #include "mkql_block_if.h" #include "mkql_block_just.h" #include "mkql_block_logical.h" @@ -297,6 +298,7 @@ struct TCallableComputationNodeBuilderFuncMapFiller { {"BlockJust", &WrapBlockJust}, {"BlockCompress", &WrapBlockCompress}, {"BlockAsTuple", &WrapBlockAsTuple}, + {"BlockMember", &WrapBlockMember}, {"BlockNth", &WrapBlockNth}, {"BlockExpandChunked", &WrapBlockExpandChunked}, {"BlockCombineAll", &WrapBlockCombineAll}, diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make.inc b/ydb/library/yql/minikql/comp_nodes/ya.make.inc index 4aa7d230ccaa..e616b8e267b4 100644 --- a/ydb/library/yql/minikql/comp_nodes/ya.make.inc +++ b/ydb/library/yql/minikql/comp_nodes/ya.make.inc @@ -16,6 +16,7 @@ SET(ORIG_SOURCES mkql_block_agg_sum.cpp mkql_block_coalesce.cpp mkql_block_exists.cpp + mkql_block_getelem.cpp mkql_block_if.cpp mkql_block_just.cpp mkql_block_logical.cpp diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 6b4b3ee62c9f..172e95b0a49b 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -1620,6 +1620,24 @@ TRuntimeNode TProgramBuilder::BlockExists(TRuntimeNode data) { return TRuntimeNode(callableBuilder.Build(), false); } +TRuntimeNode TProgramBuilder::BlockMember(TRuntimeNode structObj, const std::string_view& memberName) { + auto blockType = AS_TYPE(TBlockType, structObj.GetStaticType()); + bool isOptional; + const auto type = AS_TYPE(TStructType, UnpackOptional(blockType->GetItemType(), isOptional)); + + const auto memberIndex = type->GetMemberIndex(memberName); + auto memberType = type->GetMemberType(memberIndex); + if (isOptional && !memberType->IsOptional() && !memberType->IsNull() && !memberType->IsPg()) { + memberType = NewOptionalType(memberType); + } + + auto returnType = NewBlockType(memberType, blockType->GetShape()); + TCallableBuilder callableBuilder(Env, __func__, returnType); + callableBuilder.Add(structObj); + callableBuilder.Add(NewDataLiteral(memberIndex)); + return TRuntimeNode(callableBuilder.Build(), false); +} + TRuntimeNode TProgramBuilder::BlockNth(TRuntimeNode tuple, ui32 index) { auto blockType = AS_TYPE(TBlockType, tuple.GetStaticType()); bool isOptional; diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 87008e7871bf..03bae6a55c15 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -237,6 +237,7 @@ class TProgramBuilder : public TTypeBuilder { TRuntimeNode BlockExpandChunked(TRuntimeNode flow); TRuntimeNode BlockCoalesce(TRuntimeNode first, TRuntimeNode second); TRuntimeNode BlockExists(TRuntimeNode data); + TRuntimeNode BlockMember(TRuntimeNode structure, const std::string_view& memberName); TRuntimeNode BlockNth(TRuntimeNode tuple, ui32 index); TRuntimeNode BlockAsTuple(const TArrayRef& args); TRuntimeNode BlockToPg(TRuntimeNode input, TType* returnType); diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp index 23c4f16c4932..a593f5d30df6 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp @@ -2725,6 +2725,12 @@ TMkqlCommonCallableCompiler::TShared::TShared() { return ctx.ProgramBuilder.BlockBitCast(arg, targetType); }); + AddCallable("BlockMember", [](const TExprNode& node, TMkqlBuildContext& ctx) { + const auto structObj = MkqlBuildExpr(node.Head(), ctx); + const auto name = node.Tail().Content(); + return ctx.ProgramBuilder.BlockMember(structObj, name); + }); + AddCallable("BlockNth", [](const TExprNode& node, TMkqlBuildContext& ctx) { const auto tupleObj = MkqlBuildExpr(node.Head(), ctx); const auto index = FromString(node.Tail().Content()); diff --git a/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json index e989993e4ea1..48c488f873f2 100644 --- a/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json @@ -533,6 +533,28 @@ } ], "test.test[blocks-filter_direct_col--Results]": [], + "test.test[blocks-member--Analyze]": [ + { + "checksum": "b08274fd137c1878d90520c832f06fd3", + "size": 3676, + "uri": "https://{canondata_backend}/1889210/75a1d72834c0a9de8b328ec130be934f6cc6cea0/resource.tar.gz#test.test_blocks-member--Analyze_/plan.txt" + } + ], + "test.test[blocks-member--Debug]": [ + { + "checksum": "a30b76ba380ee4f694dc731aa2771fed", + "size": 1467, + "uri": "https://{canondata_backend}/1937027/96028d31f8e29253c9276a86f02284e2a71add76/resource.tar.gz#test.test_blocks-member--Debug_/opt.yql_patched" + } + ], + "test.test[blocks-member--Plan]": [ + { + "checksum": "b08274fd137c1878d90520c832f06fd3", + "size": 3676, + "uri": "https://{canondata_backend}/1889210/75a1d72834c0a9de8b328ec130be934f6cc6cea0/resource.tar.gz#test.test_blocks-member--Plan_/plan.txt" + } + ], + "test.test[blocks-member--Results]": [], "test.test[blocks-pg_to_dates--Analyze]": [ { "checksum": "2a21c8668a5f5f9b4304777c6aa3c3ca", diff --git a/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json index 86afd1d29cde..4485ac9f0a09 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json @@ -671,6 +671,20 @@ "uri": "https://{canondata_backend}/1923547/14c0d60ad63ffaedb974b51b52039901f095b5c5/resource.tar.gz#test.test_blocks-finalize_hashed_keys--Plan_/plan.txt" } ], + "test.test[blocks-member--Debug]": [ + { + "checksum": "49012dc14c1d467d864bc5079aa4542f", + "size": 1982, + "uri": "https://{canondata_backend}/1597364/2a4f282ea286021ee8f9098fb8207324c7ebe684/resource.tar.gz#test.test_blocks-member--Debug_/opt.yql_patched" + } + ], + "test.test[blocks-member--Plan]": [ + { + "checksum": "794e5e7aaffc457f9e8f888953e1c89e", + "size": 4045, + "uri": "https://{canondata_backend}/1597364/07eb39555ae99a903261332d5998f32599b281fe/resource.tar.gz#test.test_blocks-member--Plan_/plan.txt" + } + ], "test.test[blocks-minmax_strings--Debug]": [ { "checksum": "f17fd002ee0286a709010b0cb6a5805e", diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index 7079e80882bc..b01f8b875724 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -3569,6 +3569,13 @@ "uri": "https://{canondata_backend}/1917492/bf361ba6aca9974e26535026fb228b74d0ff24ef/resource.tar.gz#test_sql2yql.test_blocks-if_/sql.yql" } ], + "test_sql2yql.test[blocks-member]": [ + { + "checksum": "88c9131d7ea3c80ecc268cb7f458fc86", + "size": 1230, + "uri": "https://{canondata_backend}/1937027/f015781f1ee8e2e10d049f80211c1f81b56abfb9/resource.tar.gz#test_sql2yql.test_blocks-member_/sql.yql" + } + ], "test_sql2yql.test[blocks-minmax_strings]": [ { "checksum": "a73aaced3247f72c7a4ddda05dbc53e0", @@ -21818,6 +21825,13 @@ "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_blocks-if_/formatted.sql" } ], + "test_sql_format.test[blocks-member]": [ + { + "checksum": "7fcb44569c4f84aee80ea32b5961e0ed", + "size": 146, + "uri": "https://{canondata_backend}/1937027/f015781f1ee8e2e10d049f80211c1f81b56abfb9/resource.tar.gz#test_sql_format.test_blocks-member_/formatted.sql" + } + ], "test_sql_format.test[blocks-minmax_strings]": [ { "checksum": "4670b0be936e794320d37534a739df3d", diff --git a/ydb/library/yql/tests/sql/suites/blocks/member.cfg b/ydb/library/yql/tests/sql/suites/blocks/member.cfg new file mode 100644 index 000000000000..ed506aaf28d2 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/blocks/member.cfg @@ -0,0 +1 @@ +in Input input_struct.txt diff --git a/ydb/library/yql/tests/sql/suites/blocks/member.sql b/ydb/library/yql/tests/sql/suites/blocks/member.sql new file mode 100644 index 000000000000..2367f16cdda5 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/blocks/member.sql @@ -0,0 +1,7 @@ +USE plato; +/* XXX: Enable UseBlocks pragma and provide input to trigger block execution. */ +pragma UseBlocks; + +SELECT + val.a as a, +FROM Input; diff --git a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json index 6736cf4ee2b6..ec922f253fde 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json @@ -548,6 +548,34 @@ "uri": "https://{canondata_backend}/1689644/8bfa47b4b6b4d6f6543bfef6c07a8937dfb0470d/resource.tar.gz#test.test_blocks-filter_direct_col--Results_/results.txt" } ], + "test.test[blocks-member--Debug]": [ + { + "checksum": "bc4498c90892a7f1e023ac57dbcbc16d", + "size": 1459, + "uri": "https://{canondata_backend}/1597364/82c77c5ee80726b7fecf97522a463a7edce9a1be/resource.tar.gz#test.test_blocks-member--Debug_/opt.yql" + } + ], + "test.test[blocks-member--Peephole]": [ + { + "checksum": "b4c9c617dd8d2fa940d076319eb8928d", + "size": 1343, + "uri": "https://{canondata_backend}/1597364/82c77c5ee80726b7fecf97522a463a7edce9a1be/resource.tar.gz#test.test_blocks-member--Peephole_/opt.yql" + } + ], + "test.test[blocks-member--Plan]": [ + { + "checksum": "082fc363e364c6bd7e557a48f4a982e4", + "size": 4537, + "uri": "https://{canondata_backend}/1937424/677956b7b1d51ac69cbef1b401a74f8916d215bb/resource.tar.gz#test.test_blocks-member--Plan_/plan.txt" + } + ], + "test.test[blocks-member--Results]": [ + { + "checksum": "2cbf207655f2d864cbcbda63073d76f5", + "size": 1282, + "uri": "https://{canondata_backend}/1597364/82c77c5ee80726b7fecf97522a463a7edce9a1be/resource.tar.gz#test.test_blocks-member--Results_/results.txt" + } + ], "test.test[blocks-pg_to_dates--Debug]": [ { "checksum": "93f519572585eb5003ddbd5d67797b48",