From 1bc1712497b647e0bc394afbeb5a60d0faf825cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3?= Date: Wed, 7 Feb 2024 14:34:43 +0300 Subject: [PATCH 1/3] =?UTF-8?q?KIKIMR-21016:=20=D0=9F=D1=80=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D0=B2=D0=B8=D1=82=D1=8C=20=D0=B2=D0=B5=D1=80=D1=81?= =?UTF-8?q?=D0=B8=D1=8E=20SSA=20Runtime=20=D0=B2=2024-1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ydb/core/formats/arrow/ssa_runtime_version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/formats/arrow/ssa_runtime_version.h b/ydb/core/formats/arrow/ssa_runtime_version.h index 5b6559980b52..7f9b78aa760a 100644 --- a/ydb/core/formats/arrow/ssa_runtime_version.h +++ b/ydb/core/formats/arrow/ssa_runtime_version.h @@ -19,7 +19,7 @@ namespace NKikimr::NSsa { // Bump this version every time incompatible runtime functions are introduced. #ifndef SSA_RUNTIME_VERSION -#define SSA_RUNTIME_VERSION 4U +#define SSA_RUNTIME_VERSION 3U #endif // History: From 80bd66946e33f3fe298a4760561fd16fcdf3c27b Mon Sep 17 00:00:00 2001 From: Tony-Romanov <150126326+Tony-Romanov@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:28:01 +0100 Subject: [PATCH 2/3] Fix build and behavior with SSA runtime < 4 (#1634) * Fix build with SSA runtime < 4 * Fix behavior and test on version 3 --- .../opt/physical/kqp_opt_phy_olap_filter.cpp | 1 + .../kqp/query_compiler/kqp_olap_compiler.cpp | 38 +++++++++++++++---- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 4 ++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp index de70e17d40f1..40a07ed8a083 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp @@ -140,6 +140,7 @@ std::vector> ExtractComparisonParameters(const T TMaybeNode ComparisonPushdown(const std::vector>& parameters, const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos); +[[maybe_unused]] TMaybeNode YqlCoalescePushdown(const TCoCoalesce& coalesce, TExprContext& ctx) { if (const auto params = ExtractBinaryFunctionParameters(coalesce, ctx, coalesce.Pos())) { return Build(ctx, coalesce.Pos()) diff --git a/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp b/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp index 99020fee6c3c..bd9336adb041 100644 --- a/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp +++ b/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp @@ -489,6 +489,12 @@ ui64 CompileSimpleArrowComparison(const TKqpOlapFilterBinaryOp& comparison, TKqp function = TProgram::TAssignment::FUNC_CMP_GREATER; } else if (comparison.Operator() == "gte") { function = TProgram::TAssignment::FUNC_CMP_GREATER_EQUAL; + } else if (comparison.Operator() == "string_contains") { + function = TProgram::TAssignment::FUNC_STR_MATCH; + } else if (comparison.Operator() == "starts_with") { + function = TProgram::TAssignment::FUNC_STR_STARTS_WITH; + } else if (comparison.Operator() == "ends_with") { + function = TProgram::TAssignment::FUNC_STR_ENDS_WITH; } cmpFunc->SetId(function); @@ -620,6 +626,7 @@ TTypedColumn CompileYqlKernelUnaryOperation(const TKqpOlapFilterUnaryOp& operati return {command->GetColumn().GetId(), resultType}; } +[[maybe_unused]] TTypedColumn CompileYqlKernelBinaryOperation(const TKqpOlapFilterBinaryOp& operation, TKqpOlapCompileContext& ctx) { // Columns should be created before operation, otherwise operation fail to find columns @@ -708,7 +715,6 @@ const TTypedColumn BuildLogicalProgram(const TExprNode::TChildrenType& args, con logicalFunc->SetFunctionType(TProgram::YQL_KERNEL); logicalFunc->SetYqlOperationId((ui32)function); } else { - logicalFunc->SetFunctionType(function); logicalFunc->SetId((ui32)function); } @@ -740,22 +746,38 @@ const TTypedColumn BuildLogicalNot(const TExprBase& arg, TKqpOlapCompileContext& TTypedColumn GetOrCreateColumnIdAndType(const TExprBase& node, TKqpOlapCompileContext& ctx) { if (const auto& maybeBinaryOp = node.Maybe()) { - if (const auto& binaryOp = maybeBinaryOp.Cast(); ctx.CheckYqlCompatibleArgsTypes(binaryOp)) { - return CompileYqlKernelBinaryOperation(binaryOp, ctx); + if constexpr (NSsa::RuntimeVersion >= 4U) { + if (const auto& binaryOp = maybeBinaryOp.Cast(); ctx.CheckYqlCompatibleArgsTypes(binaryOp)) { + return CompileYqlKernelBinaryOperation(binaryOp, ctx); + } else { + return { + ConvertSafeCastToColumn(CompileSimpleArrowComparison(binaryOp, ctx), "Uint8", ctx), + ctx.ExprCtx().MakeType(ctx.ExprCtx().MakeType(EDataSlot::Bool)) + }; + } } else { return { - ConvertSafeCastToColumn(CompileSimpleArrowComparison(binaryOp, ctx), "Uint8", ctx), - ctx.ExprCtx().MakeType(ctx.ExprCtx().MakeType(EDataSlot::Uint8)) + CompileSimpleArrowComparison(maybeBinaryOp.Cast(), ctx), + ctx.ExprCtx().MakeType(ctx.ExprCtx().MakeType(EDataSlot::Bool)) }; } } else if (const auto& maybeUnaryOp = node.Maybe()) { return CompileYqlKernelUnaryOperation(maybeUnaryOp.Cast(), ctx); } else if (const auto& maybeAnd = node.Maybe()) { - return BuildLogicalProgram(maybeAnd.Ref().Children(), TKernelRequestBuilder::EBinaryOp::And, ctx); + if constexpr (NSsa::RuntimeVersion >= 4U) + return BuildLogicalProgram(maybeAnd.Ref().Children(), TKernelRequestBuilder::EBinaryOp::And, ctx); + else + return BuildLogicalProgram(maybeAnd.Ref().Children(), TProgram::TAssignment::FUNC_BINARY_AND, ctx); } else if (const auto& maybeOr = node.Maybe()) { - return BuildLogicalProgram(maybeOr.Ref().Children(), TKernelRequestBuilder::EBinaryOp::Or, ctx); + if constexpr (NSsa::RuntimeVersion >= 4U) + return BuildLogicalProgram(maybeOr.Ref().Children(), TKernelRequestBuilder::EBinaryOp::Or, ctx); + else + return BuildLogicalProgram(maybeOr.Ref().Children(), TProgram::TAssignment::FUNC_BINARY_OR, ctx); } else if (const auto& maybeXor = node.Maybe()) { - return BuildLogicalProgram(maybeXor.Ref().Children(), TKernelRequestBuilder::EBinaryOp::Xor, ctx); + if constexpr (NSsa::RuntimeVersion >= 4U) + return BuildLogicalProgram(maybeXor.Ref().Children(), TKernelRequestBuilder::EBinaryOp::Xor, ctx); + else + return BuildLogicalProgram(maybeXor.Ref().Children(), TProgram::TAssignment::FUNC_BINARY_XOR, ctx); } else if (const auto& maybeNot = node.Maybe()) { return BuildLogicalNot(maybeNot.Cast().Value(), ctx); } else if (const auto& maybeJsonValue = node.Maybe()) { diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 6f53286948f2..76a94877e767 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -1709,7 +1709,11 @@ Y_UNIT_TEST_SUITE(KqpOlap) { scanSettings.Explain(true); TLocalHelper(kikimr).CreateTestOlapTable(); +#if SSA_RUNTIME_VERSION >= 4U WriteTestData(kikimr, "/Root/olapStore/olapTable", 10000, 3000000, 5, true); +#else + WriteTestData(kikimr, "/Root/olapStore/olapTable", 10000, 3000000, 5, false); +#endif Tests::NCommon::TLoggerInit(kikimr).Initialize(); auto tableClient = kikimr.GetTableClient(); From 7b5da837f9ddd27abe859a0696ab0c01c577dca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3?= Date: Sun, 11 Feb 2024 16:36:30 +0300 Subject: [PATCH 3/3] fix test --- ydb/core/kqp/ut/query/kqp_explain_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp index 0a101ca68902..e84fa7c3cc9c 100644 --- a/ydb/core/kqp/ut/query/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_explain_ut.cpp @@ -855,7 +855,11 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::ReadJsonTree(*streamRes.PlanJson, &plan, true); UNIT_ASSERT(ValidatePlanNodeIds(plan)); +#if SSA_RUNTIME_VERSION >= 4U auto readNode = FindPlanNodeByKv(plan, "Node Type", "TableFullScan"); +#else + auto readNode = FindPlanNodeByKv(plan, "Node Type", "Filter-TableFullScan"); +#endif UNIT_ASSERT(readNode.IsDefined()); auto& operators = readNode.GetMapSafe().at("Operators").GetArraySafe();