From 224330a8a6b6e9707f4fefde803082b7ad75bc5d Mon Sep 17 00:00:00 2001 From: Pavel Ivanov Date: Mon, 25 Nov 2024 16:42:46 +0300 Subject: [PATCH 1/3] [CBO] user warning added if cbo didn't work --- ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 5018e4a70644..b9ee3c002e5a 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -357,6 +357,7 @@ TExprBase DqOptimizeEquiJoinWithCosts( // of the EquiJoin and n-1 argument are the parameters to EquiJoin if (!DqCollectJoinRelationsWithStats(rels, typesCtx, equiJoin, providerCollect)){ + ctx.AddWarning(YqlIssue({}, TIssuesIds::WARNING, "Cost Based Optimizer didn't work: couldn't load statistics")); return node; } From b39d5f944a5de54a0d98f7d7e8bd41569a484fb5 Mon Sep 17 00:00:00 2001 From: Pavel Ivanov Date: Mon, 25 Nov 2024 16:47:33 +0300 Subject: [PATCH 2/3] [CBO] user warning added if cbo hadn't worked --- ydb/core/kqp/opt/logical/kqp_opt_log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index 3a3df0e38d48..fae010c24b94 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -94,7 +94,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { if (status == TStatus::Ok) { for (const auto& hint: KqpCtx.GetOptimizerHints().GetUnappliedString()) { - ctx.AddWarning(YqlIssue({}, TIssuesIds::YQL_UNUSED_HINT, "Unapplied hint: " + hint)); + ctx.AddWarning(YqlIssue({}, TIssuesIds::DQ_OPTIMIZE_ERROR, "Unapplied hint: " + hint)); } } From a04d54134b1a9ca518e0207a90c38eb8668d9db4 Mon Sep 17 00:00:00 2001 From: Pavel Ivanov Date: Mon, 25 Nov 2024 17:02:11 +0300 Subject: [PATCH 3/3] fix --- ydb/core/kqp/opt/logical/kqp_opt_log.cpp | 4 ++-- ydb/core/kqp/provider/yql_kikimr_settings.cpp | 2 +- ydb/core/kqp/provider/yql_kikimr_settings.h | 2 +- ydb/library/yql/dq/opt/dq_opt_log.h | 2 +- ydb/library/yql/providers/dq/common/yql_dq_settings.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index fae010c24b94..3a64525239a6 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -160,10 +160,10 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } TMaybeNode OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) { - auto maxDPccpDPTableSize = Config->MaxDPccpDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPccpDPTableSize); + auto maxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize); auto optLevel = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel); auto providerCtx = TKqpProviderContext(KqpCtx, optLevel); - auto opt = std::unique_ptr(MakeNativeOptimizerNew(providerCtx, maxDPccpDPTableSize)); + auto opt = std::unique_ptr(MakeNativeOptimizerNew(providerCtx, maxDPhypDPTableSize)); TExprBase output = DqOptimizeEquiJoinWithCosts(node, ctx, TypesCtx, optLevel, *opt, [](auto& rels, auto label, auto node, auto stat) { rels.emplace_back(std::make_shared(TString(label), *stat, node)); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.cpp b/ydb/core/kqp/provider/yql_kikimr_settings.cpp index e720d06977ab..65d960ee452c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_settings.cpp @@ -93,7 +93,7 @@ TKikimrConfiguration::TKikimrConfiguration() { REGISTER_SETTING(*this, EnableSpillingNodes) .Parser([](const TString& v) { return ParseEnableSpillingNodes(v); }); - REGISTER_SETTING(*this, MaxDPccpDPTableSize); + REGISTER_SETTING(*this, MaxDPHypDPTableSize); REGISTER_SETTING(*this, MaxTasksPerStage); REGISTER_SETTING(*this, MaxSequentialReadsInFlight); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 85c5a8829541..28bab1b62aba 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -69,7 +69,7 @@ struct TKikimrSettings { NCommon::TConfSetting OptUseFinalizeByKey; NCommon::TConfSetting CostBasedOptimizationLevel; - NCommon::TConfSetting MaxDPccpDPTableSize; + NCommon::TConfSetting MaxDPHypDPTableSize; NCommon::TConfSetting MaxTasksPerStage; diff --git a/ydb/library/yql/dq/opt/dq_opt_log.h b/ydb/library/yql/dq/opt/dq_opt_log.h index 34816f163c26..1b8a79fefb0b 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.h +++ b/ydb/library/yql/dq/opt/dq_opt_log.h @@ -63,7 +63,7 @@ IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPt NNodes::TExprBase DqExpandMatchRecognize(NNodes::TExprBase node, TExprContext& ctx, TTypeAnnotationContext& typeAnnCtx); -IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPccpDPTableSize); +IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize); NNodes::TMaybeNode UnorderedOverDqReadWrap(NNodes::TExprBase node, TExprContext& ctx, const std::function& getParents, bool enableDqReplicate, TTypeAnnotationContext& typeAnnCtx); diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h index 76ce9c7b438b..8ea2477e6ae4 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h @@ -57,7 +57,7 @@ struct TDqSettings { static constexpr ETaskRunnerStats TaskRunnerStats = ETaskRunnerStats::Basic; static constexpr ESpillingEngine SpillingEngine = ESpillingEngine::Disable; static constexpr ui32 CostBasedOptimizationLevel = 4; - static constexpr ui32 MaxDPccpDPTableSize = 40000U; + static constexpr ui32 MaxDPHypDPTableSize = 40000U; static constexpr ui64 MaxAttachmentsSize = 2_GB; static constexpr bool SplitStageOnDqReplicate = true; static constexpr ui64 EnableSpillingNodes = 0;