diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index 3a3df0e38d48..6dbb3ece8c0d 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -160,10 +160,10 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } TMaybeNode OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) { - auto maxDPccpDPTableSize = Config->MaxDPccpDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPccpDPTableSize); + auto maxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize); auto optLevel = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel); auto providerCtx = TKqpProviderContext(KqpCtx, optLevel); - auto opt = std::unique_ptr(MakeNativeOptimizerNew(providerCtx, maxDPccpDPTableSize)); + auto opt = std::unique_ptr(MakeNativeOptimizerNew(providerCtx, maxDPhypDPTableSize)); TExprBase output = DqOptimizeEquiJoinWithCosts(node, ctx, TypesCtx, optLevel, *opt, [](auto& rels, auto label, auto node, auto stat) { rels.emplace_back(std::make_shared(TString(label), *stat, node)); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.cpp b/ydb/core/kqp/provider/yql_kikimr_settings.cpp index e720d06977ab..65d960ee452c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_settings.cpp @@ -93,7 +93,7 @@ TKikimrConfiguration::TKikimrConfiguration() { REGISTER_SETTING(*this, EnableSpillingNodes) .Parser([](const TString& v) { return ParseEnableSpillingNodes(v); }); - REGISTER_SETTING(*this, MaxDPccpDPTableSize); + REGISTER_SETTING(*this, MaxDPHypDPTableSize); REGISTER_SETTING(*this, MaxTasksPerStage); REGISTER_SETTING(*this, MaxSequentialReadsInFlight); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 85c5a8829541..28bab1b62aba 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -69,7 +69,7 @@ struct TKikimrSettings { NCommon::TConfSetting OptUseFinalizeByKey; NCommon::TConfSetting CostBasedOptimizationLevel; - NCommon::TConfSetting MaxDPccpDPTableSize; + NCommon::TConfSetting MaxDPHypDPTableSize; NCommon::TConfSetting MaxTasksPerStage; diff --git a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp index 5018e4a70644..537acd9db638 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp @@ -235,9 +235,10 @@ void ComputeStatistics(const std::shared_ptr& join, IProvide class TOptimizerNativeNew: public IOptimizerNew { public: - TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize) + TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize, TExprContext* exprCtx = nullptr) : IOptimizerNew(ctx) - , MaxDPhypTableSize_(maxDPhypDPTableSize) + , MaxDPHypTableSize_(maxDPhypDPTableSize) + , ExprCtx(exprCtx) {} std::shared_ptr JoinSearch( @@ -272,8 +273,17 @@ class TOptimizerNativeNew: public IOptimizerNew { TJoinHypergraph hypergraph = MakeJoinHypergraph(joinTree, hints); TDPHypSolver solver(hypergraph, this->Pctx); - if (solver.CountCC(MaxDPhypTableSize_) >= MaxDPhypTableSize_) { + if (solver.CountCC(MaxDPHypTableSize_) >= MaxDPHypTableSize_) { YQL_CLOG(TRACE, CoreDq) << "Maximum DPhyp threshold exceeded"; + if (ExprCtx) { + ExprCtx.AddWarning( + YqlIssue( + {}, TIssuesIds::DQ_OPTIMIZE_ERROR, + "Cost Based Optimizer didn't work: " + "Enumeration is too large, use PRAGMA MaxDPHypDPTableSize='4294967295' to disable the limitation" + ) + ); + } ComputeStatistics(joinTree, this->Pctx); return joinTree; } @@ -304,11 +314,12 @@ class TOptimizerNativeNew: public IOptimizerNew { } private: - ui32 MaxDPhypTableSize_; + ui32 MaxDPHypTableSize_; + TExprContext* ExprCtx; }; -IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize) { - return new TOptimizerNativeNew(ctx, maxDPhypDPTableSize); +IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize, TExprContext* exprCtx = nullptr) { + return new TOptimizerNativeNew(ctx, maxDPhypDPTableSize, exprCtx); } TExprBase DqOptimizeEquiJoinWithCosts( @@ -357,6 +368,7 @@ TExprBase DqOptimizeEquiJoinWithCosts( // of the EquiJoin and n-1 argument are the parameters to EquiJoin if (!DqCollectJoinRelationsWithStats(rels, typesCtx, equiJoin, providerCollect)){ + ctx.AddWarning(YqlIssue({}, TIssuesIds::DQ_OPTIMIZE_ERROR, "Cost Based Optimizer didn't work: couldn't load statistics")); return node; } diff --git a/ydb/library/yql/dq/opt/dq_opt_log.h b/ydb/library/yql/dq/opt/dq_opt_log.h index 34816f163c26..1b8a79fefb0b 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.h +++ b/ydb/library/yql/dq/opt/dq_opt_log.h @@ -63,7 +63,7 @@ IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPt NNodes::TExprBase DqExpandMatchRecognize(NNodes::TExprBase node, TExprContext& ctx, TTypeAnnotationContext& typeAnnCtx); -IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPccpDPTableSize); +IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize); NNodes::TMaybeNode UnorderedOverDqReadWrap(NNodes::TExprBase node, TExprContext& ctx, const std::function& getParents, bool enableDqReplicate, TTypeAnnotationContext& typeAnnCtx); diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h index 76ce9c7b438b..8ea2477e6ae4 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h @@ -57,7 +57,7 @@ struct TDqSettings { static constexpr ETaskRunnerStats TaskRunnerStats = ETaskRunnerStats::Basic; static constexpr ESpillingEngine SpillingEngine = ESpillingEngine::Disable; static constexpr ui32 CostBasedOptimizationLevel = 4; - static constexpr ui32 MaxDPccpDPTableSize = 40000U; + static constexpr ui32 MaxDPHypDPTableSize = 40000U; static constexpr ui64 MaxAttachmentsSize = 2_GB; static constexpr bool SplitStageOnDqReplicate = true; static constexpr ui64 EnableSpillingNodes = 0;