From 5f4459883704c895f7ddf0ee43f2522c19d15545 Mon Sep 17 00:00:00 2001
From: Andrey Neporada <aneporada@ydb.tech>
Date: Sun, 1 Dec 2024 13:15:47 +0300
Subject: [PATCH] [YQL-19309] Switch YT provider to DQ nodes from
 yql/essentials (#12158)

---
 .../yt/gateway/file/yql_yt_file_mkql_compiler.cpp         | 6 ++++--
 ydb/library/yql/providers/yt/gateway/native/ya.make       | 2 --
 .../yql/providers/yt/gateway/native/yql_yt_native.cpp     | 3 ---
 .../yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp      | 3 ++-
 .../yt/provider/phy_opt/yql_yt_phy_opt_write.cpp          | 8 ++++----
 ydb/library/yql/providers/yt/provider/ya.make             | 4 ++--
 .../yql/providers/yt/provider/yql_yt_dq_hybrid.cpp        | 6 +++---
 .../yql/providers/yt/provider/yql_yt_dq_integration.cpp   | 5 +++--
 .../yql/providers/yt/provider/yql_yt_dq_optimize.cpp      | 6 ++++--
 ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp  | 5 +++--
 .../yql/providers/yt/provider/yql_yt_mkql_compiler.cpp    | 8 +++++---
 ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp  | 4 +++-
 12 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp
index ff1ab5064448..a22d6307e6c6 100644
--- a/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp
+++ b/ydb/library/yql/providers/yt/gateway/file/yql_yt_file_mkql_compiler.cpp
@@ -1,14 +1,15 @@
 #include "yql_yt_file_mkql_compiler.h"
 
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_table.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_helpers.h>
 #include <ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
+
 #include <yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h>
 #include <yql/essentials/providers/common/mkql/yql_type_mkql.h>
 #include <yql/essentials/providers/common/schema/mkql/yql_mkql_schema.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
 #include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/utils/yql_panic.h>
@@ -31,6 +32,7 @@ namespace NYql {
 using namespace NKikimr;
 using namespace NKikimr::NMiniKQL;
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 namespace {
 
diff --git a/ydb/library/yql/providers/yt/gateway/native/ya.make b/ydb/library/yql/providers/yt/gateway/native/ya.make
index 55a5ddcac8fd..6b8a8fe889fc 100644
--- a/ydb/library/yql/providers/yt/gateway/native/ya.make
+++ b/ydb/library/yql/providers/yt/gateway/native/ya.make
@@ -40,8 +40,6 @@ PEERDIR(
     yql/essentials/providers/common/provider
     yql/essentials/providers/common/schema/expr
     yql/essentials/providers/result/expr_nodes
-    ydb/library/yql/providers/stat/expr_nodes
-    ydb/library/yql/providers/stat/uploader
     ydb/library/yql/providers/yt/codec
     ydb/library/yql/providers/yt/common
     ydb/library/yql/providers/yt/expr_nodes
diff --git a/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp b/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp
index f2e2b9848d9a..de888e7f4223 100644
--- a/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp
+++ b/ydb/library/yql/providers/yt/gateway/native/yql_yt_native.cpp
@@ -33,14 +33,11 @@
 #include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h>
 #include <yql/essentials/providers/common/proto/gateways_config.pb.h>
 #include <yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h>
-#include <ydb/library/yql/providers/stat/expr_nodes/yql_stat_expr_nodes.h>
-#include <ydb/library/yql/providers/stat/uploader/yql_stat_uploader.h>
 
 #include <yql/essentials/ast/yql_expr.h>
 #include <yql/essentials/core/issue/yql_issue.h>
 #include <yql/essentials/core/yql_type_helpers.h>
 #include <yql/essentials/core/yql_graph_transformer.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 #include <yql/essentials/minikql/mkql_node.h>
 #include <yql/essentials/minikql/mkql_node_cast.h>
diff --git a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp
index f8263b580c30..f15add647ad6 100644
--- a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp
+++ b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_key_range.cpp
@@ -2,13 +2,14 @@
 #include "yql_yt_phy_opt_helper.h"
 
 #include <ydb/library/yql/providers/yt/provider/yql_yt_helpers.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 using namespace NPrivate;
 
 // All keyFilter settings are combined by OR.
diff --git a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp
index dda1b76b4d3c..8f32b0088aa3 100644
--- a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp
+++ b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_write.cpp
@@ -5,17 +5,17 @@
 #include <ydb/library/yql/providers/yt/provider/yql_yt_optimize.h>
 #include <ydb/library/yql/providers/yt/opt/yql_yt_key_selector.h>
 #include <ydb/library/yql/providers/stat/expr_nodes/yql_stat_expr_nodes.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
-
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/core/yql_type_helpers.h>
+#include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 using namespace NPrivate;
 
 TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::DqWrite(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) const {
diff --git a/ydb/library/yql/providers/yt/provider/ya.make b/ydb/library/yql/providers/yt/provider/ya.make
index 35500a01c2a1..ed0ab17938bf 100644
--- a/ydb/library/yql/providers/yt/provider/ya.make
+++ b/ydb/library/yql/providers/yt/provider/ya.make
@@ -77,6 +77,8 @@ PEERDIR(
     yql/essentials/utils
     yql/essentials/utils/log
     yql/essentials/core
+    yql/essentials/core/dq_expr_nodes
+    yql/essentials/core/dqs_expr_nodes
     yql/essentials/core/expr_nodes
     yql/essentials/core/issue
     yql/essentials/core/issue/protos
@@ -96,8 +98,6 @@ PEERDIR(
     yql/essentials/providers/common/schema/expr
     yql/essentials/providers/common/structured_token
     yql/essentials/providers/common/transform
-    ydb/library/yql/dq/expr_nodes
-    ydb/library/yql/providers/dq/expr_nodes
     yql/essentials/providers/result/expr_nodes
     ydb/library/yql/providers/stat/expr_nodes
     ydb/library/yql/providers/yt/common
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
index aad68af8127e..d38739e3d211 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_hybrid.cpp
@@ -9,14 +9,14 @@
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/providers/common/provider/yql_provider.h>
 #include <yql/essentials/providers/common/transform/yql_optimize.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
 #include <yql/essentials/core/yql_expr_type_annotation.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
 #include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/core/yql_type_helpers.h>
 #include <yql/essentials/core/yql_data_provider.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 #include <yql/essentials/utils/yql_panic.h>
 #include <yql/essentials/minikql/mkql_program_builder.h>
@@ -31,7 +31,7 @@ namespace NYql {
 namespace {
 
 using namespace NNodes;
-using namespace NDq;
+using namespace NNodes::NDq;
 
 class TYtDqHybridTransformer : public TOptimizeTransformerBase {
 public:
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
index 3cf9d99cf69e..cb6d93d892d8 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
@@ -13,10 +13,10 @@
 #include <yql/essentials/providers/common/dq/yql_dq_integration_impl.h>
 #include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 #include <yql/essentials/providers/common/config/yql_dispatch.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/providers/result/expr_nodes/yql_res_expr_nodes.h>
 #include <yql/essentials/ast/yql_expr.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/yql_type_helpers.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
 #include <yql/essentials/core/yql_opt_utils.h>
@@ -39,6 +39,7 @@ static const THashSet<TStringBuf> UNSUPPORTED_YT_PRAGMAS = {"maxrowweight",  "la
 static const THashSet<TStringBuf> POOL_TREES_WHITELIST = {"physical",  "cloud", "cloud_default"};
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 bool CheckSupportedTypesOld(const TTypeAnnotationNode::TListType& typesToCheck, const TSet<TString>& supportedTypes, const TSet<NUdf::EDataSlot>& supportedDataTypes, std::function<void(const TString&)> unsupportedTypeHandler) {
     TSet<ETypeAnnotationKind> supported;
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_optimize.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_optimize.cpp
index 8f78d7b57ce0..341517fa6ae9 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_optimize.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_optimize.cpp
@@ -3,15 +3,17 @@
 #include "yql_yt_optimize.h"
 
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
-#include <yql/essentials/providers/common/dq/yql_dq_optimization_impl.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
+
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
+#include <yql/essentials/providers/common/dq/yql_dq_optimization_impl.h>
 #include <yql/essentials/utils/log/log.h>
 
 
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 class TYtDqOptimizers: public TDqOptimizationBase {
 public:
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp
index de3310e9bf18..ff1257a36ad6 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp
@@ -4,14 +4,14 @@
 #include "yql_yt_op_hash.h"
 #include "yql_yt_optimize.h"
 
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/lib/mkql_helpers/mkql_helpers.h>
 #include <ydb/library/yql/providers/yt/common/yql_configuration.h>
 #include <ydb/library/yql/providers/yt/opt/yql_yt_key_selector.h>
 #include <yql/essentials/providers/common/provider/yql_provider.h>
 #include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 #include <yql/essentials/providers/common/codec/yql_codec.h>
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
 #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h>
 #include <yql/essentials/core/type_ann/type_ann_expr.h>
 #include <yql/essentials/core/type_ann/type_ann_core.h>
@@ -37,6 +37,7 @@
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 namespace {
 
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp
index fbd00690180f..c2c7c2222e00 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_mkql_compiler.cpp
@@ -6,14 +6,15 @@
 #include <ydb/library/yql/providers/yt/lib/row_spec/yql_row_spec.h>
 #include <ydb/library/yql/providers/yt/lib/skiff/yql_skiff_schema.h>
 #include <ydb/library/yql/providers/yt/lib/mkql_helpers/mkql_helpers.h>
-#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
+
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
+#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h>
+#include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/providers/common/codec/yql_codec_type_flags.h>
 #include <yql/essentials/providers/common/mkql/yql_type_mkql.h>
 #include <yql/essentials/minikql/mkql_program_builder.h>
 #include <yql/essentials/minikql/mkql_node_cast.h>
 #include <yql/essentials/minikql/defs.h>
-#include <yql/essentials/core/yql_opt_utils.h>
 #include <yql/essentials/utils/log/context.h>
 
 #include <library/cpp/yson/node/node_io.h>
@@ -28,6 +29,7 @@ namespace NYql {
 using namespace NKikimr;
 using namespace NKikimr::NMiniKQL;
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 TRuntimeNode BuildTableContentCall(TStringBuf callName,
     TType* outItemType,
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp
index 7d498af7a731..926a5e51faa6 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_op_hash.cpp
@@ -4,7 +4,8 @@
 
 #include <ydb/library/yql/providers/yt/expr_nodes/yql_yt_expr_nodes.h>
 #include <ydb/library/yql/providers/yt/lib/hash/yql_hash_builder.h>
-#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
+
+#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h>
 #include <yql/essentials/utils/log/log.h>
 
 #include <yql/essentials/utils/yql_panic.h>
@@ -17,6 +18,7 @@
 namespace NYql {
 
 using namespace NNodes;
+using namespace NNodes::NDq;
 
 TYtNodeHashCalculator::TYtNodeHashCalculator(const TYtState::TPtr& state, const TString& cluster, const TYtSettings::TConstPtr& config)
     : TNodeHashCalculator(*state->Types, state->NodeHash, MakeSalt(config, cluster))