From d7b6641d84f33422b6270a433fe329bb5de396c3 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Mon, 25 Dec 2023 16:59:04 +0000 Subject: [PATCH 1/2] init --- ydb/library/yql/minikql/codegen/codegen.cpp | 4 ++ ydb/library/yql/minikql/codegen/codegen.h | 2 + .../minikql/codegen/no_llvm/codegen_dummy.cpp | 25 ++++++++ .../yql/minikql/codegen/no_llvm/ya.make | 17 ++++++ ydb/library/yql/minikql/codegen/ya.make | 2 +- .../mkql_computation_node_graph.cpp | 2 +- .../yql/providers/common/comp_nodes/ya.make | 16 ------ .../yt/codec/codegen/no_llvm/ya.make | 22 +++++++ .../yql/providers/yt/codec/codegen/ya.make | 6 ++ .../yt/codec/codegen/yt_codec_cg_dummy.cpp | 38 +++++++++++++ .../yql/providers/yt/codec/ut/no_llvm/ya.make | 26 +++++++++ ydb/library/yql/providers/yt/codec/ut/ya.make | 1 + ydb/library/yql/providers/yt/codec/ya.make | 7 +-- .../yql/providers/yt/codec/yt_codec_io.cpp | 4 +- .../yt/comp_nodes/dq/dq_yt_block_reader.cpp | 2 +- .../yql/providers/yt/comp_nodes/dq/ya.make | 3 - .../yql/providers/yt/comp_nodes/llvm/ya.make | 23 ++++++++ .../providers/yt/comp_nodes/llvm14/ya.make | 23 ++++++++ .../providers/yt/comp_nodes/no_llvm/ya.make | 16 ++++++ .../yql/providers/yt/comp_nodes/ya.make | 36 +++--------- .../yql/providers/yt/comp_nodes/ya.make.inc | 45 +++++++++++++++ .../yt/comp_nodes/yql_mkql_input.cpp | 2 +- .../yt/comp_nodes/yql_mkql_output.cpp | 2 +- .../yql/public/embedded/no_llvm/ya.make | 57 +++++++++++++++++++ .../yql/public/embedded/ut/no_llvm/ya.make | 23 ++++++++ ydb/library/yql/public/embedded/ut/ya.make | 19 +++++++ .../public/embedded/ut/yql_embedded_ut.cpp | 30 ++++++++++ ydb/library/yql/public/embedded/ya.make | 12 ++++ .../yql/public/purecalc/common/ya.make | 3 - .../purecalc/io_specs/mkql/ut/no_llvm/ya.make | 24 ++++++++ .../public/purecalc/io_specs/mkql/ut/ya.make | 2 +- .../yql/public/purecalc/io_specs/mkql/ya.make | 1 + .../yql/public/purecalc/no_llvm/purecalc.h | 4 ++ .../yql/public/purecalc/no_llvm/ya.make | 30 ++++++++++ ydb/library/yql/public/purecalc/ya.make | 7 +++ ydb/library/yql/tools/dq/service_node/ya.make | 2 + ydb/library/yql/tools/dq/worker_node/ya.make | 2 + ydb/library/yql/tools/dqrun/ya.make | 1 + ydb/library/yql/tools/mrjob/ya.make | 2 + ydb/library/yql/tools/pgrun/ya.make | 2 + ydb/library/yql/tools/yqlrun/ya.make | 2 + 41 files changed, 483 insertions(+), 64 deletions(-) create mode 100644 ydb/library/yql/minikql/codegen/no_llvm/codegen_dummy.cpp create mode 100644 ydb/library/yql/minikql/codegen/no_llvm/ya.make create mode 100644 ydb/library/yql/providers/yt/codec/codegen/no_llvm/ya.make create mode 100644 ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg_dummy.cpp create mode 100644 ydb/library/yql/providers/yt/codec/ut/no_llvm/ya.make create mode 100644 ydb/library/yql/providers/yt/comp_nodes/llvm/ya.make create mode 100644 ydb/library/yql/providers/yt/comp_nodes/llvm14/ya.make create mode 100644 ydb/library/yql/providers/yt/comp_nodes/no_llvm/ya.make create mode 100644 ydb/library/yql/providers/yt/comp_nodes/ya.make.inc create mode 100644 ydb/library/yql/public/embedded/no_llvm/ya.make create mode 100644 ydb/library/yql/public/embedded/ut/no_llvm/ya.make create mode 100644 ydb/library/yql/public/embedded/ut/ya.make create mode 100644 ydb/library/yql/public/embedded/ut/yql_embedded_ut.cpp create mode 100644 ydb/library/yql/public/purecalc/io_specs/mkql/ut/no_llvm/ya.make create mode 100644 ydb/library/yql/public/purecalc/no_llvm/purecalc.h create mode 100644 ydb/library/yql/public/purecalc/no_llvm/ya.make diff --git a/ydb/library/yql/minikql/codegen/codegen.cpp b/ydb/library/yql/minikql/codegen/codegen.cpp index 1882d28fa007..c84f6622cae7 100644 --- a/ydb/library/yql/minikql/codegen/codegen.cpp +++ b/ydb/library/yql/minikql/codegen/codegen.cpp @@ -230,6 +230,10 @@ namespace { } } +bool ICodegen::IsCodegenAvailable() { + return true; +} + class TCodegen : public ICodegen, private llvm::JITEventListener { public: TCodegen(ETarget target, ESanitize sanitize) diff --git a/ydb/library/yql/minikql/codegen/codegen.h b/ydb/library/yql/minikql/codegen/codegen.h index 6bfac89317e1..4c7e2ab4f12f 100644 --- a/ydb/library/yql/minikql/codegen/codegen.h +++ b/ydb/library/yql/minikql/codegen/codegen.h @@ -67,6 +67,8 @@ class ICodegen { using TSharedPtr = std::shared_ptr; static TSharedPtr MakeShared(ETarget target, ESanitize sanitize = ESanitize::Auto); + + static bool IsCodegenAvailable(); }; } diff --git a/ydb/library/yql/minikql/codegen/no_llvm/codegen_dummy.cpp b/ydb/library/yql/minikql/codegen/no_llvm/codegen_dummy.cpp new file mode 100644 index 000000000000..88f37f8d481d --- /dev/null +++ b/ydb/library/yql/minikql/codegen/no_llvm/codegen_dummy.cpp @@ -0,0 +1,25 @@ +#include "codegen.h" + +#include + +namespace NYql { +namespace NCodegen { + +ICodegen::TPtr ICodegen::Make(ETarget target, ESanitize sanitize) { + Y_UNUSED(target); + Y_UNUSED(sanitize); + throw yexception() << "Codegen is not available"; +} + +ICodegen::TSharedPtr ICodegen::MakeShared(ETarget target, ESanitize sanitize) { + Y_UNUSED(target); + Y_UNUSED(sanitize); + throw yexception() << "Codegen is not available"; +} + +bool ICodegen::IsCodegenAvailable() { + return false; +} + +} +} diff --git a/ydb/library/yql/minikql/codegen/no_llvm/ya.make b/ydb/library/yql/minikql/codegen/no_llvm/ya.make new file mode 100644 index 000000000000..c1cb52b33928 --- /dev/null +++ b/ydb/library/yql/minikql/codegen/no_llvm/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +SRCDIR( + ydb/library/yql/minikql/codegen +) + +ADDINCL( + ydb/library/yql/minikql/codegen +) + +SRCS( + codegen_dummy.cpp +) + +PROVIDES(MINIKQL_CODEGEN) + +END() diff --git a/ydb/library/yql/minikql/codegen/ya.make b/ydb/library/yql/minikql/codegen/ya.make index 5d082d5ae979..31b75b3c4272 100644 --- a/ydb/library/yql/minikql/codegen/ya.make +++ b/ydb/library/yql/minikql/codegen/ya.make @@ -1 +1 @@ -RECURSE(llvm llvm14) \ No newline at end of file +RECURSE(no_llvm llvm llvm14) \ No newline at end of file diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index 7519d036b40a..9a2bf24a2818 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -779,7 +779,7 @@ class TComputationPatternImpl final : public IComputationPattern { #elif defined(MKQL_FORCE_USE_CODEGEN) : Codegen(NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native)) #else - : Codegen(opts.OptLLVM != "OFF" || GetEnv(TString("MKQL_FORCE_USE_LLVM")) ? NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native) : NYql::NCodegen::ICodegen::TPtr()) + : Codegen((NYql::NCodegen::ICodegen::IsCodegenAvailable() && opts.OptLLVM != "OFF") || GetEnv(TString("MKQL_FORCE_USE_LLVM")) ? NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native) : NYql::NCodegen::ICodegen::TPtr()) #endif { /// TODO: Enable JIT for AARCH64 diff --git a/ydb/library/yql/providers/common/comp_nodes/ya.make b/ydb/library/yql/providers/common/comp_nodes/ya.make index 1befa2f92656..6eb2ce582edd 100644 --- a/ydb/library/yql/providers/common/comp_nodes/ya.make +++ b/ydb/library/yql/providers/common/comp_nodes/ya.make @@ -41,22 +41,6 @@ PEERDIR( ydb/library/yql/providers/common/schema/expr ) -IF (NOT MKQL_DISABLE_CODEGEN) - PEERDIR( - ydb/library/yql/minikql/codegen/llvm - contrib/libs/llvm12/lib/IR - contrib/libs/llvm12/lib/ExecutionEngine/MCJIT - contrib/libs/llvm12/lib/Linker - contrib/libs/llvm12/lib/Target/X86 - contrib/libs/llvm12/lib/Target/X86/AsmParser - contrib/libs/llvm12/lib/Transforms/IPO - ) -ELSE() - CFLAGS( - -DMKQL_DISABLE_CODEGEN - ) -ENDIF() - YQL_LAST_ABI_VERSION() END() diff --git a/ydb/library/yql/providers/yt/codec/codegen/no_llvm/ya.make b/ydb/library/yql/providers/yt/codec/codegen/no_llvm/ya.make new file mode 100644 index 000000000000..bab5c892a8ab --- /dev/null +++ b/ydb/library/yql/providers/yt/codec/codegen/no_llvm/ya.make @@ -0,0 +1,22 @@ +LIBRARY() + +ADDINCL( + ydb/library/yql/providers/yt/codec/codegen +) + +SRCDIR( + ydb/library/yql/providers/yt/codec/codegen +) + +SRCS( + yt_codec_cg_dummy.cpp +) + +PEERDIR( +) + +PROVIDES(YT_CODEC_CODEGEN) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/yt/codec/codegen/ya.make b/ydb/library/yql/providers/yt/codec/codegen/ya.make index eb25e02da122..c915d0f1caf9 100644 --- a/ydb/library/yql/providers/yt/codec/codegen/ya.make +++ b/ydb/library/yql/providers/yt/codec/codegen/ya.make @@ -72,8 +72,14 @@ ENDIF() YQL_LAST_ABI_VERSION() +PROVIDES(YT_CODEC_CODEGEN) + END() +RECURSE( + no_llvm +) + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg_dummy.cpp b/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg_dummy.cpp new file mode 100644 index 000000000000..b9bdb23c3572 --- /dev/null +++ b/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg_dummy.cpp @@ -0,0 +1,38 @@ +#include "yt_codec_cg.h" + +#include + +namespace NYql { + +TString GetYtCodecBitCode() { + throw yexception() << "Codegen is not available"; +} + +void YtCodecAddMappings(NCodegen::ICodegen& codegen) { + Y_UNUSED(codegen); + throw yexception() << "Codegen is not available"; +} + +template +THolder MakeYtCodecCgWriter(const std::unique_ptr& codegen, const void* cookie) { + Y_UNUSED(codegen); + Y_UNUSED(cookie); + throw yexception() << "Codegen is not available"; +} + +template THolder MakeYtCodecCgWriter(const std::unique_ptr& codegen, const void* cookie); +template THolder MakeYtCodecCgWriter(const std::unique_ptr& codegen, const void* cookie); + +THolder MakeYtCodecCgReader(const std::unique_ptr& codegen, + const NKikimr::NMiniKQL::THolderFactory& holderFactory, const void* cookie) { + Y_UNUSED(codegen); + Y_UNUSED(holderFactory); + Y_UNUSED(cookie); + throw yexception() << "Codegen is not available"; +} + +extern "C" void ThrowBadDecimal() { + throw yexception() << "Codegen is not available"; +} + +} diff --git a/ydb/library/yql/providers/yt/codec/ut/no_llvm/ya.make b/ydb/library/yql/providers/yt/codec/ut/no_llvm/ya.make new file mode 100644 index 000000000000..d55ad63985bc --- /dev/null +++ b/ydb/library/yql/providers/yt/codec/ut/no_llvm/ya.make @@ -0,0 +1,26 @@ +UNITTEST_FOR(ydb/library/yql/providers/yt/codec) + +SRCDIR( + ydb/library/yql/providers/yt/codec/ut +) + +SRCS( + yt_codec_io_ut.cpp +) + +PEERDIR( + library/cpp/yson/node + ydb/library/yql/minikql/codegen/no_llvm + ydb/library/yql/minikql/computation/no_llvm + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/sql + ydb/library/yql/sql/pg_dummy + ydb/library/yql/providers/common/codec + ydb/library/yql/providers/common/mkql + ydb/library/yql/providers/yt/lib/yson_helpers + ydb/library/yql/providers/yt/codec/codegen/no_llvm +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/yt/codec/ut/ya.make b/ydb/library/yql/providers/yt/codec/ut/ya.make index 5ab9bc2a896a..11ce1a9c49dd 100644 --- a/ydb/library/yql/providers/yt/codec/ut/ya.make +++ b/ydb/library/yql/providers/yt/codec/ut/ya.make @@ -13,6 +13,7 @@ PEERDIR( ydb/library/yql/providers/common/codec ydb/library/yql/providers/common/mkql ydb/library/yql/providers/yt/lib/yson_helpers + ydb/library/yql/providers/yt/codec/codegen ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/yt/codec/ya.make b/ydb/library/yql/providers/yt/codec/ya.make index 558538625b48..9298c9230de0 100644 --- a/ydb/library/yql/providers/yt/codec/ya.make +++ b/ydb/library/yql/providers/yt/codec/ya.make @@ -26,11 +26,7 @@ PEERDIR( ydb/library/yql/providers/yt/lib/skiff ) -IF (NOT MKQL_DISABLE_CODEGEN) - PEERDIR( - ydb/library/yql/providers/yt/codec/codegen - ) -ELSE() +IF (MKQL_DISABLE_CODEGEN) CFLAGS( -DMKQL_DISABLE_CODEGEN ) @@ -46,4 +42,5 @@ RECURSE( RECURSE_FOR_TESTS( ut + ut/no_llvm ) diff --git a/ydb/library/yql/providers/yt/codec/yt_codec_io.cpp b/ydb/library/yql/providers/yt/codec/yt_codec_io.cpp index 768680272555..e93c7b411dd1 100644 --- a/ydb/library/yql/providers/yt/codec/yt_codec_io.cpp +++ b/ydb/library/yql/providers/yt/codec/yt_codec_io.cpp @@ -1469,7 +1469,7 @@ void TMkqlReaderImpl::SetSpecs(const TMkqlIOSpecs& specs, const NKikimr::NMiniKQ Buf_.SetStats(JobStats_); if (Specs_->UseSkiff_) { #ifndef MKQL_DISABLE_CODEGEN - if (Specs_->OptLLVM_ != "OFF") { + if (Specs_->OptLLVM_ != "OFF" && NCodegen::ICodegen::IsCodegenAvailable()) { Decoder_.Reset(new TSkiffLLVMDecoder(Buf_, *Specs_, holderFactory)); } else @@ -1976,7 +1976,7 @@ void TMkqlWriterImpl::SetSpecs(const TMkqlIOSpecs& specs, const TVector #ifndef MKQL_DISABLE_CODEGEN THashMap> llvmFunctions; - if (Specs_->UseSkiff_ && Specs_->OptLLVM_ != "OFF") { + if (Specs_->UseSkiff_ && Specs_->OptLLVM_ != "OFF" && NCodegen::ICodegen::IsCodegenAvailable()) { for (size_t i: xrange(Specs_->Outputs.size())) { auto rowType = Specs_->Outputs[i].RowType; if (rowType->GetMembersCount() != 0 && !llvmFunctions.contains(rowType)) { diff --git a/ydb/library/yql/providers/yt/comp_nodes/dq/dq_yt_block_reader.cpp b/ydb/library/yql/providers/yt/comp_nodes/dq/dq_yt_block_reader.cpp index 777e153e25c5..69ef6df5b42a 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/dq/dq_yt_block_reader.cpp +++ b/ydb/library/yql/providers/yt/comp_nodes/dq/dq_yt_block_reader.cpp @@ -624,4 +624,4 @@ IComputationNode* CreateDqYtReadBlockWrapper(const TComputationNodeFactoryContex { return new TDqYtReadBlockWrapper(ctx, clusterName, token, inputSpec, samplingSpec, inputGroups, itemType, tableNames, std::move(tables), jobStats, inflight, timeout); } -} \ No newline at end of file +} diff --git a/ydb/library/yql/providers/yt/comp_nodes/dq/ya.make b/ydb/library/yql/providers/yt/comp_nodes/dq/ya.make index 955b3691d4a2..cba05d599a94 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/dq/ya.make +++ b/ydb/library/yql/providers/yt/comp_nodes/dq/ya.make @@ -3,7 +3,6 @@ LIBRARY() PEERDIR( ydb/library/yql/minikql ydb/library/yql/minikql/computation/llvm - ydb/library/yql/minikql/computation ydb/library/yql/providers/yt/comp_nodes ydb/library/yql/providers/yt/codec ydb/library/yql/providers/common/codec @@ -45,8 +44,6 @@ SRCS( dq_yt_writer.cpp ) -INCLUDE(../../../../minikql/computation/header.ya.make.inc) - YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/yt/comp_nodes/llvm/ya.make b/ydb/library/yql/providers/yt/comp_nodes/llvm/ya.make new file mode 100644 index 000000000000..69fcd4370d86 --- /dev/null +++ b/ydb/library/yql/providers/yt/comp_nodes/llvm/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +OWNER( + g:yql + g:yql_ydb_core +) + +NO_COMPILER_WARNINGS() + +PEERDIR( + ydb/library/yql/minikql/codegen/llvm + ydb/library/yql/minikql/invoke_builtins/llvm + contrib/libs/llvm12/lib/IR + contrib/libs/llvm12/lib/ExecutionEngine/MCJIT + contrib/libs/llvm12/lib/Linker + contrib/libs/llvm12/lib/Target/X86 + contrib/libs/llvm12/lib/Target/X86/AsmParser + contrib/libs/llvm12/lib/Transforms/IPO +) + +INCLUDE(../ya.make.inc) + +END() diff --git a/ydb/library/yql/providers/yt/comp_nodes/llvm14/ya.make b/ydb/library/yql/providers/yt/comp_nodes/llvm14/ya.make new file mode 100644 index 000000000000..06e6fa7b2b60 --- /dev/null +++ b/ydb/library/yql/providers/yt/comp_nodes/llvm14/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +OWNER( + g:yql + g:yql_ydb_core +) + +NO_COMPILER_WARNINGS() + +PEERDIR( + ydb/library/yql/minikql/codegen/llvm14 + ydb/library/yql/minikql/invoke_builtins/llvm14 + contrib/libs/llvm14/lib/IR + contrib/libs/llvm14/lib/ExecutionEngine/MCJIT + contrib/libs/llvm14/lib/Linker + contrib/libs/llvm14/lib/Target/X86 + contrib/libs/llvm14/lib/Target/X86/AsmParser + contrib/libs/llvm14/lib/Transforms/IPO +) + +INCLUDE(../ya.make.inc) + +END() diff --git a/ydb/library/yql/providers/yt/comp_nodes/no_llvm/ya.make b/ydb/library/yql/providers/yt/comp_nodes/no_llvm/ya.make new file mode 100644 index 000000000000..58bb2d80c89f --- /dev/null +++ b/ydb/library/yql/providers/yt/comp_nodes/no_llvm/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + g:yql + g:yql_ydb_core +) + +CXXFLAGS(-DMKQL_DISABLE_CODEGEN) + +ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub) + +INCLUDE(../ya.make.inc) + +PEERDIR(ydb/library/yql/minikql/invoke_builtins/no_llvm) + +END() diff --git a/ydb/library/yql/providers/yt/comp_nodes/ya.make b/ydb/library/yql/providers/yt/comp_nodes/ya.make index d4cde6e54be6..0fe8f7415bdd 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/ya.make +++ b/ydb/library/yql/providers/yt/comp_nodes/ya.make @@ -1,37 +1,15 @@ -LIBRARY() - -SRCS( - yql_mkql_file_input_state.cpp - yql_mkql_file_list.cpp - yql_mkql_input_stream.cpp - yql_mkql_input.cpp - yql_mkql_output.cpp - yql_mkql_table_content.cpp - yql_mkql_table.cpp - yql_mkql_ungrouping_list.cpp -) - -PEERDIR( - library/cpp/streams/brotli - ydb/library/yql/minikql - ydb/library/yql/public/udf - ydb/library/yql/utils - ydb/library/yql/providers/common/codec - ydb/library/yql/providers/common/mkql - ydb/library/yql/providers/yt/codec - ydb/library/yql/providers/yt/expr_nodes +OWNER( + g:yql + g:yql_ydb_core ) -# INCLUDE(../../../minikql/computation/header.ya.make.inc) - -YQL_LAST_ABI_VERSION() +LIBRARY() END() RECURSE( - dq + llvm + llvm14 + no_llvm ) -RECURSE_FOR_TESTS( - ut -) diff --git a/ydb/library/yql/providers/yt/comp_nodes/ya.make.inc b/ydb/library/yql/providers/yt/comp_nodes/ya.make.inc new file mode 100644 index 000000000000..c793cc052a92 --- /dev/null +++ b/ydb/library/yql/providers/yt/comp_nodes/ya.make.inc @@ -0,0 +1,45 @@ +INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/minikql/computation/header.ya.make.inc) +INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/minikql/invoke_builtins/header.ya.make.inc) + +SET(ORIG_SRC_DIR ${ARCADIA_ROOT}/ydb/library/yql/providers/yt/comp_nodes) + +SET(ORIG_SOURCES + yql_mkql_file_input_state.cpp + yql_mkql_file_list.cpp + yql_mkql_input_stream.cpp + yql_mkql_input.cpp + yql_mkql_output.cpp + yql_mkql_table_content.cpp + yql_mkql_table.cpp + yql_mkql_ungrouping_list.cpp +) + +PEERDIR( + library/cpp/streams/brotli + ydb/library/yql/minikql + ydb/library/yql/public/udf + ydb/library/yql/utils + ydb/library/yql/providers/common/codec + ydb/library/yql/providers/common/mkql + ydb/library/yql/providers/yt/codec + ydb/library/yql/providers/yt/expr_nodes +) + +YQL_LAST_ABI_VERSION() + +ADDINCL( + ${ORIG_SRC_DIR} +) + +COPY( + WITH_CONTEXT + AUTO + FROM ${ORIG_SRC_DIR} + ${ORIG_SOURCES} + OUTPUT_INCLUDES + ${BINDIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h + ${BINDIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.h + ${BINDIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.h +) + +PROVIDES(YT_COMP_NODES) diff --git a/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_input.cpp b/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_input.cpp index 69e6e9bce506..31cd7ce811a3 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_input.cpp +++ b/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_input.cpp @@ -2,7 +2,7 @@ #include "yql_mkql_table.h" #include -#include +#include // Y_IGNORE #include #include #include diff --git a/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_output.cpp b/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_output.cpp index 6cffb5179ec6..c784c3dac75b 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_output.cpp +++ b/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_output.cpp @@ -1,6 +1,6 @@ #include "yql_mkql_output.h" -#include +#include // Y_IGNORE #include namespace NYql { diff --git a/ydb/library/yql/public/embedded/no_llvm/ya.make b/ydb/library/yql/public/embedded/no_llvm/ya.make new file mode 100644 index 000000000000..b495ae6544bb --- /dev/null +++ b/ydb/library/yql/public/embedded/no_llvm/ya.make @@ -0,0 +1,57 @@ +LIBRARY() + +SRCDIR( + ydb/library/yql/public/embedded +) + +ADDINCL( + ydb/library/yql/public/embedded +) + +SRCS( + yql_embedded.cpp +) + +PEERDIR( + contrib/libs/protobuf + library/cpp/resource + library/cpp/yson + library/cpp/yson/node + yt/cpp/mapreduce/client + yt/cpp/mapreduce/common + ydb/library/yql/ast + ydb/library/yql/sql/pg + ydb/library/yql/core/facade + ydb/library/yql/core/file_storage + ydb/library/yql/core/file_storage/defs + ydb/library/yql/core/file_storage/proto + ydb/library/yql/core/file_storage/http_download + ydb/library/yql/core/services/mounts + ydb/library/yql/core/user_data + ydb/library/yql/minikql + ydb/library/yql/minikql/invoke_builtins/no_llvm + ydb/library/yql/minikql/comp_nodes/no_llvm + ydb/library/yql/minikql/computation/no_llvm + ydb/library/yql/minikql/codegen/no_llvm + ydb/library/yql/protos + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/utils/backtrace + ydb/library/yql/utils/log + ydb/library/yql/parser/pg_wrapper + ydb/library/yql/providers/common/proto + ydb/library/yql/providers/common/udf_resolve + ydb/library/yql/core/url_preprocessing + ydb/library/yql/core/url_lister + ydb/library/yql/providers/yt/gateway/native + ydb/library/yql/providers/yt/lib/log + ydb/library/yql/providers/yt/lib/yt_download + ydb/library/yql/providers/yt/lib/yt_url_lister + ydb/library/yql/providers/yt/provider + ydb/library/yql/providers/yt/codec/codegen/no_llvm + ydb/library/yql/providers/yt/comp_nodes/no_llvm +) + +YQL_LAST_ABI_VERSION() + +END() + diff --git a/ydb/library/yql/public/embedded/ut/no_llvm/ya.make b/ydb/library/yql/public/embedded/ut/no_llvm/ya.make new file mode 100644 index 000000000000..7ad0c1ec2b85 --- /dev/null +++ b/ydb/library/yql/public/embedded/ut/no_llvm/ya.make @@ -0,0 +1,23 @@ +UNITTEST() + +SIZE(MEDIUM) + +TIMEOUT(300) + +PEERDIR( + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/public/embedded/no_llvm +) + +YQL_LAST_ABI_VERSION() + +SRCDIR( + ydb/library/yql/public/embedded/ut +) + +SRCS( + yql_embedded_ut.cpp +) + +END() + diff --git a/ydb/library/yql/public/embedded/ut/ya.make b/ydb/library/yql/public/embedded/ut/ya.make new file mode 100644 index 000000000000..06964f28504c --- /dev/null +++ b/ydb/library/yql/public/embedded/ut/ya.make @@ -0,0 +1,19 @@ +UNITTEST() + +SIZE(MEDIUM) + +TIMEOUT(300) + +PEERDIR( + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/public/embedded +) + +YQL_LAST_ABI_VERSION() + +SRCS( + yql_embedded_ut.cpp +) + +END() + diff --git a/ydb/library/yql/public/embedded/ut/yql_embedded_ut.cpp b/ydb/library/yql/public/embedded/ut/yql_embedded_ut.cpp new file mode 100644 index 000000000000..bed7321cb5e6 --- /dev/null +++ b/ydb/library/yql/public/embedded/ut/yql_embedded_ut.cpp @@ -0,0 +1,30 @@ +#include + +#include + +namespace NYql { +namespace NEmbedded { + +namespace { + THolder MakeFactory() { + TOperationFactoryOptions options; + return MakeOperationFactory(options, "", [](const TFileStorageConfig&) { + return nullptr; + }); + } +} + +Y_UNIT_TEST_SUITE(TValidateTests) { + Y_UNIT_TEST(SimpleOk) { + auto factory = MakeFactory(); + auto res = factory->Run("select 1", {.Mode = EExecuteMode::Validate}); + } + + Y_UNIT_TEST(SimpleFail) { + auto factory = MakeFactory(); + UNIT_ASSERT_EXCEPTION(factory->Run("select foo", {.Mode = EExecuteMode::Validate}), yexception); + } +} + +} // NEmbedded +} // NYql diff --git a/ydb/library/yql/public/embedded/ya.make b/ydb/library/yql/public/embedded/ya.make index 33019594e26c..321aecc090f1 100644 --- a/ydb/library/yql/public/embedded/ya.make +++ b/ydb/library/yql/public/embedded/ya.make @@ -39,8 +39,20 @@ PEERDIR( ydb/library/yql/providers/yt/lib/yt_download ydb/library/yql/providers/yt/lib/yt_url_lister ydb/library/yql/providers/yt/provider + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ) YQL_LAST_ABI_VERSION() END() + +RECURSE( + no_llvm +) + +RECURSE_FOR_TESTS( + ut + ut/no_llvm +) + diff --git a/ydb/library/yql/public/purecalc/common/ya.make b/ydb/library/yql/public/purecalc/common/ya.make index 06fa66111b45..1ab76c9fdd67 100644 --- a/ydb/library/yql/public/purecalc/common/ya.make +++ b/ydb/library/yql/public/purecalc/common/ya.make @@ -26,9 +26,6 @@ PEERDIR( ydb/library/yql/core/services ydb/library/yql/core/services/mounts ydb/library/yql/core/user_data - ydb/library/yql/minikql/computation/llvm - ydb/library/yql/minikql/invoke_builtins/llvm - ydb/library/yql/minikql/comp_nodes/llvm ydb/library/yql/utils/backtrace ydb/library/yql/utils/log ydb/library/yql/core diff --git a/ydb/library/yql/public/purecalc/io_specs/mkql/ut/no_llvm/ya.make b/ydb/library/yql/public/purecalc/io_specs/mkql/ut/no_llvm/ya.make new file mode 100644 index 000000000000..4fd86e8e085c --- /dev/null +++ b/ydb/library/yql/public/purecalc/io_specs/mkql/ut/no_llvm/ya.make @@ -0,0 +1,24 @@ +UNITTEST() + +SIZE(MEDIUM) + +TIMEOUT(300) + +PEERDIR( + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/public/purecalc/no_llvm + ydb/library/yql/public/purecalc/io_specs/mkql + ydb/library/yql/public/purecalc/ut/lib +) + +YQL_LAST_ABI_VERSION() + +SRCDIR( + ydb/library/yql/public/purecalc/io_specs/mkql/ut +) + +SRCS( + test_spec.cpp +) + +END() diff --git a/ydb/library/yql/public/purecalc/io_specs/mkql/ut/ya.make b/ydb/library/yql/public/purecalc/io_specs/mkql/ut/ya.make index afc48d4356ee..d6853059bc7c 100644 --- a/ydb/library/yql/public/purecalc/io_specs/mkql/ut/ya.make +++ b/ydb/library/yql/public/purecalc/io_specs/mkql/ut/ya.make @@ -6,7 +6,7 @@ TIMEOUT(300) PEERDIR( ydb/library/yql/public/udf/service/exception_policy - ydb/library/yql/public/purecalc/common + ydb/library/yql/public/purecalc ydb/library/yql/public/purecalc/io_specs/mkql ydb/library/yql/public/purecalc/ut/lib ) diff --git a/ydb/library/yql/public/purecalc/io_specs/mkql/ya.make b/ydb/library/yql/public/purecalc/io_specs/mkql/ya.make index b6066163f7cb..4cd8eb41cad8 100644 --- a/ydb/library/yql/public/purecalc/io_specs/mkql/ya.make +++ b/ydb/library/yql/public/purecalc/io_specs/mkql/ya.make @@ -22,4 +22,5 @@ END() RECURSE_FOR_TESTS( ut + ut/no_llvm ) diff --git a/ydb/library/yql/public/purecalc/no_llvm/purecalc.h b/ydb/library/yql/public/purecalc/no_llvm/purecalc.h new file mode 100644 index 000000000000..9b281a7caa71 --- /dev/null +++ b/ydb/library/yql/public/purecalc/no_llvm/purecalc.h @@ -0,0 +1,4 @@ +#pragma once + +#include "common/interface.h" + diff --git a/ydb/library/yql/public/purecalc/no_llvm/ya.make b/ydb/library/yql/public/purecalc/no_llvm/ya.make new file mode 100644 index 000000000000..58d1142359f3 --- /dev/null +++ b/ydb/library/yql/public/purecalc/no_llvm/ya.make @@ -0,0 +1,30 @@ +LIBRARY() + +ADDINCL( + ydb/library/yql/public/purecalc +) + +SRCDIR( + ydb/library/yql/public/purecalc +) + +SRCS( + purecalc.cpp +) + +PEERDIR( + ydb/library/yql/public/udf/service/exception_policy + ydb/library/yql/public/purecalc/common + ydb/library/yql/providers/yt/codec/codegen/no_llvm + ydb/library/yql/minikql/codegen/no_llvm + ydb/library/yql/minikql/computation/no_llvm + ydb/library/yql/minikql/invoke_builtins/no_llvm + ydb/library/yql/minikql/comp_nodes/no_llvm +) + +YQL_LAST_ABI_VERSION() + +PROVIDES(YQL_PURECALC) + +END() + diff --git a/ydb/library/yql/public/purecalc/ya.make b/ydb/library/yql/public/purecalc/ya.make index cb99c1d31112..6688b4aa5df3 100644 --- a/ydb/library/yql/public/purecalc/ya.make +++ b/ydb/library/yql/public/purecalc/ya.make @@ -7,10 +7,16 @@ SRCS( PEERDIR( ydb/library/yql/public/udf/service/exception_policy ydb/library/yql/public/purecalc/common + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/minikql/computation/llvm + ydb/library/yql/minikql/invoke_builtins/llvm + ydb/library/yql/minikql/comp_nodes/llvm ) YQL_LAST_ABI_VERSION() +PROVIDES(YQL_PURECALC) + END() RECURSE( @@ -18,6 +24,7 @@ RECURSE( examples helpers io_specs + no_llvm ) RECURSE_FOR_TESTS( diff --git a/ydb/library/yql/tools/dq/service_node/ya.make b/ydb/library/yql/tools/dq/service_node/ya.make index 730bc565e43d..906e344fde9d 100644 --- a/ydb/library/yql/tools/dq/service_node/ya.make +++ b/ydb/library/yql/tools/dq/service_node/ya.make @@ -20,6 +20,8 @@ IF (NOT OS_WINDOWS) ydb/library/yql/providers/yt/dq_task_preprocessor ydb/library/yql/providers/dq/global_worker_manager ydb/library/yql/providers/dq/actors/yt + ydb/library/yql/providers/yt/comp_nodes/llvm + ydb/library/yql/providers/yt/codec/codegen yt/yt/client ) diff --git a/ydb/library/yql/tools/dq/worker_node/ya.make b/ydb/library/yql/tools/dq/worker_node/ya.make index 51b9f8babc0c..c8b090f90715 100644 --- a/ydb/library/yql/tools/dq/worker_node/ya.make +++ b/ydb/library/yql/tools/dq/worker_node/ya.make @@ -30,6 +30,8 @@ IF (NOT OS_WINDOWS) ydb/library/yql/utils/failure_injector ydb/library/yql/utils/backtrace ydb/library/yql/providers/yt/comp_nodes/dq + ydb/library/yql/providers/yt/comp_nodes/llvm + ydb/library/yql/providers/yt/codec/codegen ydb/library/yql/providers/yt/mkql_dq ydb/library/yql/providers/dq/actors/yt ydb/library/yql/providers/dq/global_worker_manager diff --git a/ydb/library/yql/tools/dqrun/ya.make b/ydb/library/yql/tools/dqrun/ya.make index 1b0ad5c9b0b2..6a1893092197 100644 --- a/ydb/library/yql/tools/dqrun/ya.make +++ b/ydb/library/yql/tools/dqrun/ya.make @@ -71,6 +71,7 @@ ENDIF() ydb/library/yql/providers/yt/dq_task_preprocessor ydb/library/yql/providers/yt/gateway/file ydb/library/yql/providers/yt/gateway/native + ydb/library/yql/providers/yt/codec/codegen ydb/library/yql/providers/yt/mkql_dq ydb/library/yql/providers/yt/provider ydb/library/yql/providers/yt/lib/yt_download diff --git a/ydb/library/yql/tools/mrjob/ya.make b/ydb/library/yql/tools/mrjob/ya.make index d9abeab689af..8839c6eb16c0 100644 --- a/ydb/library/yql/tools/mrjob/ya.make +++ b/ydb/library/yql/tools/mrjob/ya.make @@ -20,6 +20,8 @@ PEERDIR( ydb/library/yql/parser/pg_wrapper ydb/library/yql/sql/pg ydb/library/yql/providers/yt/job + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/minikql/computation/llvm ydb/library/yql/minikql/invoke_builtins/llvm ydb/library/yql/minikql/comp_nodes/llvm diff --git a/ydb/library/yql/tools/pgrun/ya.make b/ydb/library/yql/tools/pgrun/ya.make index db51090bd906..8420b3a60235 100644 --- a/ydb/library/yql/tools/pgrun/ya.make +++ b/ydb/library/yql/tools/pgrun/ya.make @@ -30,6 +30,8 @@ PEERDIR( ydb/library/yql/providers/common/udf_resolve ydb/library/yql/providers/dq/provider ydb/library/yql/providers/yt/gateway/file + ydb/library/yql/providers/yt/comp_nodes/llvm + ydb/library/yql/providers/yt/codec/codegen ydb/library/yql/core/url_preprocessing ydb/library/yql/parser/pg_wrapper ) diff --git a/ydb/library/yql/tools/yqlrun/ya.make b/ydb/library/yql/tools/yqlrun/ya.make index 0a50190dfd13..1c64983b2413 100644 --- a/ydb/library/yql/tools/yqlrun/ya.make +++ b/ydb/library/yql/tools/yqlrun/ya.make @@ -37,6 +37,8 @@ PEERDIR( ydb/library/yql/providers/common/udf_resolve ydb/library/yql/providers/dq/provider ydb/library/yql/providers/yt/gateway/file + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/core/url_preprocessing ydb/library/yql/tools/yqlrun/http ydb/library/yql/parser/pg_wrapper From 1c410386f4216f9a9ce89aadf5c3bf8ca3ef1d10 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Mon, 25 Dec 2023 18:44:01 +0000 Subject: [PATCH 2/2] build --- ydb/library/yql/core/extract_predicate/ut/ya.make | 2 ++ ydb/library/yql/core/ut/ya.make | 2 ++ ydb/library/yql/providers/yt/gateway/native/ut/ya.make | 2 ++ ydb/library/yql/providers/yt/provider/ut/ya.make | 2 ++ ydb/library/yql/public/purecalc/io_specs/protobuf/ut/ya.make | 2 +- ydb/library/yql/tools/dqrun/ya.make | 2 ++ 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ydb/library/yql/core/extract_predicate/ut/ya.make b/ydb/library/yql/core/extract_predicate/ut/ya.make index 5f35d9a3310b..05f327ea789e 100644 --- a/ydb/library/yql/core/extract_predicate/ut/ya.make +++ b/ydb/library/yql/core/extract_predicate/ut/ya.make @@ -18,6 +18,8 @@ PEERDIR( ydb/library/yql/providers/result/provider ydb/library/yql/providers/yt/gateway/file ydb/library/yql/providers/yt/provider + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/minikql/comp_nodes/llvm ydb/library/yql/minikql/invoke_builtins/llvm ydb/library/yql/sql/pg diff --git a/ydb/library/yql/core/ut/ya.make b/ydb/library/yql/core/ut/ya.make index 6f7fea234333..51e01803bf78 100644 --- a/ydb/library/yql/core/ut/ya.make +++ b/ydb/library/yql/core/ut/ya.make @@ -28,6 +28,8 @@ PEERDIR( ydb/library/yql/providers/result/provider ydb/library/yql/providers/yt/gateway/file ydb/library/yql/providers/yt/provider + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/minikql/comp_nodes/llvm ydb/library/yql/minikql/invoke_builtins/llvm ydb/library/yql/sql/pg diff --git a/ydb/library/yql/providers/yt/gateway/native/ut/ya.make b/ydb/library/yql/providers/yt/gateway/native/ut/ya.make index baeab8d74a8b..b0c48624f09a 100644 --- a/ydb/library/yql/providers/yt/gateway/native/ut/ya.make +++ b/ydb/library/yql/providers/yt/gateway/native/ut/ya.make @@ -7,6 +7,8 @@ SRCS( PEERDIR( ydb/library/yql/providers/yt/gateway/native ydb/library/yql/providers/yt/gateway/file + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/core/ut_common library/cpp/testing/mock_server library/cpp/testing/common diff --git a/ydb/library/yql/providers/yt/provider/ut/ya.make b/ydb/library/yql/providers/yt/provider/ut/ya.make index 67a0fd541fd7..2fc6b06cbcd0 100644 --- a/ydb/library/yql/providers/yt/provider/ut/ya.make +++ b/ydb/library/yql/providers/yt/provider/ut/ya.make @@ -12,6 +12,8 @@ PEERDIR( ydb/library/yql/providers/yt/lib/schema ydb/library/yql/providers/yt/provider ydb/library/yql/providers/yt/gateway/file + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/core/ut_common ydb/library/yql/ast ydb/library/yql/public/udf/service/terminate_policy diff --git a/ydb/library/yql/public/purecalc/io_specs/protobuf/ut/ya.make b/ydb/library/yql/public/purecalc/io_specs/protobuf/ut/ya.make index 8fb331b0f753..cb590a66a474 100644 --- a/ydb/library/yql/public/purecalc/io_specs/protobuf/ut/ya.make +++ b/ydb/library/yql/public/purecalc/io_specs/protobuf/ut/ya.make @@ -5,7 +5,7 @@ UNITTEST() PEERDIR( library/cpp/protobuf/util ydb/library/yql/public/udf/service/exception_policy - ydb/library/yql/public/purecalc/common + ydb/library/yql/public/purecalc ydb/library/yql/public/purecalc/io_specs/protobuf ydb/library/yql/public/purecalc/ut/protos ) diff --git a/ydb/library/yql/tools/dqrun/ya.make b/ydb/library/yql/tools/dqrun/ya.make index 6a1893092197..5643a263eacf 100644 --- a/ydb/library/yql/tools/dqrun/ya.make +++ b/ydb/library/yql/tools/dqrun/ya.make @@ -74,6 +74,8 @@ ENDIF() ydb/library/yql/providers/yt/codec/codegen ydb/library/yql/providers/yt/mkql_dq ydb/library/yql/providers/yt/provider + ydb/library/yql/providers/yt/codec/codegen + ydb/library/yql/providers/yt/comp_nodes/llvm ydb/library/yql/providers/yt/lib/yt_download ydb/library/yql/providers/yt/lib/yt_url_lister ydb/library/yql/providers/yt/lib/config_clusters