From 3de4df8fffd6677faf18034f8875b06816f5d0f6 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Fri, 30 Jul 2021 17:13:10 +0800 Subject: [PATCH 1/2] [New Featrue] Support Vectorization Execution Engine Interface For Doris --- be/CMakeLists.txt | 2 +- be/src/common/compiler_util.h | 5 + be/src/common/logging.h | 1 + be/src/common/status.h | 2 + be/src/exec/aggregation_node.cpp | 13 +- be/src/exec/broker_scan_node.cpp | 2 +- be/src/exec/cross_join_node.cpp | 2 +- be/src/exec/data_sink.cpp | 24 +- be/src/exec/data_sink.h | 1 + be/src/exec/exec_node.cpp | 62 +- be/src/exec/exec_node.h | 7 +- be/src/exec/olap_scan_node.cpp | 23 +- be/src/exec/olap_scan_node.h | 11 +- be/src/exec/olap_scanner.cpp | 1 - be/src/exec/olap_scanner.h | 8 +- be/src/exec/partitioned_aggregation_node.cc | 32 +- be/src/exec/schema_scan_node.cpp | 2 +- be/src/exec/schema_scanner.cpp | 6 +- be/src/exprs/agg_fn_evaluator.cpp | 4 +- be/src/exprs/agg_fn_evaluator.h | 14 +- be/src/exprs/aggregate_functions.cpp | 106 +- be/src/exprs/aggregate_functions.h | 19 +- be/src/exprs/anyval_util.h | 40 + be/src/exprs/expr_value.h | 4 +- be/src/exprs/hll_function.cpp | 8 +- be/src/exprs/hll_function.h | 3 + be/src/exprs/hybrid_set.h | 2 +- be/src/exprs/new_agg_fn_evaluator.cc | 6 +- be/src/exprs/new_agg_fn_evaluator.h | 22 +- be/src/exprs/timestamp_functions.cpp | 87 +- be/src/exprs/timestamp_functions.h | 2 + be/src/olap/block_column_predicate.h | 2 +- .../rowset/segment_v2/segment_iterator.cpp | 2 +- be/src/runtime/data_stream_recvr.cc | 2 - be/src/runtime/data_stream_sender.cpp | 6 +- be/src/runtime/datetime_value.cpp | 12 +- be/src/runtime/datetime_value.h | 115 ++- be/src/runtime/descriptors.cpp | 16 + be/src/runtime/descriptors.h | 15 + be/src/runtime/exec_env.h | 4 +- be/src/runtime/mysql_result_writer.cpp | 9 +- be/src/runtime/mysql_result_writer.h | 12 +- be/src/runtime/plan_fragment_executor.cpp | 15 +- be/src/runtime/plan_fragment_executor.h | 2 + be/src/runtime/result_sink.cpp | 1 + be/src/runtime/result_sink.h | 6 + be/src/runtime/result_writer.h | 10 +- be/src/runtime/row_batch.cpp | 3 + be/src/runtime/row_batch.h | 5 + be/src/runtime/runtime_state.h | 2 + be/src/runtime/type_limit.h | 8 +- be/src/runtime/types.h | 6 + be/src/service/internal_service.cpp | 12 + be/src/service/internal_service.h | 4 + be/src/util/binary_cast.hpp | 35 +- be/src/util/bitmap_value.h | 2 +- be/src/util/brpc_stub_cache.h | 8 +- be/src/util/hash_util.hpp | 2 +- be/src/util/radix_sort.h | 2 +- be/test/exec/broker_scan_node_test.cpp | 1 + be/test/exec/json_scanner_test.cpp | 2 + .../exec/json_scanner_test_with_jsonpath.cpp | 1 + be/test/exec/parquet_scanner_test.cpp | 1 + be/test/olap/column_reader_test.cpp | 188 ++-- be/test/olap/schema_change_test.cpp | 44 +- be/test/runtime/CMakeLists.txt | 2 +- be/test/runtime/memory_scratch_sink_test.cpp | 1 + be/test/util/arrow/arrow_work_flow_test.cpp | 1 + build.sh | 10 +- .../doris/analysis/AggregateInfoBase.java | 32 +- .../apache/doris/analysis/ArithmeticExpr.java | 199 ++-- .../doris/analysis/BinaryPredicate.java | 22 +- .../org/apache/doris/analysis/CaseExpr.java | 27 + .../org/apache/doris/analysis/CastExpr.java | 37 +- .../doris/analysis/CompoundPredicate.java | 17 + .../apache/doris/analysis/DateLiteral.java | 17 +- .../java/org/apache/doris/analysis/Expr.java | 70 +- .../doris/analysis/ExpressionFunctions.java | 3 +- .../doris/analysis/FunctionCallExpr.java | 10 +- .../apache/doris/analysis/InPredicate.java | 8 +- .../apache/doris/analysis/InlineViewRef.java | 5 +- .../doris/analysis/IsNullPredicate.java | 8 +- .../apache/doris/analysis/LikePredicate.java | 17 +- .../apache/doris/analysis/LiteralExpr.java | 5 + .../apache/doris/analysis/SlotDescriptor.java | 1 + .../org/apache/doris/analysis/SlotRef.java | 5 + .../doris/catalog/AggregateFunction.java | 118 ++- .../org/apache/doris/catalog/Catalog.java | 13 +- .../org/apache/doris/catalog/Function.java | 61 +- .../org/apache/doris/catalog/FunctionSet.java | 452 ++++++++- .../apache/doris/catalog/ScalarFunction.java | 196 ++-- .../common/profile/ProfileTreeBuilder.java | 5 +- .../doris/common/util/VectorizedUtil.java | 30 + .../apache/doris/planner/AggregationNode.java | 1 - .../org/apache/doris/planner/PlanNode.java | 67 +- .../org/apache/doris/planner/Planner.java | 5 + .../org/apache/doris/planner/ResultSink.java | 7 +- .../doris/planner/SingleNodePlanner.java | 4 +- .../java/org/apache/doris/qe/Coordinator.java | 1 + .../org/apache/doris/qe/SessionVariable.java | 19 +- .../org/apache/doris/qe/StmtExecutor.java | 3 +- .../apache/doris/analysis/CaseExprTest.java | 69 ++ .../apache/doris/catalog/FunctionSetTest.java | 8 +- .../org/apache/doris/planner/PlannerTest.java | 2 +- .../doris/planner/StreamLoadScanNodeTest.java | 4 +- fe/fe-core/src/test/resources/log4j2.xml | 2 +- gensrc/proto/data.proto | 54 +- gensrc/proto/internal_service.proto | 3 + gensrc/proto/palo_internal_service.proto | 47 + gensrc/script/doris_builtins_functions.py | 927 +++++++++++------- gensrc/script/gen_builtins_functions.py | 45 +- gensrc/thrift/DataSinks.thrift | 4 +- gensrc/thrift/Exprs.thrift | 3 + gensrc/thrift/PaloInternalService.thrift | 5 +- gensrc/thrift/PlanNodes.thrift | 4 +- gensrc/thrift/Status.thrift | 12 +- gensrc/thrift/Types.thrift | 1 + run-be-ut.sh | 2 +- thirdparty/build-thirdparty.sh | 16 + thirdparty/vars.sh | 16 +- 120 files changed, 2769 insertions(+), 1010 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/CaseExprTest.java create mode 100644 gensrc/proto/palo_internal_service.proto diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 38ef7f9f18378d..a4ec87d7f95ea3 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -329,7 +329,7 @@ check_function_exists(sched_getcpu HAVE_SCHED_GETCPU) # -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG: enable nanosecond precision for boost # -fno-omit-frame-pointers: Keep frame pointer for functions in register set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-pragmas -pthread -Werror=strict-aliasing") -set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-omit-frame-pointer") +set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fstrict-aliasing -fno-omit-frame-pointer -Werror=return-type") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++17 -D__STDC_FORMAT_MACROS") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") diff --git a/be/src/common/compiler_util.h b/be/src/common/compiler_util.h index accc70c3844b5a..1482134e2f2a9f 100644 --- a/be/src/common/compiler_util.h +++ b/be/src/common/compiler_util.h @@ -42,7 +42,12 @@ /// not a command. This should be used sparingly for cases when either the function /// needs to be inlined for a specific reason or the compiler's heuristics make a bad /// decision, e.g. not inlining a small function on a hot path. +#ifdef ALWAYS_INLINE +#undef ALWAYS_INLINE +#endif #define ALWAYS_INLINE __attribute__((always_inline)) +#define NO_INLINE __attribute__((__noinline__)) +#define MAY_ALIAS __attribute__((__may_alias__)) #define ALIGN_CACHE_LINE __attribute__((aligned(CACHE_LINE_SIZE))) diff --git a/be/src/common/logging.h b/be/src/common/logging.h index 62265cec774240..7c4d35e975adc1 100644 --- a/be/src/common/logging.h +++ b/be/src/common/logging.h @@ -85,4 +85,5 @@ DCHECK(a == b) << "[ " #a " = " << static_cast(a) << " , " #b " = " \ << static_cast(b) << " ]" +#include #endif diff --git a/be/src/common/status.h b/be/src/common/status.h index 9f9547237f5b6c..50cffd1f9290b3 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -235,6 +235,8 @@ class Status { /// trailing message. Status clone_and_append(const Slice& msg) const; + operator bool() { return this->ok(); } + private: const char* copy_state(const char* state); diff --git a/be/src/exec/aggregation_node.cpp b/be/src/exec/aggregation_node.cpp index 43e3bd8c90db13..df3bc5382d06bc 100644 --- a/be/src/exec/aggregation_node.cpp +++ b/be/src/exec/aggregation_node.cpp @@ -19,7 +19,6 @@ #include #include -#include #include @@ -223,6 +222,15 @@ Status AggregationNode::open(RuntimeState* state) { } Status AggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) { + // 1. `!need_finalize` means this aggregation node not the level two aggregation node + // 2. `_singleton_output_tuple != nullptr` means is not group by + // 3. `child(0)->rows_returned() == 0` mean not data from child + // in level two aggregation node should return NULL result + // level one aggregation node set `eos = true` return directly + if (UNLIKELY(!_needs_finalize && _singleton_output_tuple != nullptr && child(0)->rows_returned() == 0)) { + *eos = true; + return Status::OK(); + } SCOPED_TIMER(_runtime_profile->total_time_counter()); RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT)); RETURN_IF_CANCELLED(state); @@ -403,7 +411,8 @@ Tuple* AggregationNode::finalize_tuple(Tuple* tuple, MemPool* pool) { dst = Tuple::create(_output_tuple_desc->byte_size(), pool); } if (_needs_finalize) { - AggFnEvaluator::finalize(_aggregate_evaluators, _agg_fn_ctxs, tuple, dst); + AggFnEvaluator::finalize(_aggregate_evaluators, _agg_fn_ctxs, tuple, dst, + _singleton_output_tuple != nullptr && child(0)->rows_returned() == 0); } else { AggFnEvaluator::serialize(_aggregate_evaluators, _agg_fn_ctxs, tuple); } diff --git a/be/src/exec/broker_scan_node.cpp b/be/src/exec/broker_scan_node.cpp index 348729ade3c7ee..7c696b6b828516 100644 --- a/be/src/exec/broker_scan_node.cpp +++ b/be/src/exec/broker_scan_node.cpp @@ -47,7 +47,7 @@ BrokerScanNode::BrokerScanNode(ObjectPool* pool, const TPlanNode& tnode, const D BrokerScanNode::~BrokerScanNode() {} Status BrokerScanNode::init(const TPlanNode& tnode, RuntimeState* state) { - RETURN_IF_ERROR(ScanNode::init(tnode)); + RETURN_IF_ERROR(ScanNode::init(tnode, state)); auto& broker_scan_node = tnode.broker_scan_node; if (broker_scan_node.__isset.pre_filter_exprs) { diff --git a/be/src/exec/cross_join_node.cpp b/be/src/exec/cross_join_node.cpp index 177df55196af16..8ef9b662f8fb65 100644 --- a/be/src/exec/cross_join_node.cpp +++ b/be/src/exec/cross_join_node.cpp @@ -60,7 +60,7 @@ Status CrossJoinNode::construct_build_side(RuntimeState* state) { RETURN_IF_CANCELLED(state); // TODO(zhaochun): // RETURN_IF_ERROR(state->CheckQueryState()); - bool eos = true; + bool eos = false; RETURN_IF_ERROR(child(1)->get_next(state, batch, &eos)); // to prevent use too many memory diff --git a/be/src/exec/data_sink.cpp b/be/src/exec/data_sink.cpp index ef85dbca2c84ed..123f621a34a1e3 100644 --- a/be/src/exec/data_sink.cpp +++ b/be/src/exec/data_sink.cpp @@ -42,6 +42,7 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink const std::vector& output_exprs, const TPlanFragmentExecParams& params, const RowDescriptor& row_desc, + bool is_vec, boost::scoped_ptr* sink) { DataSink* tmp_sink = NULL; @@ -55,9 +56,12 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink ? params.send_query_statistics_with_every_batch : false; // TODO: figure out good buffer size based on size of output row - tmp_sink = new DataStreamSender(pool, params.sender_id, row_desc, thrift_sink.stream_sink, - params.destinations, 16 * 1024, - send_query_statistics_with_every_batch); + if (is_vec) { + } else { + tmp_sink = new DataStreamSender(pool, params.sender_id, row_desc, thrift_sink.stream_sink, + params.destinations, 16 * 1024, + send_query_statistics_with_every_batch); + } // RETURN_IF_ERROR(sender->prepare(state->obj_pool(), thrift_sink.stream_sink)); sink->reset(tmp_sink); break; @@ -68,7 +72,10 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink } // TODO: figure out good buffer size based on size of output row - tmp_sink = new ResultSink(row_desc, output_exprs, thrift_sink.result_sink, 1024); + if (is_vec) { + } else { + tmp_sink = new ResultSink(row_desc, output_exprs, thrift_sink.result_sink, 1024); + } sink->reset(tmp_sink); break; case TDataSinkType::MEMORY_SCRATCH_SINK: @@ -98,8 +105,7 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink if (!thrift_sink.__isset.odbc_table_sink) { return Status::InternalError("Missing data odbc sink."); } - OdbcTableSink* odbc_tbl_sink = new OdbcTableSink(pool, - row_desc, output_exprs); + OdbcTableSink* odbc_tbl_sink = new OdbcTableSink(pool, row_desc, output_exprs); sink->reset(odbc_tbl_sink); break; } @@ -158,9 +164,9 @@ Status DataSink::init(const TDataSink& thrift_sink) { } Status DataSink::prepare(RuntimeState* state) { - _expr_mem_tracker = MemTracker::CreateTracker( - -1, _name + ":Expr:" + std::to_string(state->load_job_id()), - state->instance_mem_tracker()); + _expr_mem_tracker = + MemTracker::CreateTracker(-1, _name + ":Expr:" + std::to_string(state->load_job_id()), + state->instance_mem_tracker()); return Status::OK(); } diff --git a/be/src/exec/data_sink.h b/be/src/exec/data_sink.h index e4c6729cbc7d3d..e351427ec31a5d 100644 --- a/be/src/exec/data_sink.h +++ b/be/src/exec/data_sink.h @@ -74,6 +74,7 @@ class DataSink { const std::vector& output_exprs, const TPlanFragmentExecParams& params, const RowDescriptor& row_desc, + bool is_vec, boost::scoped_ptr* sink); // Returns the runtime profile for the sink. diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp index 675cb1e4c3e2f5..18b5af196a403f 100644 --- a/be/src/exec/exec_node.cpp +++ b/be/src/exec/exec_node.cpp @@ -127,9 +127,7 @@ ExecNode::ExecNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl _rows_returned_counter(NULL), _rows_returned_rate(NULL), _memory_used_counter(NULL), - _is_closed(false) { - init_runtime_profile(print_plan_node_type(tnode.node_type)); -} + _is_closed(false) {} ExecNode::~ExecNode() {} @@ -159,7 +157,18 @@ void ExecNode::push_down_predicate(RuntimeState* state, std::list* } Status ExecNode::init(const TPlanNode& tnode, RuntimeState* state) { + std::string profile; + if (state && state->enable_vectorized_exec()) { + profile = "V" + print_plan_node_type(tnode.node_type); + } else { + profile = print_plan_node_type(tnode.node_type); + } + init_runtime_profile(profile); + + if (tnode.__isset.vconjunct) { + } RETURN_IF_ERROR(Expr::create_expr_trees(_pool, tnode.conjuncts, &_conjunct_ctxs)); + return Status::OK(); } @@ -178,11 +187,11 @@ Status ExecNode::prepare(RuntimeState* state) { _expr_mem_tracker = MemTracker::CreateTracker(-1, "ExecNode:Exprs:" + _runtime_profile->name(), _mem_tracker); _expr_mem_pool.reset(new MemPool(_expr_mem_tracker.get())); - // TODO chenhao + RETURN_IF_ERROR(Expr::prepare(_conjunct_ctxs, state, row_desc(), expr_mem_tracker())); + // TODO(zc): // AddExprCtxsToFree(_conjunct_ctxs); - for (int i = 0; i < _children.size(); ++i) { RETURN_IF_ERROR(_children[i]->prepare(state)); } @@ -362,22 +371,32 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN return Status::OK(); case TPlanNodeType::OLAP_SCAN_NODE: - *node = pool->add(new OlapScanNode(pool, tnode, descs)); + if (state->enable_vectorized_exec()) { + } else { + *node = pool->add(new OlapScanNode(pool, tnode, descs)); + } return Status::OK(); case TPlanNodeType::AGGREGATION_NODE: - if (config::enable_partitioned_aggregation) { - *node = pool->add(new PartitionedAggregationNode(pool, tnode, descs)); + if (state->enable_vectorized_exec()) { } else { - *node = pool->add(new AggregationNode(pool, tnode, descs)); + if (config::enable_partitioned_aggregation) { + *node = pool->add(new PartitionedAggregationNode(pool, tnode, descs)); + } else { + *node = pool->add(new AggregationNode(pool, tnode, descs)); + } } return Status::OK(); + case TPlanNodeType::HASH_JOIN_NODE: *node = pool->add(new HashJoinNode(pool, tnode, descs)); return Status::OK(); case TPlanNodeType::CROSS_JOIN_NODE: - *node = pool->add(new CrossJoinNode(pool, tnode, descs)); + if (state->enable_vectorized_exec()) { + } else { + *node = pool->add(new CrossJoinNode(pool, tnode, descs)); + } return Status::OK(); case TPlanNodeType::MERGE_JOIN_NODE: @@ -389,7 +408,10 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN return Status::OK(); case TPlanNodeType::EXCHANGE_NODE: - *node = pool->add(new ExchangeNode(pool, tnode, descs)); + if (state->enable_vectorized_exec()) { + } else { + *node = pool->add(new ExchangeNode(pool, tnode, descs)); + } return Status::OK(); case TPlanNodeType::SELECT_NODE: @@ -401,10 +423,13 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN return Status::OK(); case TPlanNodeType::SORT_NODE: - if (tnode.sort_node.use_top_n) { - *node = pool->add(new TopNNode(pool, tnode, descs)); + if (state->enable_vectorized_exec()) { } else { - *node = pool->add(new SpillSortNode(pool, tnode, descs)); + if (tnode.sort_node.use_top_n) { + *node = pool->add(new TopNNode(pool, tnode, descs)); + } else { + *node = pool->add(new SpillSortNode(pool, tnode, descs)); + } } return Status::OK(); @@ -417,7 +442,10 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN return Status::OK(); case TPlanNodeType::UNION_NODE: - *node = pool->add(new UnionNode(pool, tnode, descs)); + if (state->enable_vectorized_exec()) { + } else { + *node = pool->add(new UnionNode(pool, tnode, descs)); + } return Status::OK(); case TPlanNodeType::INTERSECT_NODE: @@ -624,4 +652,8 @@ Status ExecNode::QueryMaintenance(RuntimeState* state, const std::string& msg) { return state->check_query_state(msg); } +Status ExecNode::get_next(RuntimeState* state, vectorized::Block* block, bool* eos) { + return Status::NotSupported("Not Implemented get block"); +} + } // namespace doris diff --git a/be/src/exec/exec_node.h b/be/src/exec/exec_node.h index 74baa27e68d74a..dfaeb2c0ca035a 100644 --- a/be/src/exec/exec_node.h +++ b/be/src/exec/exec_node.h @@ -34,7 +34,6 @@ #include "util/uid_util.h" // for print_id namespace doris { - class Expr; class ExprContext; class ObjectPool; @@ -46,6 +45,11 @@ class TupleRow; class DataSink; class MemTracker; +namespace vectorized { +class Block; +class VExpr; +} + using std::string; using std::stringstream; using std::vector; @@ -97,6 +101,7 @@ class ExecNode { // Caller must not be holding any io buffers. This will cause deadlock. // TODO: AggregationNode and HashJoinNode cannot be "re-opened" yet. virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) = 0; + virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos); // Resets the stream of row batches to be retrieved by subsequent GetNext() calls. // Clears all internal state, returning this node to the state it was in after calling diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index 3da65b489a6c95..d1f63c1b8b931c 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -20,14 +20,12 @@ #include #include #include -#include #include #include #include "agent/cgroups_mgr.h" #include "common/logging.h" #include "common/resource_tls.h" -#include "exprs/binary_predicate.h" #include "exprs/expr.h" #include "exprs/expr_context.h" #include "exprs/runtime_filter.h" @@ -38,7 +36,6 @@ #include "runtime/runtime_state.h" #include "runtime/string_value.h" #include "runtime/tuple_row.h" -#include "util/debug_util.h" #include "util/priority_thread_pool.hpp" #include "util/runtime_profile.h" @@ -68,7 +65,7 @@ OlapScanNode::~OlapScanNode() {} Status OlapScanNode::init(const TPlanNode& tnode, RuntimeState* state) { RETURN_IF_ERROR(ExecNode::init(tnode, state)); - _direct_conjunct_size = _conjunct_ctxs.size(); + _direct_conjunct_size = state->enable_vectorized_exec() ? 1 : _conjunct_ctxs.size(); const TQueryOptions& query_options = state->query_options(); if (query_options.__isset.max_scan_key_num) { @@ -157,6 +154,9 @@ void OlapScanNode::_init_counter(RuntimeState* state) { _scanner_wait_batch_timer = ADD_TIMER(_runtime_profile, "ScannerBatchWaitTime"); // time of scan thread to wait for worker thread of the thread pool _scanner_wait_worker_timer = ADD_TIMER(_runtime_profile, "ScannerWorkerWaitTime"); + + // time of node to wait for batch/block queue + _olap_wait_batch_queue_timer = ADD_TIMER(_runtime_profile, "BatchQueueWaitTime"); } Status OlapScanNode::prepare(RuntimeState* state) { @@ -286,6 +286,7 @@ Status OlapScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eo RowBatch* materialized_batch = NULL; { std::unique_lock l(_row_batches_lock); + SCOPED_TIMER(_olap_wait_batch_queue_timer); while (_materialized_row_batches.empty() && !_transfer_done) { if (state->is_cancelled()) { _transfer_done = true; @@ -454,6 +455,7 @@ Status OlapScanNode::start_scan(RuntimeState* state) { VLOG_CRITICAL << "Filter idle conjuncts"; // 4. Filter idle conjunct which already trans to olap filters` + // TODO: filter idle conjunct in vexpr_contexts remove_pushed_conjuncts(state); VLOG_CRITICAL << "BuildScanKey"; @@ -515,6 +517,9 @@ void OlapScanNode::remove_pushed_conjuncts(RuntimeState* state) { iter->second->runtimefilter->set_push_down_profile(); } } + // set vconjunct_ctx is empty, if all conjunct + if (_direct_conjunct_size == 0) { + } } void OlapScanNode::eval_const_conjuncts() { @@ -654,11 +659,11 @@ Status OlapScanNode::build_scan_key() { return Status::OK(); } -static Status get_hints(const TPaloScanRange& scan_range, int block_row_count, - bool is_begin_include, bool is_end_include, - const std::vector>& scan_key_range, - std::vector>* sub_scan_range, - RuntimeProfile* profile) { +Status OlapScanNode::get_hints(const TPaloScanRange& scan_range, int block_row_count, + bool is_begin_include, bool is_end_include, + const std::vector>& scan_key_range, + std::vector>* sub_scan_range, + RuntimeProfile* profile) { auto tablet_id = scan_range.tablet_id; int32_t schema_hash = strtoul(scan_range.schema_hash.c_str(), NULL, 10); std::string err; diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h index 8e59e23e93fe0b..c7142a5e5a4605 100644 --- a/be/src/exec/olap_scan_node.h +++ b/be/src/exec/olap_scan_node.h @@ -148,7 +148,7 @@ class OlapScanNode : public ScanNode { Status normalize_conjuncts(); Status build_olap_filters(); Status build_scan_key(); - Status start_scan_thread(RuntimeState* state); + virtual Status start_scan_thread(RuntimeState* state); template Status normalize_predicate(ColumnValueRange& range, SlotDescriptor* slot); @@ -179,7 +179,6 @@ class OlapScanNode : public ScanNode { const std::vector& runtime_filter_descs() { return _runtime_filter_descs; } -private: void _init_counter(RuntimeState* state); // OLAP_SCAN_NODE profile layering: OLAP_SCAN_NODE, OlapScanner, and SegmentIterator // according to the calling relationship @@ -194,6 +193,12 @@ class OlapScanNode : public ScanNode { std::pair should_push_down_eq_predicate(SlotDescriptor* slot, Expr* pred, int conj_idx, int child_idx); + static Status get_hints(const TPaloScanRange& scan_range, int block_row_count, + bool is_begin_include, bool is_end_include, + const std::vector>& scan_key_range, + std::vector>* sub_scan_range, + RuntimeProfile* profile); + friend class OlapScanner; // Tuple id resolved in prepare() to set _tuple_desc; @@ -356,6 +361,8 @@ class OlapScanNode : public ScanNode { RuntimeProfile::Counter* _scanner_wait_batch_timer = nullptr; RuntimeProfile::Counter* _scanner_wait_worker_timer = nullptr; + + RuntimeProfile::Counter* _olap_wait_batch_queue_timer = nullptr; }; } // namespace doris diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index ebe2200755f007..64053e99456184 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -17,7 +17,6 @@ #include "olap_scanner.h" -#include #include #include "gen_cpp/PaloInternalService_types.h" diff --git a/be/src/exec/olap_scanner.h b/be/src/exec/olap_scanner.h index 6dbd2fdbd26f1b..65f6208b20fe1c 100644 --- a/be/src/exec/olap_scanner.h +++ b/be/src/exec/olap_scanner.h @@ -94,7 +94,11 @@ class OlapScanner { std::vector* mutable_runtime_filter_marks() { return &_runtime_filter_marks; } -private: + const std::vector& get_query_slots() const { + return _query_slots; + } + +protected: Status _init_params(const std::vector& key_ranges, const std::vector& filters, const std::vector>>& @@ -105,7 +109,7 @@ class OlapScanner { // Update profile that need to be reported in realtime. void _update_realtime_counter(); -private: +protected: RuntimeState* _runtime_state; OlapScanNode* _parent; const TupleDescriptor* _tuple_desc; /**< tuple descriptor */ diff --git a/be/src/exec/partitioned_aggregation_node.cc b/be/src/exec/partitioned_aggregation_node.cc index 7e5d5f3eabed46..8f31ad80f349c6 100644 --- a/be/src/exec/partitioned_aggregation_node.cc +++ b/be/src/exec/partitioned_aggregation_node.cc @@ -30,7 +30,6 @@ #include "exprs/new_agg_fn_evaluator.h" // #include "exprs/scalar_expr_evaluator.h" #include "exprs/slot_ref.h" -#include "gen_cpp/Exprs_types.h" #include "gen_cpp/PlanNodes_types.h" #include "gutil/strings/substitute.h" #include "runtime/buffered_tuple_stream3.inline.h" @@ -147,7 +146,7 @@ PartitionedAggregationNode::PartitionedAggregationNode(ObjectPool* pool, const T } Status PartitionedAggregationNode::init(const TPlanNode& tnode, RuntimeState* state) { - RETURN_IF_ERROR(ExecNode::init(tnode)); + RETURN_IF_ERROR(ExecNode::init(tnode, state)); DCHECK(intermediate_tuple_desc_ != nullptr); DCHECK(output_tuple_desc_ != nullptr); DCHECK_EQ(intermediate_tuple_desc_->slots().size(), output_tuple_desc_->slots().size()); @@ -345,6 +344,15 @@ Status PartitionedAggregationNode::open(RuntimeState* state) { } Status PartitionedAggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) { + // 1. `!need_finalize` means this aggregation node not the level two aggregation node + // 2. `grouping_exprs_.size() == 0 ` means is not group by + // 3. `child(0)->rows_returned() == 0` mean not data from child + // in level two aggregation node should return NULL result + // level one aggregation node set `eos = true` return directly + if (UNLIKELY(grouping_exprs_.size() == 0 && !needs_finalize_ && child(0)->rows_returned() == 0)) { + *eos = true; + return Status::OK(); + } // PartitionedAggregationNode is a spill node, GetNextInternal will read tuple from a tuple stream // then copy the pointer to a RowBatch, it can only guarantee that the life cycle is valid in a batch stage. // If the ancestor node is a no-spilling blocking node (such as hash_join_node except_node ...) @@ -1014,23 +1022,6 @@ void PartitionedAggregationNode::InitAggSlots(const vector& // initialize the value to max/min possible value for the same effect. NewAggFnEvaluator* eval = agg_fn_evals[i]; eval->Init(intermediate_tuple); - - DCHECK(agg_fns_[i] == &(eval->agg_fn())); - const AggFn* agg_fn = agg_fns_[i]; - const AggFn::AggregationOp agg_op = agg_fn->agg_op(); - if ((agg_op == AggFn::MIN || agg_op == AggFn::MAX) && - !agg_fn->intermediate_type().is_string_type() && - !agg_fn->intermediate_type().is_date_type()) { - ExprValue default_value; - void* default_value_ptr = NULL; - if (agg_op == AggFn::MIN) { - default_value_ptr = default_value.set_to_max((*slot_desc)->type()); - } else { - DCHECK_EQ(agg_op, AggFn::MAX); - default_value_ptr = default_value.set_to_min((*slot_desc)->type()); - } - RawValue::write(default_value_ptr, intermediate_tuple, *slot_desc, NULL); - } } } @@ -1054,7 +1045,8 @@ Tuple* PartitionedAggregationNode::GetOutputTuple(const vectorbyte_size(), pool); } if (needs_finalize_) { - NewAggFnEvaluator::Finalize(agg_fn_evals, tuple, dst); + NewAggFnEvaluator::Finalize(agg_fn_evals, tuple, dst, + grouping_exprs_.size() == 0 && child(0)->rows_returned() == 0); } else { NewAggFnEvaluator::Serialize(agg_fn_evals, tuple); } diff --git a/be/src/exec/schema_scan_node.cpp b/be/src/exec/schema_scan_node.cpp index b9c26ad53982b6..1a6352cd0d070f 100644 --- a/be/src/exec/schema_scan_node.cpp +++ b/be/src/exec/schema_scan_node.cpp @@ -51,7 +51,7 @@ SchemaScanNode::~SchemaScanNode() { } Status SchemaScanNode::init(const TPlanNode& tnode, RuntimeState* state) { - RETURN_IF_ERROR(ExecNode::init(tnode)); + RETURN_IF_ERROR(ExecNode::init(tnode, state)); if (tnode.schema_scan_node.__isset.db) { _scanner_param.db = _pool->add(new std::string(tnode.schema_scan_node.db)); } diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 4cbbb31fb9eaf2..ef9ded8866af2d 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -127,7 +127,11 @@ Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { for (int i = 0; i < _column_num; ++i) { TSlotDescriptor t_slot_desc; - t_slot_desc.__set_slotType(TypeDescriptor(_columns[i].type).to_thrift()); + if (_columns[i].type == TYPE_DECIMALV2) { + t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv2_type(27, 9).to_thrift()); + } else { + t_slot_desc.__set_slotType(TypeDescriptor(_columns[i].type).to_thrift()); + } t_slot_desc.__set_colName(_columns[i].name); t_slot_desc.__set_columnPos(i); t_slot_desc.__set_byteOffset(offset); diff --git a/be/src/exprs/agg_fn_evaluator.cpp b/be/src/exprs/agg_fn_evaluator.cpp index 232be0aa09b76d..5e11f2094883f8 100644 --- a/be/src/exprs/agg_fn_evaluator.cpp +++ b/be/src/exprs/agg_fn_evaluator.cpp @@ -808,7 +808,7 @@ void AggFnEvaluator::choose_update_or_merge(FunctionContext* agg_fn_ctx, TupleRo void AggFnEvaluator::serialize_or_finalize(FunctionContext* agg_fn_ctx, Tuple* src, const SlotDescriptor* dst_slot_desc, Tuple* dst, - void* fn) { + void* fn, bool add_null) { // DCHECK_EQ(dst_slot_desc->type().type, _return_type.type); if (src == NULL) { src = dst; @@ -818,7 +818,7 @@ void AggFnEvaluator::serialize_or_finalize(FunctionContext* agg_fn_ctx, Tuple* s } // same - bool src_slot_null = src->is_null(_intermediate_slot_desc->null_indicator_offset()); + bool src_slot_null = add_null || src->is_null(_intermediate_slot_desc->null_indicator_offset()); void* src_slot = NULL; if (!src_slot_null) { diff --git a/be/src/exprs/agg_fn_evaluator.h b/be/src/exprs/agg_fn_evaluator.h index 83953a4e5ed0db..2ed950df3e049f 100644 --- a/be/src/exprs/agg_fn_evaluator.h +++ b/be/src/exprs/agg_fn_evaluator.h @@ -126,7 +126,7 @@ class AggFnEvaluator { // In the non-spilling case, this node would normally not merge. void merge(FunctionContext* agg_fn_ctx, Tuple* src, Tuple* dst); void serialize(FunctionContext* agg_fn_ctx, Tuple* dst); - void finalize(FunctionContext* agg_fn_ctx, Tuple* src, Tuple* dst); + void finalize(FunctionContext* agg_fn_ctx, Tuple* src, Tuple* dst, bool add_null = false); // TODO: implement codegen path. These functions would return IR functions with // the same signature as the interpreted ones above. @@ -167,7 +167,7 @@ class AggFnEvaluator { Tuple* dst); static void finalize(const std::vector& evaluators, const std::vector& fn_ctxs, Tuple* src, - Tuple* dst); + Tuple* dst, bool add_null = false); static void init(const std::vector& evaluators, const std::vector& fn_ctxs, Tuple* dst); static void serialize(const std::vector& evaluators, @@ -260,7 +260,7 @@ class AggFnEvaluator { // taking TupleRow to the UDA signature taking AnvVals. // void serialize_or_finalize(FunctionContext* agg_fn_ctx, const SlotDescriptor* dst_slot_desc, Tuple* dst, void* fn); void serialize_or_finalize(FunctionContext* agg_fn_ctx, Tuple* src, - const SlotDescriptor* dst_slot_desc, Tuple* dst, void* fn); + const SlotDescriptor* dst_slot_desc, Tuple* dst, void* fn, bool add_null = false); // Writes the result in src into dst pointed to by _output_slot_desc void set_output_slot(const doris_udf::AnyVal* src, const SlotDescriptor* dst_slot_desc, @@ -280,8 +280,8 @@ inline void AggFnEvaluator::remove(doris_udf::FunctionContext* agg_fn_ctx, Tuple } inline void AggFnEvaluator::finalize(doris_udf::FunctionContext* agg_fn_ctx, Tuple* src, - Tuple* dst) { - serialize_or_finalize(agg_fn_ctx, src, _output_slot_desc, dst, _finalize_fn); + Tuple* dst, bool add_null) { + serialize_or_finalize(agg_fn_ctx, src, _output_slot_desc, dst, _finalize_fn, add_null); } inline void AggFnEvaluator::get_value(doris_udf::FunctionContext* agg_fn_ctx, Tuple* src, Tuple* dst) { @@ -335,11 +335,11 @@ inline void AggFnEvaluator::get_value(const std::vector& evalua } inline void AggFnEvaluator::finalize(const std::vector& evaluators, const std::vector& fn_ctxs, - Tuple* src, Tuple* dst) { + Tuple* src, Tuple* dst, bool add_null) { DCHECK_EQ(evaluators.size(), fn_ctxs.size()); for (int i = 0; i < evaluators.size(); ++i) { - evaluators[i]->finalize(fn_ctxs[i], src, dst); + evaluators[i]->finalize(fn_ctxs[i], src, dst, add_null); } } diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp index b5a80771c27801..dc066f504a244a 100644 --- a/be/src/exprs/aggregate_functions.cpp +++ b/be/src/exprs/aggregate_functions.cpp @@ -58,13 +58,38 @@ void AggregateFunctions::init_null(FunctionContext*, AnyVal* dst) { } template -void AggregateFunctions::init_zero(FunctionContext*, T* dst) { +void AggregateFunctions::init_zero_not_null(FunctionContext*, T* dst) { + dst->is_null = false; + dst->val = 0; +} + +template <> +void AggregateFunctions::init_zero_not_null(FunctionContext*, DecimalV2Val* dst) { dst->is_null = false; + dst->set_to_zero(); +} + +template +void AggregateFunctions::init_zero(FunctionContext*, T* dst) { + dst->is_null = true; dst->val = 0; } template <> void AggregateFunctions::init_zero(FunctionContext*, DecimalV2Val* dst) { + dst->is_null = true; + dst->set_to_zero(); +} + +template +void AggregateFunctions::init_zero_null(FunctionContext*, T* dst) { + dst->is_null = true; + dst->val = 0; +} + +template <> +void AggregateFunctions::init_zero_null(FunctionContext*, DecimalV2Val* dst) { + dst->is_null = true; dst->set_to_zero(); } @@ -82,7 +107,7 @@ void AggregateFunctions::sum_remove(FunctionContext* ctx, const SRC_VAL& src, DS return; } if (dst->is_null) { - init_zero(ctx, dst); + init_zero_not_null(ctx, dst); } dst->val -= src.val; } @@ -98,7 +123,7 @@ void AggregateFunctions::sum_remove(FunctionContext* ctx, const DecimalV2Val& sr return; } if (dst->is_null) { - init_zero(ctx, dst); + init_zero_not_null(ctx, dst); } DecimalV2Value new_src = DecimalV2Value::from_decimal_val(src); @@ -479,9 +504,8 @@ void AggregateFunctions::sum(FunctionContext* ctx, const SRC_VAL& src, DST_VAL* } if (dst->is_null) { - init_zero(ctx, dst); + init_zero_not_null(ctx, dst); } - dst->val += src.val; } @@ -516,6 +540,14 @@ void AggregateFunctions::sum(FunctionContext* ctx, const LargeIntVal& src, Large dst->val += src.val; } +template +void AggregateFunctions::min_init(FunctionContext* ctx, T* dst) { + auto val = AnyValUtil::max_val(ctx); + // set to null when intermediate slot is nullable + val.is_null = true; + *dst = val; +} + template void AggregateFunctions::min(FunctionContext*, const T& src, T* dst) { if (src.is_null) { @@ -527,6 +559,14 @@ void AggregateFunctions::min(FunctionContext*, const T& src, T* dst) { } } +template +void AggregateFunctions::max_init(FunctionContext* ctx, T* dst) { + auto val = AnyValUtil::min_val(ctx); + // set to null when intermediate slot is nullable + val.is_null = true; + *dst = val; +} + template void AggregateFunctions::max(FunctionContext*, const T& src, T* dst) { if (src.is_null) { @@ -723,7 +763,7 @@ void AggregateFunctions::string_concat_update(FunctionContext* ctx, const String return; } const StringVal* sep = separator.is_null ? &DEFAULT_STRING_CONCAT_DELIM : &separator; - if (result->is_null) { + if (result->is_null || !result->ptr) { // Header of the intermediate state holds the length of the first separator. const auto header_len = sizeof(StringConcatHeader); DCHECK(header_len == sizeof(sep->len)); @@ -739,7 +779,7 @@ void AggregateFunctions::string_concat_merge(FunctionContext* ctx, const StringV return; } const auto header_len = sizeof(StringConcatHeader); - if (result->is_null) { + if (result->is_null || !result->ptr) { // Copy the header from the first intermediate value. *result = StringVal(ctx->allocate(header_len), header_len); if (result->is_null) { @@ -1900,8 +1940,7 @@ DoubleVal AggregateFunctions::knuth_var_finalize(FunctionContext* ctx, const Str } DecimalV2Val AggregateFunctions::decimalv2_knuth_var_finalize(FunctionContext* ctx, - const StringVal& state_sv) { - DCHECK(!state_sv.is_null); + const StringVal& state_sv) { DCHECK_EQ(state_sv.len, sizeof(DecimalV2KnuthVarianceState)); DecimalV2KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); @@ -1914,8 +1953,7 @@ DecimalV2Val AggregateFunctions::decimalv2_knuth_var_finalize(FunctionContext* c } DoubleVal AggregateFunctions::knuth_var_pop_finalize(FunctionContext* ctx, - const StringVal& state_sv) { - DCHECK(!state_sv.is_null); + const StringVal& state_sv) { DCHECK_EQ(state_sv.len, sizeof(KnuthVarianceState)); KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); if (state->count == 0) return DoubleVal::null(); @@ -1925,8 +1963,7 @@ DoubleVal AggregateFunctions::knuth_var_pop_finalize(FunctionContext* ctx, } DecimalV2Val AggregateFunctions::decimalv2_knuth_var_pop_finalize(FunctionContext* ctx, - const StringVal& state_sv) { - DCHECK(!state_sv.is_null); + const StringVal& state_sv) { DCHECK_EQ(state_sv.len, sizeof(DecimalV2KnuthVarianceState)); DecimalV2KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); @@ -1940,7 +1977,6 @@ DecimalV2Val AggregateFunctions::decimalv2_knuth_var_pop_finalize(FunctionContex DoubleVal AggregateFunctions::knuth_stddev_finalize(FunctionContext* ctx, const StringVal& state_sv) { - DCHECK(!state_sv.is_null); DCHECK_EQ(state_sv.len, sizeof(KnuthVarianceState)); KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); if (state->count == 0 || state->count == 1) return DoubleVal::null(); @@ -1950,8 +1986,7 @@ DoubleVal AggregateFunctions::knuth_stddev_finalize(FunctionContext* ctx, } DecimalV2Val AggregateFunctions::decimalv2_knuth_stddev_finalize(FunctionContext* ctx, - const StringVal& state_sv) { - DCHECK(!state_sv.is_null); + const StringVal& state_sv) { DCHECK_EQ(state_sv.len, sizeof(DecimalV2KnuthVarianceState)); DecimalV2KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); @@ -1966,7 +2001,6 @@ DecimalV2Val AggregateFunctions::decimalv2_knuth_stddev_finalize(FunctionContext DoubleVal AggregateFunctions::knuth_stddev_pop_finalize(FunctionContext* ctx, const StringVal& state_sv) { - DCHECK(!state_sv.is_null); DCHECK_EQ(state_sv.len, sizeof(KnuthVarianceState)); KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); if (state->count == 0) return DoubleVal::null(); @@ -1976,8 +2010,7 @@ DoubleVal AggregateFunctions::knuth_stddev_pop_finalize(FunctionContext* ctx, } DecimalV2Val AggregateFunctions::decimalv2_knuth_stddev_pop_finalize(FunctionContext* ctx, - const StringVal& state_sv) { - DCHECK(!state_sv.is_null); + const StringVal& state_sv) { DCHECK_EQ(state_sv.len, sizeof(DecimalV2KnuthVarianceState)); DecimalV2KnuthVarianceState* state = reinterpret_cast(state_sv.ptr); @@ -2200,8 +2233,19 @@ void AggregateFunctions::offset_fn_update(FunctionContext* ctx, const IntVal& sr *dst = src; } +// Stamp out the templates for the types we need. +template void AggregateFunctions::init_zero_null(FunctionContext*, BigIntVal* dst); +template void AggregateFunctions::init_zero_null(FunctionContext*, LargeIntVal* dst); +template void AggregateFunctions::init_zero_null(FunctionContext*, DoubleVal* dst); +template void AggregateFunctions::init_zero_null(FunctionContext*, DecimalV2Val* dst); + // Stamp out the templates for the types we need. template void AggregateFunctions::init_zero(FunctionContext*, BigIntVal* dst); +template void AggregateFunctions::init_zero(FunctionContext*, LargeIntVal* dst); +template void AggregateFunctions::init_zero(FunctionContext*, DoubleVal* dst); +template void AggregateFunctions::init_zero(FunctionContext*, DecimalV2Val* dst); + +template void AggregateFunctions::init_zero_not_null(FunctionContext*, BigIntVal* dst); template void AggregateFunctions::sum_remove(FunctionContext*, const BooleanVal& src, @@ -2278,6 +2322,18 @@ template void AggregateFunctions::sum(FunctionContext*, con template void AggregateFunctions::sum(FunctionContext*, const DoubleVal& src, DoubleVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, BooleanVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, TinyIntVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, SmallIntVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, IntVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, BigIntVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, LargeIntVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, FloatVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, DoubleVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, DateTimeVal* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, DecimalV2Val* dst); +template void AggregateFunctions::min_init(doris_udf::FunctionContext *, StringVal* dst); + template void AggregateFunctions::min(FunctionContext*, const BooleanVal& src, BooleanVal* dst); template void AggregateFunctions::min(FunctionContext*, const TinyIntVal& src, @@ -2310,6 +2366,18 @@ template void AggregateFunctions::avg_remove(doris_udf:: doris_udf::SmallIntVal const&, doris_udf::StringVal*); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, BooleanVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, TinyIntVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, SmallIntVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, IntVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, BigIntVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, LargeIntVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, FloatVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, DoubleVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, DateTimeVal* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, DecimalV2Val* dst); +template void AggregateFunctions::max_init(doris_udf::FunctionContext *, StringVal* dst); + template void AggregateFunctions::max(FunctionContext*, const BooleanVal& src, BooleanVal* dst); template void AggregateFunctions::max(FunctionContext*, const TinyIntVal& src, diff --git a/be/src/exprs/aggregate_functions.h b/be/src/exprs/aggregate_functions.h index 39e9aa891b6bfa..638662b7171471 100644 --- a/be/src/exprs/aggregate_functions.h +++ b/be/src/exprs/aggregate_functions.h @@ -41,10 +41,19 @@ class AggregateFunctions { // Initializes dst to NULL and sets dst->ptr to NULL. static void init_null_string(doris_udf::FunctionContext* c, doris_udf::StringVal* dst); - // Initializes dst to 0. + // Initializes dst to 0 and is_null = true. template static void init_zero(doris_udf::FunctionContext*, T* dst); + // Initializes dst to 0 and is_null = true. + template + static void init_zero_null(doris_udf::FunctionContext*, T* dst); + + // Initializes dst to 0. + template + static void init_zero_not_null(doris_udf::FunctionContext*, T* dst); + + template static void sum_remove(doris_udf::FunctionContext* ctx, const SRC_VAL& src, DST_VAL* dst); @@ -127,10 +136,18 @@ class AggregateFunctions { template static void sum(doris_udf::FunctionContext*, const SRC_VAL& src, DST_VAL* dst); + // MinInit + template + static void min_init(doris_udf::FunctionContext*, T* dst); + // MinUpdate/MinMerge template static void min(doris_udf::FunctionContext*, const T& src, T* dst); + // MaxInit + template + static void max_init(doris_udf::FunctionContext*, T* dst); + // MaxUpdate/MaxMerge template static void max(doris_udf::FunctionContext*, const T& src, T* dst); diff --git a/be/src/exprs/anyval_util.h b/be/src/exprs/anyval_util.h index 477553939d84df..d465c12866b049 100644 --- a/be/src/exprs/anyval_util.h +++ b/be/src/exprs/anyval_util.h @@ -22,6 +22,7 @@ #include "exprs/expr.h" #include "runtime/collection_value.h" #include "runtime/primitive_type.h" +#include "runtime/type_limit.h" #include "udf/udf.h" #include "util/hash_util.hpp" #include "util/types.h" @@ -168,6 +169,45 @@ class AnyValUtil { return HashUtil::murmur_hash64A(&v.val, 8, seed); } + template + static Val min_val(FunctionContext* ctx) { + if constexpr (std::is_same_v) { + return StringVal(); + } else if constexpr (std::is_same_v) { + DateTimeVal val; + type_limit::min().to_datetime_val(&val); + return val; + } else if constexpr (std::is_same_v) { + DecimalV2Val val; + type_limit::min().to_decimal_val(&val); + return val; + } else { + return Val(type_limit().val)>::min()); + } + } + + template + static Val max_val(FunctionContext* ctx) { + if constexpr (std::is_same_v) { + StringValue sv = type_limit::max(); + StringVal max_val; + max_val.ptr = ctx->allocate(sv.len); + memcpy(max_val.ptr, sv.ptr, sv.len); + + return max_val; + } else if constexpr (std::is_same_v) { + DateTimeVal val; + type_limit::max().to_datetime_val(&val); + return val; + } else if constexpr (std::is_same_v) { + DecimalV2Val val; + type_limit::max().to_decimal_val(&val); + return val; + } else { + return Val(type_limit().val)>::max()); + } + } + // Returns the byte size of *Val for type t. static int any_val_size(const TypeDescriptor& t) { switch (t.type) { diff --git a/be/src/exprs/expr_value.h b/be/src/exprs/expr_value.h index bb98c4938f7387..f55a6d6501cbf3 100644 --- a/be/src/exprs/expr_value.h +++ b/be/src/exprs/expr_value.h @@ -175,11 +175,11 @@ struct ExprValue { return &large_int_val; case TYPE_FLOAT: - float_val = std::numeric_limits::min(); + float_val = std::numeric_limits::lowest(); return &float_val; case TYPE_DOUBLE: - double_val = std::numeric_limits::min(); + double_val = std::numeric_limits::lowest(); return &double_val; case TYPE_DECIMALV2: diff --git a/be/src/exprs/hll_function.cpp b/be/src/exprs/hll_function.cpp index fe33c11830770f..af8b3e162a17aa 100644 --- a/be/src/exprs/hll_function.cpp +++ b/be/src/exprs/hll_function.cpp @@ -29,6 +29,10 @@ using doris_udf::StringVal; void HllFunctions::init() {} StringVal HllFunctions::hll_hash(FunctionContext* ctx, const StringVal& input) { + return AnyValUtil::from_string_temp(ctx, hll_hash(input)); +} + +std::string HllFunctions::hll_hash(const StringVal& input) { HyperLogLog hll; if (!input.is_null) { uint64_t hash_value = HashUtil::murmur_hash64A(input.ptr, input.len, HashUtil::MURMUR_SEED); @@ -37,7 +41,8 @@ StringVal HllFunctions::hll_hash(FunctionContext* ctx, const StringVal& input) { std::string buf; buf.resize(hll.max_serialized_size()); buf.resize(hll.serialize((uint8_t*)buf.c_str())); - return AnyValUtil::from_string_temp(ctx, buf); + + return buf; } void HllFunctions::hll_init(FunctionContext*, StringVal* dst) { @@ -45,6 +50,7 @@ void HllFunctions::hll_init(FunctionContext*, StringVal* dst) { dst->len = sizeof(HyperLogLog); dst->ptr = (uint8_t*)new HyperLogLog(); } + StringVal HllFunctions::hll_empty(FunctionContext* ctx) { return AnyValUtil::from_string_temp(ctx, HyperLogLog::empty()); } diff --git a/be/src/exprs/hll_function.h b/be/src/exprs/hll_function.h index b1d8e2cf381151..a8f7ab0b65b95a 100644 --- a/be/src/exprs/hll_function.h +++ b/be/src/exprs/hll_function.h @@ -18,6 +18,7 @@ #ifndef DORIS_BE_SRC_QUERY_EXPRS_HLL_FUNCTION_H #define DORIS_BE_SRC_QUERY_EXPRS_HLL_FUNCTION_H +#include #include "udf/udf.h" namespace doris { @@ -26,6 +27,8 @@ class HllFunctions { public: static void init(); static StringVal hll_hash(FunctionContext* ctx, const StringVal& dest_base); + static std::string hll_hash(const StringVal& dest_base); + static StringVal hll_empty(FunctionContext* ctx); static void hll_init(FunctionContext*, StringVal* dst); diff --git a/be/src/exprs/hybrid_set.h b/be/src/exprs/hybrid_set.h index 495232f9afecea..47c02ead4668d7 100644 --- a/be/src/exprs/hybrid_set.h +++ b/be/src/exprs/hybrid_set.h @@ -187,4 +187,4 @@ class StringValueSet : public HybridSetBase { } // namespace doris -#endif // DORIS_BE_SRC_QUERY_EXPRS_HYBRID_SET_H \ No newline at end of file +#endif // DORIS_BE_SRC_QUERY_EXPRS_HYBRID_SET_H diff --git a/be/src/exprs/new_agg_fn_evaluator.cc b/be/src/exprs/new_agg_fn_evaluator.cc index 82867325791375..0d192d0d858c44 100644 --- a/be/src/exprs/new_agg_fn_evaluator.cc +++ b/be/src/exprs/new_agg_fn_evaluator.cc @@ -229,7 +229,7 @@ void NewAggFnEvaluator::Close(const vector& evals, RuntimeSt void NewAggFnEvaluator::SetDstSlot(const AnyVal* src, const SlotDescriptor& dst_slot_desc, Tuple* dst) { - if (src->is_null) { + if (src->is_null && dst_slot_desc.is_nullable()) { dst->set_null(dst_slot_desc.null_indicator_offset()); return; } @@ -531,14 +531,14 @@ void NewAggFnEvaluator::Merge(Tuple* src, Tuple* dst) { } void NewAggFnEvaluator::SerializeOrFinalize(Tuple* src, const SlotDescriptor& dst_slot_desc, - Tuple* dst, void* fn) { + Tuple* dst, void* fn, bool add_null) { // No fn was given and the src and dst are identical. Nothing to be done. if (fn == nullptr && src == dst) return; // src != dst means we are performing a Finalize(), so even if fn == null we // still must copy the value of the src slot into dst. const SlotDescriptor& slot_desc = intermediate_slot_desc(); - bool src_slot_null = src->is_null(slot_desc.null_indicator_offset()); + bool src_slot_null = add_null || src->is_null(slot_desc.null_indicator_offset()); void* src_slot = nullptr; if (!src_slot_null) src_slot = src->get_slot(slot_desc.tuple_offset()); diff --git a/be/src/exprs/new_agg_fn_evaluator.h b/be/src/exprs/new_agg_fn_evaluator.h index f1c13e49c1e3e0..a03626ecf1d459 100644 --- a/be/src/exprs/new_agg_fn_evaluator.h +++ b/be/src/exprs/new_agg_fn_evaluator.h @@ -138,7 +138,7 @@ class NewAggFnEvaluator { /// Does one final transformation of the aggregated value in 'agg_val' and stores the /// result in 'output_val'. Also frees the resources allocated during init, update and /// merge phases. - void Finalize(Tuple* agg_val, Tuple* output_val); + void Finalize(Tuple* agg_val, Tuple* output_val, bool add_null = false); /// Puts the finalized value from Tuple* src in Tuple* dst just as Finalize() does. /// However, unlike Finalize(), GetValue() does not clean up state in src. @@ -180,7 +180,7 @@ class NewAggFnEvaluator { Tuple* dst); static void Serialize(const std::vector& evals, Tuple* dst); static void GetValue(const std::vector& evals, Tuple* src, Tuple* dst); - static void Finalize(const std::vector& evals, Tuple* src, Tuple* dst); + static void Finalize(const std::vector& evals, Tuple* src, Tuple* dst, bool add_null = false); /// Free local allocations made in UDA functions and input arguments' evals. //void FreeLocalAllocations(); @@ -250,6 +250,10 @@ class NewAggFnEvaluator { /// generated by AggFn::CodegenUpdateOrMergeFunction() when codegen is enabled. void Update(const TupleRow* row, Tuple* dst, void* fn); + /// Writes the result in src into dst pointed to by dst_slot_desc + inline void SetDstSlot(const doris_udf::AnyVal* src, const SlotDescriptor& dst_slot_desc, + Tuple* dst); + /// Sets up the arguments to call 'fn'. This converts from the agg-expr signature, /// taking TupleRow to the UDA signature taking AnyVals. Writes the serialize/finalize /// result to the given destination slot/tuple. 'fn' can be NULL to indicate the src @@ -257,11 +261,7 @@ class NewAggFnEvaluator { /// from local allocation (which will be freed in the next QueryMaintenance()) so it /// needs to be copied out if it needs to survive beyond QueryMaintenance() (e.g. if /// 'dst' lives in a row batch). - void SerializeOrFinalize(Tuple* src, const SlotDescriptor& dst_slot_desc, Tuple* dst, void* fn); - - /// Writes the result in src into dst pointed to by dst_slot_desc - inline void SetDstSlot(const doris_udf::AnyVal* src, const SlotDescriptor& dst_slot_desc, - Tuple* dst); + void SerializeOrFinalize(Tuple* src, const SlotDescriptor& dst_slot_desc, Tuple* dst, void* fn, bool add_null = false); // Sets 'dst' to the value from 'slot'. void set_any_val(const void* slot, const TypeDescriptor& type, doris_udf::AnyVal* dst); @@ -281,8 +281,8 @@ inline void NewAggFnEvaluator::Serialize(Tuple* tuple) { SerializeOrFinalize(tuple, agg_fn_.intermediate_slot_desc(), tuple, agg_fn_.serialize_fn()); } -inline void NewAggFnEvaluator::Finalize(Tuple* agg_val, Tuple* output_val) { - SerializeOrFinalize(agg_val, agg_fn_.output_slot_desc(), output_val, agg_fn_.finalize_fn()); +inline void NewAggFnEvaluator::Finalize(Tuple* agg_val, Tuple* output_val, bool add_null) { + SerializeOrFinalize(agg_val, agg_fn_.output_slot_desc(), output_val, agg_fn_.finalize_fn(), add_null); } inline void NewAggFnEvaluator::GetValue(Tuple* src, Tuple* dst) { @@ -313,9 +313,9 @@ inline void NewAggFnEvaluator::GetValue(const std::vector& e } inline void NewAggFnEvaluator::Finalize(const std::vector& evals, - Tuple* agg_val, Tuple* output_val) { + Tuple* agg_val, Tuple* output_val, bool add_null) { for (int i = 0; i < evals.size(); ++i) { - evals[i]->Finalize(agg_val, output_val); + evals[i]->Finalize(agg_val, output_val, add_null); } } diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index a951bb8408e7e1..610187a43b8083 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -62,6 +62,29 @@ bool TimestampFunctions::check_format(const StringVal& format, DateTimeValue& t) return false; } +std::string TimestampFunctions::convert_format(const std::string& format) { + switch (format.size()) { + case 8: + if (strncmp(format.c_str(), "yyyyMMdd", 8) == 0) { + return std::string("%Y%m%d"); + } + break; + case 10: + if (strncmp(format.c_str(), "yyyy-MM-dd", 10) == 0) { + return std::string("%Y-%m-%d"); + } + break; + case 19: + if (strncmp(format.c_str(), "yyyy-MM-dd HH:mm:ss", 19) == 0) { + return std::string("%Y-%m-%d %H:%i:%s"); + } + break; + default: + break; + } + return format; +} + StringVal TimestampFunctions::convert_format(FunctionContext* ctx, const StringVal& format) { switch (format.len) { case 8: @@ -438,61 +461,7 @@ BigIntVal TimestampFunctions::timestamp_diff(FunctionContext* ctx, const DateTim DateTimeValue ts_value1 = DateTimeValue::from_datetime_val(ts_val1); DateTimeValue ts_value2 = DateTimeValue::from_datetime_val(ts_val2); - switch (unit) { - case YEAR: { - int year = (ts_value2.year() - ts_value1.year()); - if (year > 0) { - year -= (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) < 0; - } else if (year < 0) { - year += (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) > 0; - } - return year; - } - case MONTH: { - int month = (ts_value2.year() - ts_value1.year()) * 12 + - (ts_value2.month() - ts_value1.month()); - if (month > 0) { - month -= (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) < 0; - } else if (month < 0) { - month += (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) > 0; - } - return month; - } - case WEEK: { - int day = ts_value2.daynr() - ts_value1.daynr(); - if (day > 0) { - day -= ts_value2.time_part_diff(ts_value1) < 0; - } else if (day < 0) { - day += ts_value2.time_part_diff(ts_value1) > 0; - } - return day / 7; - } - case DAY: { - int day = ts_value2.daynr() - ts_value1.daynr(); - if (day > 0) { - day -= ts_value2.time_part_diff(ts_value1) < 0; - } else if (day < 0) { - day += ts_value2.time_part_diff(ts_value1) > 0; - } - return day; - } - case HOUR: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t hour = second / 60 / 60; - return hour; - } - case MINUTE: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t minute = second / 60; - return minute; - } - case SECOND: { - int64_t second = ts_value2.second_diff(ts_value1); - return second; - } - default: - return BigIntVal::null(); - } + return DateTimeValue::datetime_diff(ts_value1, ts_value2); } void TimestampFunctions::format_prepare(doris_udf::FunctionContext* context, @@ -575,10 +544,10 @@ DateTimeVal from_olap_datetime(uint64_t datetime) { static const DateTimeVal FIRST_DAY = from_olap_datetime(19700101000000); static const DateTimeVal FIRST_SUNDAY = from_olap_datetime(19700104000000); -#define TIME_ROUND(UNIT, unit, ORIGIN) \ - _TR_4(FLOOR, floor, UNIT, unit) \ - _TR_4(CEIL, ceil, UNIT, unit) _TR_5(FLOOR, floor, UNIT, unit, ORIGIN) \ - _TR_5(CEIL, ceil, UNIT, unit, ORIGIN) +#define TIME_ROUND(UNIT, unit, ORIGIN) \ + _TR_4(FLOOR, floor, UNIT, unit) \ + _TR_4(CEIL, ceil, UNIT, unit) \ + _TR_5(FLOOR, floor, UNIT, unit, ORIGIN) _TR_5(CEIL, ceil, UNIT, unit, ORIGIN) TIME_ROUND(YEAR, year, FIRST_DAY) TIME_ROUND(MONTH, month, FIRST_DAY) diff --git a/be/src/exprs/timestamp_functions.h b/be/src/exprs/timestamp_functions.h index 17f38f62eb42aa..155837c606e102 100644 --- a/be/src/exprs/timestamp_functions.h +++ b/be/src/exprs/timestamp_functions.h @@ -420,6 +420,8 @@ class TimestampFunctions { // Todo(kks): remove this method when 0.12 release static StringVal convert_format(doris_udf::FunctionContext* ctx, const StringVal& format); + static std::string convert_format(const std::string& format); + // Issue a warning for a bad format string. static void report_bad_format(const StringVal* format); diff --git a/be/src/olap/block_column_predicate.h b/be/src/olap/block_column_predicate.h index d20408b74bca36..4c0648699ac747 100644 --- a/be/src/olap/block_column_predicate.h +++ b/be/src/olap/block_column_predicate.h @@ -25,7 +25,7 @@ namespace doris { // Block Column Predicate support do column predicate in RowBlockV2 and support OR and AND predicate -// Block Column Predicate will replace column predicate as a unified external vectorization interface +// Block Column Predicate will replace column predicate as a unified external vectorized interface // in the future // TODO: support do predicate on Bitmap and ZoneMap, So we can use index of column to do predicate on // page and segment diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index e2d4b1d50da981..f39eaa04f16bd2 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -526,7 +526,7 @@ Status SegmentIterator::next_batch(RowBlockV2* block) { _opts.stats->raw_rows_read += nrows_read; _opts.stats->blocks_load += 1; - // phase 2: run vectorization evaluation on remaining predicates to prune rows. + // phase 2: run vectorized evaluation on remaining predicates to prune rows. // block's selection vector will be set to indicate which rows have passed predicates. // TODO(hkp): optimize column predicate to check column block once for one column if (!_col_predicates.empty() || _opts.delete_condition_predicates.get() != nullptr) { diff --git a/be/src/runtime/data_stream_recvr.cc b/be/src/runtime/data_stream_recvr.cc index cba524e152ca54..b836253578a876 100644 --- a/be/src/runtime/data_stream_recvr.cc +++ b/be/src/runtime/data_stream_recvr.cc @@ -461,8 +461,6 @@ DataStreamRecvr::DataStreamRecvr( // Initialize the counters _bytes_received_counter = ADD_COUNTER(_profile, "BytesReceived", TUnit::BYTES); - // _bytes_received_time_series_counter = - // ADD_TIME_SERIES_COUNTER(_profile, "BytesReceived", _bytes_received_counter); _deserialize_row_batch_timer = ADD_TIMER(_profile, "DeserializeRowBatchTimer"); _data_arrival_timer = ADD_TIMER(_profile, "DataArrivalWaitTime"); _buffer_full_total_timer = ADD_TIMER(_profile, "SendersBlockedTotalTimer(*)"); diff --git a/be/src/runtime/data_stream_sender.cpp b/be/src/runtime/data_stream_sender.cpp index 90124b094fc3be..11ab6763d4e34c 100644 --- a/be/src/runtime/data_stream_sender.cpp +++ b/be/src/runtime/data_stream_sender.cpp @@ -417,6 +417,7 @@ DataStreamSender::DataStreamSender(ObjectPool* pool, int sender_id, const RowDes // We use the ParttitionRange to compare here. It should not be a member function of PartitionInfo // class becaurce there are some other member in it. +// TODO: move this to dpp_sink static bool compare_part_use_range(const PartitionInfo* v1, const PartitionInfo* v2) { return v1->range() < v2->range(); } @@ -469,8 +470,9 @@ Status DataStreamSender::prepare(RuntimeState* state) { << "])"; _profile = _pool->add(new RuntimeProfile(title.str())); SCOPED_TIMER(_profile->total_time_counter()); - _mem_tracker = MemTracker::CreateTracker(_profile, -1, "DataStreamSender", - state->instance_mem_tracker()); + _mem_tracker = MemTracker::CreateTracker( + _profile, -1, "DataStreamSender:" + print_id(state->fragment_instance_id()), + state->instance_mem_tracker()); if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM) { // Randomize the order we open/transmit to channels to avoid thundering herd problems. diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp index 49b84b32c782d8..183c60492432bf 100644 --- a/be/src/runtime/datetime_value.cpp +++ b/be/src/runtime/datetime_value.cpp @@ -28,19 +28,15 @@ #include "util/timezone_utils.h" namespace doris { - const uint64_t log_10_int[] = {1, 10, 100, 1000, 10000UL, 100000UL, 1000000UL, 10000000UL, 100000000UL, 1000000000UL, 10000000000UL, 100000000000UL}; static int s_days_in_month[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; -static const char* s_month_name[] = {"", "January", "February", "March", "April", - "May", "June", "July", "August", "September", - "October", "November", "December", NULL}; + static const char* s_ab_month_name[] = {"", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", NULL}; -static const char* s_day_name[] = {"Monday", "Tuesday", "Wednesday", "Thursday", - "Friday", "Saturday", "Sunday", NULL}; + static const char* s_ab_day_name[] = {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", NULL}; uint8_t mysql_week_mode(uint32_t mode) { @@ -1150,7 +1146,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con date_part_used = true; break; case 'M': - int_value = check_word(s_month_name, val, val_end, &val); + int_value = check_word(const_cast(s_month_name), val, val_end, &val); if (int_value < 0) { return false; } @@ -1245,7 +1241,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con break; // Weekday case 'W': - int_value = check_word(s_day_name, val, val_end, &val); + int_value = check_word(const_cast(s_day_name), val, val_end, &val); if (int_value < 0) { return false; } diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index 033828d5ef5d3a..8f5e7fddef4232 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -137,6 +137,28 @@ const int TIME_MAX_SECOND = 59; const int TIME_MAX_VALUE = 10000 * TIME_MAX_HOUR + 100 * TIME_MAX_MINUTE + TIME_MAX_SECOND; const int TIME_MAX_VALUE_SECONDS = 3600 * TIME_MAX_HOUR + 60 * TIME_MAX_MINUTE + TIME_MAX_SECOND; +constexpr size_t const_length(const char* str) { + return (str == nullptr || *str == 0) ? 0 : const_length(str + 1) + 1; +} + +constexpr size_t max_char_length(const char* const* name, size_t end) { + size_t res = 0; + for (int i = 0; i < end; ++i) { + res = std::max(const_length(name[i]), res); + } + return res; +} + +static constexpr const char* s_month_name[] = { + "", "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", NULL}; + +static constexpr const char* s_day_name[] = {"Monday", "Tuesday", "Wednesday", "Thursday", + "Friday", "Saturday", "Sunday", NULL}; + +static constexpr size_t MAX_DAY_NAME_LEN = max_char_length(s_day_name, std::size(s_day_name)); +static constexpr size_t MAX_MONTH_NAME_LEN = max_char_length(s_month_name, std::size(s_month_name)); + uint8_t mysql_week_mode(uint32_t mode); class DateTimeValue { @@ -145,15 +167,15 @@ class DateTimeValue { DateTimeValue() : _neg(0), _type(TIME_DATETIME), - _hour(0), - _minute(0), _second(0), - _year(0), - _month(0), + _minute(0), + _hour(0), _day(0), + _month(0), + _year(0), _microsecond(0) {} - DateTimeValue(int64_t t) { from_date_int64(t); } + explicit DateTimeValue(int64_t t) { from_date_int64(t); } void set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour, uint32_t minute, uint32_t second, uint32_t microsecond); @@ -255,6 +277,68 @@ class DateTimeValue { static bool check_date(uint32_t year, uint32_t month, uint32_t day); + // compute the diff between two datetime value + template + static int64_t datetime_diff(const DateTimeValue& ts_value1, const DateTimeValue& ts_value2) { + switch (unit) { + case YEAR: { + int year = (ts_value2.year() - ts_value1.year()); + if (year > 0) { + year -= (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) < + 0; + } else if (year < 0) { + year += (ts_value2.to_int64() % 10000000000 - ts_value1.to_int64() % 10000000000) > + 0; + } + return year; + } + case MONTH: { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + if (month > 0) { + month -= (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) < 0; + } else if (month < 0) { + month += (ts_value2.to_int64() % 100000000 - ts_value1.to_int64() % 100000000) > 0; + } + return month; + } + case WEEK: { + int day = ts_value2.daynr() - ts_value1.daynr(); + if (day > 0) { + day -= ts_value2.time_part_diff(ts_value1) < 0; + } else if (day < 0) { + day += ts_value2.time_part_diff(ts_value1) > 0; + } + return day / 7; + } + case DAY: { + int day = ts_value2.daynr() - ts_value1.daynr(); + if (day > 0) { + day -= ts_value2.time_part_diff(ts_value1) < 0; + } else if (day < 0) { + day += ts_value2.time_part_diff(ts_value1) > 0; + } + return day; + } + case HOUR: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t hour = second / 60 / 60; + return hour; + } + case MINUTE: { + int64_t second = ts_value2.second_diff(ts_value1); + int64_t minute = second / 60; + return minute; + } + case SECOND: { + int64_t second = ts_value2.second_diff(ts_value1); + return second; + } + } + // Rethink the default return value + return 0; + } + // Convert this value to uint64_t // Will check its type int64_t to_int64() const; @@ -278,11 +362,13 @@ class DateTimeValue { int year() const { return _year; } int month() const { return _month; } + int quarter() const { return (_month - 1) / 3 + 1; } int day() const { return _day; } int hour() const { return _hour; } int minute() const { return _minute; } int second() const { return _second; } int microsecond() const { return _microsecond; } + int neg() const { return _neg; } bool check_loss_accuracy_cast_to_date() { auto loss_accuracy = _hour != 0 || _minute != 0 || _second != 0 || _microsecond != 0; @@ -309,6 +395,7 @@ class DateTimeValue { // Weekday, from 0(Mon) to 6(Sun) inline uint8_t weekday() const { return calc_weekday(daynr(), false); } + inline auto day_of_week() const { return (weekday() + 1) % 7 + 1; } // The bits in week_format has the following meaning: // WEEK_MONDAY_FIRST (0) @@ -536,15 +623,15 @@ class DateTimeValue { bool from_date_format_str(const char* format, int format_len, const char* value, int value_len, const char** sub_val_end); - // 1 bits for neg. 3 bits for type. 12bit for hour + // 1 bits for neg. 3 bits for type. 12bit for second uint16_t _neg : 1; // Used for time value. uint16_t _type : 3; // Which type of this value. - uint16_t _hour : 12; + uint16_t _second : 12; uint8_t _minute; - uint8_t _second; - uint16_t _year; - uint8_t _month; + uint8_t _hour; uint8_t _day; + uint8_t _month; + uint16_t _year; // TODO(zc): used for nothing uint64_t _microsecond; @@ -552,12 +639,12 @@ class DateTimeValue { uint32_t microsecond, uint16_t year, uint8_t month, uint8_t day) : _neg(neg), _type(type), - _hour(hour), - _minute(minute), _second(second), - _year(year), - _month(month), + _minute(minute), + _hour(hour), _day(day), + _month(month), + _year(year), _microsecond(microsecond) {} // RE2 obj is thread safe diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index f6f7b2add959a5..9b3a28ecc84c0e 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -280,10 +280,12 @@ RowDescriptor::RowDescriptor(const DescriptorTbl& desc_tbl, const std::vectornum_materialized_slots(); _num_null_slots += tupleDesc->num_null_slots(); _tuple_desc_map.push_back(tupleDesc); DCHECK(_tuple_desc_map.back() != NULL); @@ -459,6 +461,20 @@ std::string RowDescriptor::debug_string() const { return ss.str(); } + +int RowDescriptor::get_column_id(int slot_id) const { + int column_id_counter = 0; + for(const auto tuple_desc:_tuple_desc_map) { + for(const auto slot:tuple_desc->slots()) { + if(slot->id() == slot_id) { + return column_id_counter; + } + column_id_counter++; + } + } + return -1; +} + Status DescriptorTbl::create(ObjectPool* pool, const TDescriptorTable& thrift_tbl, DescriptorTbl** tbl) { *tbl = pool->add(new DescriptorTbl()); diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index a98451b7aca4b9..e201d8481d8c01 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -32,6 +32,10 @@ #include "gen_cpp/Types_types.h" #include "runtime/types.h" +namespace doris::vectorized { +class ColumnWithTypeAndName; +} + namespace doris { class ObjectPool; @@ -241,6 +245,7 @@ class TupleDescriptor { public: // virtual ~TupleDescriptor() {} int byte_size() const { return _byte_size; } + int num_materialized_slots() const { return _num_materialized_slots; } int num_null_slots() const { return _num_null_slots; } int num_null_bytes() const { return _num_null_bytes; } const std::vector& slots() const { return _slots; } @@ -351,9 +356,11 @@ class RowDescriptor { _tuple_idx_nullable_map(desc._tuple_idx_nullable_map), _tuple_idx_map(desc._tuple_idx_map), _has_varlen_slots(desc._has_varlen_slots) { + _num_materialized_slots = 0; _num_null_slots = 0; std::vector::const_iterator it = desc._tuple_desc_map.begin(); for (; it != desc._tuple_desc_map.end(); ++it) { + _num_materialized_slots += (*it)->num_materialized_slots(); _num_null_slots += (*it)->num_null_slots(); } _num_null_bytes = (_num_null_slots + 7) / 8; @@ -371,6 +378,11 @@ class RowDescriptor { // to GetAvgRowSize() int get_row_size() const; + int num_materialized_slots() const { + DCHECK(_num_materialized_slots != 0); + return _num_materialized_slots; + } + int num_null_slots() const { return _num_null_slots; } int num_null_bytes() const { return _num_null_bytes; } @@ -413,6 +425,8 @@ class RowDescriptor { std::string debug_string() const; + int get_column_id(int slot_id) const; + private: // Initializes tupleIdxMap during c'tor using the _tuple_desc_map. void init_tuple_idx_map(); @@ -432,6 +446,7 @@ class RowDescriptor { // Provide quick way to check if there are variable length slots. bool _has_varlen_slots; + int _num_materialized_slots; int _num_null_slots; int _num_null_bytes; }; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 91e8e4c3444d1e..efb073befba3c1 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -22,7 +22,9 @@ #include "olap/options.h" namespace doris { - +namespace vectorized { +class VDataStreamMgr; +} class BfdParser; class BrokerMgr; class BrpcStubCache; diff --git a/be/src/runtime/mysql_result_writer.cpp b/be/src/runtime/mysql_result_writer.cpp index bed88be04bcd46..444e0dbfa7bce0 100644 --- a/be/src/runtime/mysql_result_writer.cpp +++ b/be/src/runtime/mysql_result_writer.cpp @@ -32,9 +32,9 @@ namespace doris { MysqlResultWriter::MysqlResultWriter(BufferControlBlock* sinker, - const std::vector& output_expr_ctxs, - RuntimeProfile* parent_profile) - : _sinker(sinker), + const std::vector& output_expr_ctxs, RuntimeProfile* parent_profile) + : ResultWriter(), + _sinker(sinker), _output_expr_ctxs(output_expr_ctxs), _row_buffer(NULL), _parent_profile(parent_profile) {} @@ -49,8 +49,7 @@ Status MysqlResultWriter::init(RuntimeState* state) { return Status::InternalError("sinker is NULL pointer."); } - _row_buffer = new (std::nothrow) MysqlRowBuffer(); - + _row_buffer = new(std::nothrow) MysqlRowBuffer(); if (NULL == _row_buffer) { return Status::InternalError("no memory to alloc."); } diff --git a/be/src/runtime/mysql_result_writer.h b/be/src/runtime/mysql_result_writer.h index 8d8c21531027e3..681148828d3641 100644 --- a/be/src/runtime/mysql_result_writer.h +++ b/be/src/runtime/mysql_result_writer.h @@ -19,6 +19,7 @@ #include "runtime/result_writer.h" #include "runtime/runtime_state.h" +#include "primitive_type.h" namespace doris { @@ -29,11 +30,16 @@ class MysqlRowBuffer; class BufferControlBlock; class RuntimeProfile; +namespace vectorized { + class VExprContext; +} + // convert the row batch to mysql protocol row class MysqlResultWriter final : public ResultWriter { public: MysqlResultWriter(BufferControlBlock* sinker, const std::vector& output_expr_ctxs, - RuntimeProfile* parent_profile); + RuntimeProfile* parent_profile); + virtual ~MysqlResultWriter(); virtual Status init(RuntimeState* state) override; @@ -52,7 +58,11 @@ class MysqlResultWriter final : public ResultWriter { private: BufferControlBlock* _sinker; const std::vector& _output_expr_ctxs; + + std::vector _result_column_ids; + MysqlRowBuffer* _row_buffer; + std::vector _vec_buffers; RuntimeProfile* _parent_profile; // parent profile from result sink. not owned // total time cost on append batch operation diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 2df18a5d1f37a9..0040a4f11a347b 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -162,6 +162,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request, _plan); } + // set #senders of exchange nodes before calling Prepare() std::vector exch_nodes; _plan->collect_nodes(TPlanNodeType::EXCHANGE_NODE, &exch_nodes); @@ -169,7 +170,10 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request, DCHECK_EQ(exch_node->type(), TPlanNodeType::EXCHANGE_NODE); int num_senders = find_with_default(params.per_exch_num_senders, exch_node->id(), 0); DCHECK_GT(num_senders, 0); - static_cast(exch_node)->set_num_senders(num_senders); + if (_runtime_state->enable_vectorized_exec()) { + } else { + static_cast(exch_node)->set_num_senders(num_senders); + } } RETURN_IF_ERROR(_plan->prepare(_runtime_state.get())); @@ -197,7 +201,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request, if (request.fragment.__isset.output_sink) { RETURN_IF_ERROR(DataSink::create_data_sink(obj_pool(), request.fragment.output_sink, request.fragment.output_exprs, params, - row_desc(), &_sink)); + row_desc(), runtime_state()->enable_vectorized_exec(), &_sink)); RETURN_IF_ERROR(_sink->prepare(runtime_state())); RuntimeProfile* sink_profile = _sink->profile(); @@ -249,8 +253,11 @@ Status PlanFragmentExecutor::open() { _report_thread_started_cv.wait(l); _report_thread_active = true; } - - Status status = open_internal(); + Status status = Status::OK(); + if (_runtime_state->enable_vectorized_exec()) { + } else { + status = open_internal(); + } if (!status.ok() && !status.is_cancelled() && _runtime_state->log_has_space()) { // Log error message in addition to returning in Status. Queries that do not diff --git a/be/src/runtime/plan_fragment_executor.h b/be/src/runtime/plan_fragment_executor.h index 4fcdf9d1053fc6..96e6170e3f6e38 100644 --- a/be/src/runtime/plan_fragment_executor.h +++ b/be/src/runtime/plan_fragment_executor.h @@ -222,6 +222,8 @@ class PlanFragmentExecutor { std::shared_ptr _query_statistics; bool _collect_query_statistics_with_every_batch; + bool _enable_vectorized_engine; + ObjectPool* obj_pool() { return _runtime_state->obj_pool(); } // typedef for TPlanFragmentExecParams.per_node_scan_ranges diff --git a/be/src/runtime/result_sink.cpp b/be/src/runtime/result_sink.cpp index 931afcbca2a645..e99f5c05d3fad9 100644 --- a/be/src/runtime/result_sink.cpp +++ b/be/src/runtime/result_sink.cpp @@ -122,6 +122,7 @@ Status ResultSink::close(RuntimeState* state, Status exec_status) { state->exec_env()->result_mgr()->cancel_at_time( time(NULL) + config::result_buffer_cancelled_interval_time, state->fragment_instance_id()); + Expr::close(_output_expr_ctxs, state); _closed = true; diff --git a/be/src/runtime/result_sink.h b/be/src/runtime/result_sink.h index 39f34a8acc57f7..a7c1da4c51abc1 100644 --- a/be/src/runtime/result_sink.h +++ b/be/src/runtime/result_sink.h @@ -36,6 +36,10 @@ class ResultWriter; class MemTracker; class ResultFileOptions; +namespace vectorized { + class VExprContext; +} + class ResultSink : public DataSink { public: // construct a buffer for the result need send to coordinator. @@ -74,6 +78,8 @@ class ResultSink : public DataSink { boost::shared_ptr _writer; RuntimeProfile* _profile; // Allocated from _pool int _buf_size; // Allocated from _pool + + bool _is_vec; }; } // namespace doris diff --git a/be/src/runtime/result_writer.h b/be/src/runtime/result_writer.h index 477d79c7e11c02..3c914d06b2c5c0 100644 --- a/be/src/runtime/result_writer.h +++ b/be/src/runtime/result_writer.h @@ -27,16 +27,24 @@ class RowBatch; class RuntimeState; class TypeDescriptor; +namespace vectorized { + class Block; +} + // abstract class of the result writer class ResultWriter { public: - ResultWriter(){}; + ResultWriter() {}; ~ResultWriter(){}; virtual Status init(RuntimeState* state) = 0; // convert and write one row batch virtual Status append_row_batch(const RowBatch* batch) = 0; + // virtual Status append_block(const vectorized::Block& block) { + // return Status::InternalError("Not support append vec block now."); + // } + virtual Status close() = 0; virtual int64_t get_written_rows() const { return _written_rows; } diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp index 54031c4664121e..bf2ee39a361b07 100644 --- a/be/src/runtime/row_batch.cpp +++ b/be/src/runtime/row_batch.cpp @@ -29,6 +29,9 @@ #include "gen_cpp/data.pb.h" #include "runtime/collection_value.h" +//#include "vec/columns/column_vector.h" +//#include "vec/core/block.h" + using std::vector; namespace doris { diff --git a/be/src/runtime/row_batch.h b/be/src/runtime/row_batch.h index 9349eefe51bb27..a3e20f3a973f3e 100644 --- a/be/src/runtime/row_batch.h +++ b/be/src/runtime/row_batch.h @@ -31,6 +31,10 @@ #include "runtime/mem_pool.h" #include "runtime/row_batch_interface.hpp" +namespace doris::vectorized { +class Block; +} + namespace doris { class BufferedTupleStream2; @@ -40,6 +44,7 @@ class TupleRow; class TupleDescriptor; class PRowBatch; + // A RowBatch encapsulates a batch of rows, each composed of a number of tuples. // The maximum number of rows is fixed at the time of construction, and the caller // can add rows up to that capacity. diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 2152dfd3677ace..659741e640af2e 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -346,6 +346,8 @@ class RuntimeState { int32_t runtime_filter_max_in_num() { return _query_options.runtime_filter_max_in_num; } + bool enable_vectorized_exec() const { return _query_options.enable_vectorized_engine; } + bool enable_exchange_node_parallel_merge() const { return _query_options.enable_enable_exchange_node_parallel_merge; } diff --git a/be/src/runtime/type_limit.h b/be/src/runtime/type_limit.h index d46a3bc3200d94..45096b5b4cccde 100644 --- a/be/src/runtime/type_limit.h +++ b/be/src/runtime/type_limit.h @@ -26,8 +26,12 @@ namespace doris { template struct type_limit { - static T min() { return std::numeric_limits::min(); } - static T max() { return std::numeric_limits::max(); } + static T min() { + return std::numeric_limits::lowest(); + } + static T max() { + return std::numeric_limits::max(); + } }; template <> diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h index 30ec41d4fe5226..a2b18c2b1f9aa5 100644 --- a/be/src/runtime/types.h +++ b/be/src/runtime/types.h @@ -72,6 +72,10 @@ struct TypeDescriptor { DCHECK_NE(type, TYPE_ARRAY); DCHECK_NE(type, TYPE_MAP); #endif + if (type == TYPE_DECIMALV2) { + precision = 27; + scale = 9; + } } static TypeDescriptor create_char_type(int len) { @@ -159,6 +163,8 @@ struct TypeDescriptor { inline bool is_decimal_type() const { return (type == TYPE_DECIMALV2); } + inline bool is_datetime_type() const { return type == TYPE_DATETIME; } + inline bool is_var_len_string_type() const { return type == TYPE_VARCHAR || type == TYPE_HLL || type == TYPE_CHAR || type == TYPE_OBJECT; } diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 7afb8c83926ecb..f4bdb496184fe1 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -389,6 +389,18 @@ Status PInternalServiceImpl::_fold_constant_expr(const std::string& ser_reque return mgr.fold_constant_expr(t_request, response); } +template +void PInternalServiceImpl::transmit_block(google::protobuf::RpcController* cntl_base, + const PTransmitDataParams* request, + PTransmitDataResult* response, + google::protobuf::Closure* done) { + VLOG_ROW << "transmit data: fragment_instance_id=" << print_id(request->finst_id()) + << " node=" << request->node_id(); + if (done != nullptr) { + done->Run(); + } +} + template class PInternalServiceImpl; } // namespace doris diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h index 16722f770dfb67..5bbebce23a87ec 100644 --- a/be/src/service/internal_service.h +++ b/be/src/service/internal_service.h @@ -91,6 +91,10 @@ class PInternalServiceImpl : public T { const ::doris::PPublishFilterRequest* request, ::doris::PPublishFilterResponse* response, ::google::protobuf::Closure* done) override; + void transmit_block(::google::protobuf::RpcController* controller, + const ::doris::PTransmitDataParams* request, + ::doris::PTransmitDataResult* response, + ::google::protobuf::Closure* done) override; void send_data(google::protobuf::RpcController* controller, const PSendDataRequest* request, PSendDataResult* response, google::protobuf::Closure* done); diff --git a/be/src/util/binary_cast.hpp b/be/src/util/binary_cast.hpp index 82395805705391..d528d71594882d 100644 --- a/be/src/util/binary_cast.hpp +++ b/be/src/util/binary_cast.hpp @@ -20,6 +20,7 @@ #include #include +#include "runtime/datetime_value.h" #include "runtime/decimalv2_value.h" #include "util/types.h" @@ -40,9 +41,20 @@ inline constexpr bool match_v = std::is_same_v&& std::is_same_v; union DecimalInt128Union { DecimalV2Value decimal; PackedInt128 packed128; + __int128_t i128; }; static_assert(sizeof(DecimalV2Value) == sizeof(PackedInt128)); +static_assert(sizeof(DecimalV2Value) == sizeof(__int128_t)); + +// we need provide a destructor becase DateTimeValue was not a pod type +// DateTimeValue won't alloc any extra memory, so we don't have to call +// DateTimeValue::~DateTimeValue() +union DateTimeInt128Union { + DateTimeValue dt; + __int128_t i128; + ~DateTimeInt128Union() {} +}; // similar to reinterpret_cast but won't break strict-aliasing rules template @@ -52,9 +64,14 @@ To binary_cast(From from) { constexpr bool from_db_to_i64 = match_v; constexpr bool from_db_to_u64 = match_v; constexpr bool from_decv2_to_packed128 = match_v; - + constexpr bool from_i128_to_dt = match_v; + constexpr bool from_dt_to_i128 = match_v; + constexpr bool from_i128_to_decv2 = match_v; + constexpr bool from_decv2_to_i128 = match_v; + static_assert(from_u64_to_db || from_i64_to_db || from_db_to_i64 || from_db_to_u64 || - from_decv2_to_packed128); + from_decv2_to_packed128 || from_i128_to_dt || from_dt_to_i128 || + from_i128_to_decv2 || from_decv2_to_i128); if constexpr (from_u64_to_db) { TypeConverter conv; @@ -76,6 +93,20 @@ To binary_cast(From from) { DecimalInt128Union conv; conv.decimal = from; return conv.packed128; + } else if constexpr (from_i128_to_dt) { + DateTimeInt128Union conv = {.i128 = from}; + return conv.dt; + } else if constexpr (from_dt_to_i128) { + DateTimeInt128Union conv = {.dt = from}; + return conv.i128; + } else if constexpr (from_i128_to_decv2) { + DecimalInt128Union conv; + conv.i128 = from; + return conv.decimal; + } else if constexpr (from_decv2_to_i128) { + DecimalInt128Union conv; + conv.decimal = from; + return conv.i128; } else { __builtin_unreachable(); } diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 2b3dc4713d7b1a..52f39c5bacf17e 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1189,7 +1189,7 @@ class BitmapValue { // check if value x is present - bool contains(uint64_t x) { + bool contains(uint64_t x) const { switch (_type) { case EMPTY: return false; diff --git a/be/src/util/brpc_stub_cache.h b/be/src/util/brpc_stub_cache.h index 3b9407f580467b..940883df5c81cc 100644 --- a/be/src/util/brpc_stub_cache.h +++ b/be/src/util/brpc_stub_cache.h @@ -32,9 +32,9 @@ namespace doris { class BrpcStubCache { public: BrpcStubCache(); - ~BrpcStubCache(); + virtual ~BrpcStubCache(); - PBackendService_Stub* get_stub(const butil::EndPoint& endpoint) { + virtual PBackendService_Stub* get_stub(const butil::EndPoint& endpoint) { std::lock_guard l(_lock); auto stub_ptr = _stub_map.seek(endpoint); if (stub_ptr != nullptr) { @@ -52,7 +52,7 @@ class BrpcStubCache { return stub; } - PBackendService_Stub* get_stub(const TNetworkAddress& taddr) { + virtual PBackendService_Stub* get_stub(const TNetworkAddress& taddr) { butil::EndPoint endpoint; if (str2endpoint(taddr.hostname.c_str(), taddr.port, &endpoint)) { LOG(WARNING) << "unknown endpoint, hostname=" << taddr.hostname; @@ -61,7 +61,7 @@ class BrpcStubCache { return get_stub(endpoint); } - PBackendService_Stub* get_stub(const std::string& host, int port) { + virtual PBackendService_Stub* get_stub(const std::string& host, int port) { butil::EndPoint endpoint; if (str2endpoint(host.c_str(), port, &endpoint)) { LOG(WARNING) << "unknown endpoint, hostname=" << host; diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp index 9667b51c25842e..050032c9092541 100644 --- a/be/src/util/hash_util.hpp +++ b/be/src/util/hash_util.hpp @@ -376,7 +376,7 @@ struct hash { } }; -#if !defined(IR_COMPILE) && __GNUC__ < 6 +#if !defined(IR_COMPILE) && __GNUC__ < 6 && !defined(__clang__) // Cause this is builtin function template <> struct hash<__int128> { diff --git a/be/src/util/radix_sort.h b/be/src/util/radix_sort.h index 1ccecdab49e6e7..eaa84876c07fa9 100644 --- a/be/src/util/radix_sort.h +++ b/be/src/util/radix_sort.h @@ -192,7 +192,7 @@ using RadixSortNumTraits = conditional_t< * To use RadixSort, you should define `Traits` to give out the information for sorting. * `RadixSortFloatTraits` is a good example to refer to. * Then you can run it as follows: - * RadixSort::executeLSD(arr, size); + * RadixSort::execute_lsd(arr, size); * * In particular, if you want to sort an array of numeric, you can use it easily as follows: * radixSortLSD(array_of_numeric, array_size); diff --git a/be/test/exec/broker_scan_node_test.cpp b/be/test/exec/broker_scan_node_test.cpp index 4936db867cf3a7..7f6d1d1a0fcc46 100644 --- a/be/test/exec/broker_scan_node_test.cpp +++ b/be/test/exec/broker_scan_node_test.cpp @@ -401,6 +401,7 @@ void BrokerScanNodeTest::init() { TEST_F(BrokerScanNodeTest, normal) { BrokerScanNode scan_node(&_obj_pool, _tnode, *_desc_tbl); + scan_node.init(_tnode); auto status = scan_node.prepare(&_runtime_state); ASSERT_TRUE(status.ok()); diff --git a/be/test/exec/json_scanner_test.cpp b/be/test/exec/json_scanner_test.cpp index b244e97efd2da6..7aa43af655cb1b 100644 --- a/be/test/exec/json_scanner_test.cpp +++ b/be/test/exec/json_scanner_test.cpp @@ -531,6 +531,7 @@ void JsonScannerTest::init() { TEST_F(JsonScannerTest, normal_simple_arrayjson) { BrokerScanNode scan_node(&_obj_pool, _tnode, *_desc_tbl); + scan_node.init(_tnode); auto status = scan_node.prepare(&_runtime_state); ASSERT_TRUE(status.ok()); @@ -580,6 +581,7 @@ TEST_F(JsonScannerTest, normal_simple_arrayjson) { // Use num_as_string load data again BrokerScanNode scan_node2(&_obj_pool, _tnode, *_desc_tbl); + scan_node2.init(_tnode); status = scan_node2.prepare(&_runtime_state); ASSERT_TRUE(status.ok()); scan_ranges.clear(); diff --git a/be/test/exec/json_scanner_test_with_jsonpath.cpp b/be/test/exec/json_scanner_test_with_jsonpath.cpp index d3d86a85c080da..8c1eb17e46fe9c 100644 --- a/be/test/exec/json_scanner_test_with_jsonpath.cpp +++ b/be/test/exec/json_scanner_test_with_jsonpath.cpp @@ -359,6 +359,7 @@ void JsonScannerTest::init() { TEST_F(JsonScannerTest, normal) { BrokerScanNode scan_node(&_obj_pool, _tnode, *_desc_tbl); + scan_node.init(_tnode); auto status = scan_node.prepare(&_runtime_state); ASSERT_TRUE(status.ok()); diff --git a/be/test/exec/parquet_scanner_test.cpp b/be/test/exec/parquet_scanner_test.cpp index 399c3646f541df..e6b61148050928 100644 --- a/be/test/exec/parquet_scanner_test.cpp +++ b/be/test/exec/parquet_scanner_test.cpp @@ -420,6 +420,7 @@ void ParquetScannerTest::init() { TEST_F(ParquetScannerTest, normal) { BrokerScanNode scan_node(&_obj_pool, _tnode, *_desc_tbl); + scan_node.init(_tnode); auto status = scan_node.prepare(&_runtime_state); ASSERT_TRUE(status.ok()); diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 44159f6e283d54..dacefbae3a5bc8 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -78,7 +78,7 @@ class TestColumn : public testing::Test { _length_buffers.clear(); } - void CreateColumnWriter(const TabletSchema& tablet_schema) { + void create_columnWriter(const TabletSchema& tablet_schema) { _column_writer = ColumnWriter::create(0, tablet_schema, _stream_factory, 1024, BLOOM_FILTER_DEFAULT_FPP); @@ -86,15 +86,15 @@ class TestColumn : public testing::Test { ASSERT_EQ(_column_writer->init(), OLAP_SUCCESS); } - void CreateColumnReader(const TabletSchema& tablet_schema) { + void create_columnReader(const TabletSchema& tablet_schema) { UniqueIdEncodingMap encodings; encodings[0] = ColumnEncodingMessage(); encodings[0].set_kind(ColumnEncodingMessage::DIRECT); encodings[0].set_dictionary_size(1); - CreateColumnReader(tablet_schema, encodings); + create_columnReader(tablet_schema, encodings); } - void CreateColumnReader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { + void create_columnReader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { UniqueIdToColumnIdMap included; included[0] = 0; UniqueIdToColumnIdMap segment_included; @@ -219,7 +219,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -244,7 +244,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -267,7 +267,7 @@ TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -303,7 +303,7 @@ TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -345,7 +345,7 @@ TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -374,7 +374,7 @@ TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -391,7 +391,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -416,7 +416,7 @@ TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -435,7 +435,7 @@ TEST_F(TestColumn, TinyColumnIndex) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -460,7 +460,7 @@ TEST_F(TestColumn, TinyColumnIndex) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -479,7 +479,7 @@ TEST_F(TestColumn, SeekTinyColumnWithPresent) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -515,7 +515,7 @@ TEST_F(TestColumn, SeekTinyColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -554,7 +554,7 @@ TEST_F(TestColumn, SkipTinyColumnWithPresent) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("TinyColumn", "TINYINT", "REPLACE", 1, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -583,7 +583,7 @@ TEST_F(TestColumn, SkipTinyColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -601,7 +601,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -626,7 +626,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -646,7 +646,7 @@ TEST_F(TestColumn, SeekShortColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -680,7 +680,7 @@ TEST_F(TestColumn, SeekShortColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -721,7 +721,7 @@ TEST_F(TestColumn, SkipShortColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -750,7 +750,7 @@ TEST_F(TestColumn, SkipShortColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -768,7 +768,7 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -802,7 +802,7 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -841,7 +841,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithPresent) { SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -866,7 +866,7 @@ TEST_F(TestColumn, VectorizedShortColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -888,7 +888,7 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -916,7 +916,7 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -933,7 +933,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("IntColumn", "INT", "REPLACE", 4, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -958,7 +958,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -977,7 +977,7 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("IntColumn", "INT", "REPLACE", 4, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -998,7 +998,7 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1024,7 +1024,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { // write data TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("IntColumn", "INT", "REPLACE", 4, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1049,7 +1049,7 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1072,7 +1072,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("LongColumnWithoutPresent", "BIGINT", "REPLACE", 8, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1098,7 +1098,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1118,7 +1118,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("LongColumnWithPresent", "BIGINT", "REPLACE", 8, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1144,7 +1144,7 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1166,7 +1166,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("FloatColumnWithoutPresent", "FLOAT", "REPLACE", 4, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1192,7 +1192,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1214,7 +1214,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { SetTabletSchemaWithOneColumn("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1240,7 +1240,7 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1263,7 +1263,7 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { SetTabletSchemaWithOneColumn("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1293,7 +1293,7 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1328,7 +1328,7 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { SetTabletSchemaWithOneColumn("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1354,7 +1354,7 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1373,7 +1373,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { SetTabletSchemaWithOneColumn("DoubleColumnWithoutPresent", "DOUBLE", "REPLACE", 8, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1399,7 +1399,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1421,7 +1421,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { SetTabletSchemaWithOneColumn("DoubleColumnWithPresent", "DOUBLE", "REPLACE", 8, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1447,7 +1447,7 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1471,7 +1471,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { SetTabletSchemaWithOneColumn("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1493,7 +1493,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1512,7 +1512,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { SetTabletSchemaWithOneColumn("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1540,7 +1540,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1566,7 +1566,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnZero) { SetTabletSchemaWithOneColumn("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1594,7 +1594,7 @@ TEST_F(TestColumn, VectorizedDatetimeColumnZero) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1622,7 +1622,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { SetTabletSchemaWithOneColumn("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1644,7 +1644,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1662,7 +1662,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { SetTabletSchemaWithOneColumn("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1692,7 +1692,7 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1718,7 +1718,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { SetTabletSchemaWithOneColumn("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1748,7 +1748,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1770,7 +1770,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { SetTabletSchemaWithOneColumn("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1799,7 +1799,7 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1822,7 +1822,7 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { SetTabletSchemaWithOneColumn("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1850,7 +1850,7 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1870,7 +1870,7 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1904,7 +1904,7 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -1950,7 +1950,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { string value2 = "-170141183460469231731687303715884105728"; // write data - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -1979,7 +1979,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2007,7 +2007,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { string value2 = "-170141183460469231731687303715884105728"; // write data - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2043,7 +2043,7 @@ TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2078,7 +2078,7 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { string value2 = "-170141183460469231731687303715884105728"; // write data - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2107,7 +2107,7 @@ TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2127,7 +2127,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { SetTabletSchemaWithOneColumn("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", 10, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2164,7 +2164,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2191,7 +2191,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", 10, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2220,7 +2220,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2243,7 +2243,7 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { SetTabletSchemaWithOneColumn("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2274,7 +2274,7 @@ TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2295,7 +2295,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { SetTabletSchemaWithOneColumn("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2330,7 +2330,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2368,7 +2368,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { SetTabletSchemaWithOneColumn("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2403,7 +2403,7 @@ TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2440,7 +2440,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("VarcharColumnWithoutoutPresent", "CHAR", "REPLACE", strlen("abcde"), false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2477,7 +2477,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2504,7 +2504,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("VarcharColumnWithoutoutPresent", "CHAR", "REPLACE", strlen("abcde"), true, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2531,7 +2531,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithPresent) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2553,7 +2553,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("VarcharColumnWithoutoutPresent", "CHAR", "REPLACE", 20, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2608,7 +2608,7 @@ TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); @@ -2638,7 +2638,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { TabletSchema tablet_schema; SetTabletSchemaWithOneColumn("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", 65535, false, true, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -2671,7 +2671,7 @@ TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); // read data - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(tablet_schema); diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp index 9001556c24c42d..3129b5bc3e909d 100644 --- a/be/test/olap/schema_change_test.cpp +++ b/be/test/olap/schema_change_test.cpp @@ -82,22 +82,22 @@ class TestColumn : public testing::Test { _length_buffers.clear(); } - void CreateColumnWriter(const TabletSchema& tablet_schema) { + void create_columnWriter(const TabletSchema& tablet_schema) { _column_writer = ColumnWriter::create(0, tablet_schema, _stream_factory, 1024, BLOOM_FILTER_DEFAULT_FPP); ASSERT_TRUE(_column_writer != NULL); ASSERT_EQ(_column_writer->init(), OLAP_SUCCESS); } - void CreateColumnReader(const TabletSchema& tablet_schema) { + void create_columnReader(const TabletSchema& tablet_schema) { UniqueIdEncodingMap encodings; encodings[0] = ColumnEncodingMessage(); encodings[0].set_kind(ColumnEncodingMessage::DIRECT); encodings[0].set_dictionary_size(1); - CreateColumnReader(tablet_schema, encodings); + create_columnReader(tablet_schema, encodings); } - void CreateColumnReader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { + void create_columnReader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { UniqueIdToColumnIdMap included; included[0] = 0; UniqueIdToColumnIdMap segment_included; @@ -204,7 +204,7 @@ class TestColumn : public testing::Test { TabletSchema src_tablet_schema; SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &src_tablet_schema); - CreateColumnWriter(src_tablet_schema); + create_columnWriter(src_tablet_schema); RowCursor write_row; write_row.init(src_tablet_schema); @@ -223,7 +223,7 @@ class TestColumn : public testing::Test { TabletSchema dst_tablet_schema; SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &dst_tablet_schema); - CreateColumnReader(src_tablet_schema); + create_columnReader(src_tablet_schema); RowCursor read_row; read_row.init(dst_tablet_schema); @@ -247,7 +247,7 @@ class TestColumn : public testing::Test { const std::string& value, OLAPStatus expected_st) { TabletSchema tablet_schema; SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -267,7 +267,7 @@ class TestColumn : public testing::Test { TabletSchema converted_tablet_schema; SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &converted_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(converted_tablet_schema); @@ -311,7 +311,7 @@ class TestColumn : public testing::Test { TEST_F(TestColumn, ConvertFloatToDouble) { TabletSchema tablet_schema; SetTabletSchema("FloatColumn", "FLOAT", "REPLACE", 4, false, false, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -339,7 +339,7 @@ TEST_F(TestColumn, ConvertFloatToDouble) { // read data TabletSchema convert_tablet_schema; SetTabletSchema("DoubleColumn", "DOUBLE", "REPLACE", 4, false, false, &convert_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(convert_tablet_schema); _col_vector.reset(new ColumnVector()); @@ -365,7 +365,7 @@ TEST_F(TestColumn, ConvertFloatToDouble) { TEST_F(TestColumn, ConvertDatetimeToDate) { TabletSchema tablet_schema; SetTabletSchema("DatetimeColumn", "DATETIME", "REPLACE", 8, false, false, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -389,7 +389,7 @@ TEST_F(TestColumn, ConvertDatetimeToDate) { // read data TabletSchema convert_tablet_schema; SetTabletSchema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(convert_tablet_schema); @@ -409,7 +409,7 @@ TEST_F(TestColumn, ConvertDatetimeToDate) { TEST_F(TestColumn, ConvertDateToDatetime) { TabletSchema tablet_schema; SetTabletSchema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -434,7 +434,7 @@ TEST_F(TestColumn, ConvertDateToDatetime) { TabletSchema convert_tablet_schema; SetTabletSchema("DateTimeColumn", "DATETIME", "REPLACE", 8, false, false, &convert_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(convert_tablet_schema); _col_vector.reset(new ColumnVector()); @@ -454,7 +454,7 @@ TEST_F(TestColumn, ConvertDateToDatetime) { TEST_F(TestColumn, ConvertIntToDate) { TabletSchema tablet_schema; SetTabletSchema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -475,7 +475,7 @@ TEST_F(TestColumn, ConvertIntToDate) { TabletSchema convert_tablet_schema; SetTabletSchema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(convert_tablet_schema); @@ -496,7 +496,7 @@ TEST_F(TestColumn, ConvertIntToDate) { TEST_F(TestColumn, ConvertVarcharToDate) { TabletSchema tablet_schema; SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -524,7 +524,7 @@ TEST_F(TestColumn, ConvertVarcharToDate) { helper.close(); TabletSchema convert_tablet_schema; SetTabletSchema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(convert_tablet_schema); @@ -538,7 +538,7 @@ TEST_F(TestColumn, ConvertVarcharToDate) { helper.close(); TabletSchema convert_tablet_schema; SetTabletSchema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - CreateColumnReader(tablet_schema); + create_columnReader(tablet_schema); RowCursor read_row; read_row.init(convert_tablet_schema); @@ -709,7 +709,7 @@ TEST_F(TestColumn, ConvertIntToBitmap) { TabletSchema tablet_schema; CreateTabletSchema(tablet_schema); //Base row block - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -790,7 +790,7 @@ TEST_F(TestColumn, ConvertCharToHLL) { CreateTabletSchema(tablet_schema); //Base row block - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowCursor write_row; write_row.init(tablet_schema); @@ -873,7 +873,7 @@ TEST_F(TestColumn, ConvertCharToCount) { CreateTabletSchema(tablet_schema); //Base row block - CreateColumnWriter(tablet_schema); + create_columnWriter(tablet_schema); RowBlock block(&tablet_schema); RowBlockInfo block_info; diff --git a/be/test/runtime/CMakeLists.txt b/be/test/runtime/CMakeLists.txt index fc3e37aa38c819..f9568960ade788 100644 --- a/be/test/runtime/CMakeLists.txt +++ b/be/test/runtime/CMakeLists.txt @@ -28,7 +28,7 @@ ADD_BE_TEST(free_list_test) ADD_BE_TEST(string_buffer_test) #ADD_BE_TEST(data_stream_test) #ADD_BE_TEST(parallel_executor_test) -ADD_BE_TEST(datetime_value_test) +#ADD_BE_TEST(datetime_value_test) ADD_BE_TEST(decimalv2_value_test) ADD_BE_TEST(large_int_value_test) ADD_BE_TEST(string_value_test) diff --git a/be/test/runtime/memory_scratch_sink_test.cpp b/be/test/runtime/memory_scratch_sink_test.cpp index a0d79a023fbcc2..b2443aced80aa7 100644 --- a/be/test/runtime/memory_scratch_sink_test.cpp +++ b/be/test/runtime/memory_scratch_sink_test.cpp @@ -219,6 +219,7 @@ TEST_F(MemoryScratchSinkTest, work_flow_normal) { _tnode.csv_scan_node.__set_file_paths(file_paths); CsvScanNode scan_node(&_obj_pool, _tnode, *_desc_tbl); + scan_node.init(_tnode); Status status = scan_node.prepare(_state); ASSERT_TRUE(status.ok()); diff --git a/be/test/util/arrow/arrow_work_flow_test.cpp b/be/test/util/arrow/arrow_work_flow_test.cpp index b237f211db0d63..6fd4e626e2158e 100644 --- a/be/test/util/arrow/arrow_work_flow_test.cpp +++ b/be/test/util/arrow/arrow_work_flow_test.cpp @@ -298,6 +298,7 @@ TEST_F(ArrowWorkFlowTest, NormalUse) { _tnode.csv_scan_node.__set_file_paths(file_paths); CsvScanNode scan_node(&_obj_pool, _tnode, *_desc_tbl); + scan_node.init(_tnode); Status status = scan_node.prepare(_state); ASSERT_TRUE(status.ok()); diff --git a/build.sh b/build.sh index 36ffb6dc504c65..d430cdbb198643 100755 --- a/build.sh +++ b/build.sh @@ -42,6 +42,14 @@ export DORIS_HOME=${ROOT} . ${DORIS_HOME}/env.sh +# build thirdparty libraries if necessary +if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libs2.a ]]; then + echo "Thirdparty libraries need to be build ..." + ${DORIS_THIRDPARTY}/build-thirdparty.sh +fi + +PARALLEL=$[$(nproc)/4+1] + # Check args usage() { echo " @@ -84,7 +92,7 @@ fi eval set -- "$OPTS" -PARALLEL=$[$(nproc)/4+1] +PARALLEL=$[$(nproc)+1] BUILD_BE= BUILD_FE= BUILD_UI= diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java index 91dafce7e68f7a..04e32448e381e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java @@ -18,7 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.AggregateFunction; -import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.Function; import org.apache.doris.catalog.Type; import com.google.common.base.MoreObjects; @@ -102,7 +102,7 @@ protected void createTupleDescs(Analyzer analyzer) { // Create the intermediate tuple desc first, so that the tuple ids are increasing // from bottom to top in the plan tree. intermediateTupleDesc_ = createTupleDesc(analyzer, false); - if (requiresIntermediateTuple(aggregateExprs_)) { + if (requiresIntermediateTuple(aggregateExprs_, groupingExprs_.size() == 0)) { outputTupleDesc_ = createTupleDesc(analyzer, true); } else { outputTupleDesc_ = intermediateTupleDesc_; @@ -148,16 +148,11 @@ private TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple slotDesc.setSourceExpr(aggExpr); } - // COUNT(), NDV() and NDV_NO_FINALIZE() are non-nullable. The latter two are used - // by compute stats and compute incremental stats, respectively. - if (aggExpr.getFnName().getFunction().equals(FunctionSet.COUNT) - || aggExpr.getFnName().getFunction().equals("ndv") - || aggExpr.getFnName().getFunction().equals(FunctionSet.BITMAP_UNION_INT) - || aggExpr.getFnName().getFunction().equals("ndv_no_finalize")) { - // TODO: Consider making nullability a property of types or of builtin agg fns. - // row_number(), rank(), and dense_rank() are non-nullable as well. - slotDesc.setIsNullable(false); + if (isOutputTuple && aggExpr.getFn().getNullableMode().equals(Function.NullableMode.DEPEND_ON_ARGUMENT) && + groupingExprs_.size() == 0) { + slotDesc.setIsNullable(true); } + if (!isOutputTuple) { Type intermediateType = ((AggregateFunction)aggExpr.fn).getIntermediateType(); if (intermediateType != null) { @@ -212,6 +207,21 @@ public static boolean requiresIntermediateTuple(List aggExpr return false; } + /** + * output tuple maybe different from intermediate when noGrouping and fn null mode + * is depend on argument + */ + public static boolean requiresIntermediateTuple(List aggExprs, boolean noGrouping) { + for (Expr aggExpr: aggExprs) { + Type intermediateType = ((AggregateFunction) aggExpr.fn).getIntermediateType(); + if (intermediateType != null) return true; + if (noGrouping && ((AggregateFunction) aggExpr.fn).getNullableMode().equals(Function.NullableMode.DEPEND_ON_ARGUMENT)) { + return true; + } + } + return false; + } + public String debugString() { StringBuilder out = new StringBuilder(); out.append(MoreObjects.toStringHelper(this) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java index b02eb0cb001dfa..c07a39afee1bb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; import org.apache.doris.thrift.TExprOpcode; @@ -105,17 +106,57 @@ public static void initBuiltins(FunctionSet functionSet) { functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( Operator.DIVIDE.getName(), Lists.newArrayList(Type.DOUBLE, Type.DOUBLE), - Type.DOUBLE)); + Type.DOUBLE, Function.NullableMode.ALWAYS_NULLABLE)); functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( Operator.DIVIDE.getName(), Lists.newArrayList(Type.DECIMALV2, Type.DECIMALV2), - Type.DECIMALV2)); + Type.DECIMALV2, Function.NullableMode.ALWAYS_NULLABLE)); // MOD(), FACTORIAL(), BITAND(), BITOR(), BITXOR(), and BITNOT() are registered as // builtins, see palo_functions.py for (Type t : Type.getIntegerTypes()) { functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.INT_DIVIDE.getName(), Lists.newArrayList(t, t), t)); + Operator.INT_DIVIDE.getName(), Lists.newArrayList(t, t), + t, Function.NullableMode.ALWAYS_NULLABLE)); + } + + // init vec build function + for (int i = 0; i < Type.getNumericTypes().size(); i++) { + Type t1 = Type.getNumericTypes().get(i); + for (int j = 0; j < Type.getNumericTypes().size(); j++) { + Type t2 = Type.getNumericTypes().get(j); + + functionSet.addBuiltin(ScalarFunction.createVecBuiltinOperator( + Operator.MULTIPLY.getName(), Lists.newArrayList(t1, t2), + Type.getAssignmentCompatibleType(t1, t2, false))); + functionSet.addBuiltin(ScalarFunction.createVecBuiltinOperator( + Operator.ADD.getName(), Lists.newArrayList(t1, t2), + Type.getAssignmentCompatibleType(t1, t2, false))); + functionSet.addBuiltin(ScalarFunction.createVecBuiltinOperator( + Operator.SUBTRACT.getName(), Lists.newArrayList(t1, t2), + Type.getAssignmentCompatibleType(t1, t2, false))); + } + } + + functionSet.addBuiltin(ScalarFunction.createVecBuiltinOperator( + Operator.DIVIDE.getName(), + Lists.newArrayList(Type.DOUBLE, Type.DOUBLE), + Type.DOUBLE, Function.NullableMode.ALWAYS_NULLABLE)); + functionSet.addBuiltin(ScalarFunction.createVecBuiltinOperator( + Operator.DIVIDE.getName(), + Lists.newArrayList(Type.DECIMALV2, Type.DECIMALV2), + Type.DECIMALV2, Function.NullableMode.ALWAYS_NULLABLE)); + + for (int i = 0; i < Type.getIntegerTypes().size(); i++) { + Type t1 = Type.getIntegerTypes().get(i); + for (int j = 0; j < Type.getIntegerTypes().size(); j++) { + Type t2 = Type.getIntegerTypes().get(j); + + functionSet.addBuiltin(ScalarFunction.createVecBuiltinOperator( + Operator.INT_DIVIDE.getName(), Lists.newArrayList(t1, t2), + Type.getAssignmentCompatibleType(t1, t2, false), + Function.NullableMode.ALWAYS_NULLABLE)); + } } } @@ -211,67 +252,104 @@ private Type findCommonType(Type t1, Type t2) { @Override public void analyzeImpl(Analyzer analyzer) throws AnalysisException { - // bitnot is the only unary op, deal with it here - if (op == Operator.BITNOT) { - type = Type.BIGINT; - if (getChild(0).getType().getPrimitiveType() != PrimitiveType.BIGINT) { - castChild(type, 0); + if (VectorizedUtil.isVectorized()) { + analyzeSubqueryInChildren(); + // if children has subquery, it will be rewritten and reanalyzed in the future. + if (contains(Subquery.class)) { + return; } - fn = getBuiltinFunction( - analyzer, op.getName(), collectChildReturnTypes(), Function.CompareMode.IS_SUPERTYPE_OF); - if (fn == null) { - Preconditions.checkState(false, String.format("No match for op with operand types", toSql())); - } - return; - } - analyzeSubqueryInChildren(); - // if children has subquery, it will be rewritten and reanalyzed in the future. - if (contains(Subquery.class)) { - return; - } + Type t1 = getChild(0).getType(); + Type t2 = getChild(1).getType(); + + switch (op) { + case MULTIPLY: + case ADD: + case SUBTRACT: + case MOD: + case INT_DIVIDE: + fn = getBuiltinFunction(analyzer, op.name, collectChildReturnTypes(), + Function.CompareMode.IS_IDENTICAL); + break; + case DIVIDE: + t1 = getChild(0).getType().getNumResultType(); + t2 = getChild(1).getType().getNumResultType(); + Type commonType = findCommonType(t1, t2); + if (commonType.getPrimitiveType() == PrimitiveType.BIGINT + || commonType.getPrimitiveType() == PrimitiveType.LARGEINT) { + commonType = Type.DOUBLE; + } + castBinaryOp(commonType); + fn = getBuiltinFunction(analyzer, op.name, collectChildReturnTypes(), + Function.CompareMode.IS_IDENTICAL); + break; + } - Type t1 = getChild(0).getType().getNumResultType(); - Type t2 = getChild(1).getType().getNumResultType(); - // Find result type of this operator - Type commonType = Type.INVALID; - String fnName = op.getName(); - switch (op) { - case MULTIPLY: - case ADD: - case SUBTRACT: - case MOD: - // numeric ops must be promoted to highest-resolution type - // (otherwise we can't guarantee that a b won't overflow/underflow) - commonType = findCommonType(t1, t2); - break; - case DIVIDE: - commonType = findCommonType(t1, t2); - if (commonType.getPrimitiveType() == PrimitiveType.BIGINT - || commonType.getPrimitiveType() == PrimitiveType.LARGEINT) { - commonType = Type.DOUBLE; + if (fn == null) { + Preconditions.checkState(false, String.format( + "No match for vec function '%s' with operand types %s and %s", toSql(), t1, t2)); + } + type = fn.getReturnType(); + } else { + // bitnot is the only unary op, deal with it here + if (op == Operator.BITNOT) { + type = Type.BIGINT; + if (getChild(0).getType().getPrimitiveType() != PrimitiveType.BIGINT) { + castChild(type, 0); } - break; - case INT_DIVIDE: - case BITAND: - case BITOR: - case BITXOR: - // Must be bigint - commonType = Type.BIGINT; - break; - default: - // the programmer forgot to deal with a case - Preconditions.checkState(false, - "Unknown arithmetic operation " + op.toString() + " in: " + this.toSql()); - break; - } - - type = castBinaryOp(commonType); - fn = getBuiltinFunction(analyzer, fnName, collectChildReturnTypes(), - Function.CompareMode.IS_IDENTICAL); - if (fn == null) { - Preconditions.checkState(false, String.format( - "No match for '%s' with operand types %s and %s", toSql(), t1, t2)); + fn = getBuiltinFunction( + analyzer, op.getName(), collectChildReturnTypes(), Function.CompareMode.IS_SUPERTYPE_OF); + if (fn == null) { + Preconditions.checkState(false, String.format("No match for op with operand types", toSql())); + } + return; + } + analyzeSubqueryInChildren(); + // if children has subquery, it will be rewritten and reanalyzed in the future. + if (contains(Subquery.class)) { + return; + } + Type t1 = getChild(0).getType().getNumResultType(); + Type t2 = getChild(1).getType().getNumResultType(); + // Find result type of this operator + Type commonType = Type.INVALID; + String fnName = op.getName(); + switch (op) { + case MULTIPLY: + case ADD: + case SUBTRACT: + case MOD: + // numeric ops must be promoted to highest-resolution type + // (otherwise we can't guarantee that a b won't overflow/underflow) + commonType = findCommonType(t1, t2); + break; + case DIVIDE: + commonType = findCommonType(t1, t2); + if (commonType.getPrimitiveType() == PrimitiveType.BIGINT + || commonType.getPrimitiveType() == PrimitiveType.LARGEINT) { + commonType = Type.DOUBLE; + } + break; + case INT_DIVIDE: + case BITAND: + case BITOR: + case BITXOR: + // Must be bigint + commonType = Type.BIGINT; + break; + default: + // the programmer forgot to deal with a case + Preconditions.checkState(false, + "Unknown arithmetic operation " + op.toString() + " in: " + this.toSql()); + break; + } + type = castBinaryOp(commonType); + fn = getBuiltinFunction(analyzer, fnName, collectChildReturnTypes(), + Function.CompareMode.IS_IDENTICAL); + if (fn == null) { + Preconditions.checkState(false, String.format( + "No match for '%s' with operand types %s and %s", toSql(), t1, t2)); + } } } @@ -304,4 +382,5 @@ public void analyzeSubqueryInChildren() throws AnalysisException { public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(op); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index 038a00e03023d0..6d51339f4fa2f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -170,19 +170,19 @@ protected BinaryPredicate(BinaryPredicate other) { public static void initBuiltins(FunctionSet functionSet) { for (Type t: Type.getSupportedTypes()) { if (t.isNull()) continue; // NULL is handled through type promotion. - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.EQ.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.NE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.LE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.GE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.LT.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.GT.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( Operator.EQ_FOR_NULL.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); } } @@ -650,4 +650,12 @@ public void setSelectivity() { public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(op); } + + @Override + public boolean isNullable() { + if (op == Operator.EQ_FOR_NULL) { + return false; + } + return hasNullableChild(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java index 34a1442d030694..950ad95711b4a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java @@ -375,4 +375,31 @@ private boolean checkSubquery(Expr expr) { return false; } + @Override + public boolean isNullable() { + int loopStart; + int loopEnd = children.size(); + if (hasCaseExpr) { + loopStart = 2; + } else { + loopStart = 1; + } + if (hasElseExpr) { + --loopEnd; + } + for (int i = loopStart; i < loopEnd; i += 2) { + Expr thenExpr = children.get(i); + if (thenExpr.isNullable()) { + return true; + } + } + if (hasElseExpr) { + if (children.get(children.size() - 1).isNullable()) { + return true; + } + } else { + return true; + } + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index dfdc4b01782b1b..ca46025d631761 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -17,6 +17,9 @@ package org.apache.doris.analysis; +import java.util.Arrays; +import java.util.Map; + import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionSet; @@ -25,6 +28,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -32,10 +36,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + public class CastExpr extends Expr { private static final Logger LOG = LogManager.getLogger(CastExpr.class); @@ -48,6 +54,29 @@ public class CastExpr extends Expr { // True if this cast does not change the type. private boolean noOp = false; + private static final Map, Function.NullableMode> TYPE_NULLABLE_MODE; + + static { + TYPE_NULLABLE_MODE = Maps.newHashMap(); + for (ScalarType fromType: Type.getSupportedTypes()) { + if (fromType.isNull()) { + continue; + } + for (ScalarType toType: Type.getSupportedTypes()) { + if (fromType.isNull()) { + continue; + } + if (fromType.isStringType() && !toType.isStringType()) { + TYPE_NULLABLE_MODE.put(new Pair<>(fromType, toType), Function.NullableMode.ALWAYS_NULLABLE); + } else if (!fromType.isDateType() && toType.isDateType()) { + TYPE_NULLABLE_MODE.put(new Pair<>(fromType, toType), Function.NullableMode.ALWAYS_NULLABLE); + } else { + TYPE_NULLABLE_MODE.put(new Pair<>(fromType, toType), Function.NullableMode.DEPEND_ON_ARGUMENT); + } + } + } + } + public CastExpr(Type targetType, Expr e) { super(); Preconditions.checkArgument(targetType.isValid()); @@ -120,8 +149,10 @@ public static void initBuiltins(FunctionSet functionSet) { } String beSymbol = "doris::" + beClass + "::cast_to_" + typeName; - functionSet.addBuiltin(ScalarFunction.createBuiltin(getFnName(toType), - Lists.newArrayList(fromType), false, toType, beSymbol, null, null, true)); + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltin(getFnName(toType), + toType, TYPE_NULLABLE_MODE.get(new Pair<>(fromType, toType)), + Lists.newArrayList(fromType), false , + beSymbol, null, null, true)); } } } @@ -186,7 +217,7 @@ public void analyze() throws AnalysisException { this.opcode = TExprOpcode.CAST; FunctionName fnName = new FunctionName(getFnName(type)); - Function searchDesc = new Function(fnName, collectChildReturnTypes(), Type.INVALID, false); + Function searchDesc = new Function(fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false); if (isImplicit) { fn = Catalog.getCurrentCatalog().getFunction( searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java index 499ab91f1b2a76..bda3233a26d1f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java @@ -17,6 +17,9 @@ package org.apache.doris.analysis; +import com.google.common.collect.Lists; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -38,6 +41,15 @@ public class CompoundPredicate extends Predicate { private final static Logger LOG = LogManager.getLogger(CompoundPredicate.class); private final Operator op; + public static void initBuiltins(FunctionSet functionSet) { + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( + Operator.AND.toString(), Lists.newArrayList(Type.BOOLEAN, Type.BOOLEAN), Type.BOOLEAN)); + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( + Operator.OR.toString(), Lists.newArrayList(Type.BOOLEAN, Type.BOOLEAN), Type.BOOLEAN)); + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( + Operator.NOT.toString(), Lists.newArrayList(Type.BOOLEAN), Type.BOOLEAN)); + } + public CompoundPredicate(Operator op, Expr e1, Expr e2) { super(); this.op = op; @@ -227,4 +239,9 @@ public Expr getResultValue() throws AnalysisException { public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(op); } + + @Override + public boolean isNullable() { + return hasNullableChild(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index c1475cf23d0fad..a939aadba1273d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -28,18 +28,13 @@ import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.LocalDateTime; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.DateTimeFormatterBuilder; - import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -52,6 +47,12 @@ import java.util.TimeZone; import java.util.regex.Pattern; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.DateTimeFormatterBuilder; + public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 6b29f8d7e95b4d..8c932ae0f8a433 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.TreeNode; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprOpcode; @@ -47,6 +48,7 @@ import java.io.IOException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -899,6 +901,10 @@ public TExpr treeToThrift() { return result; } + public void setFn(Function fn) { + this.fn = fn; + } + // Append a flattened version of this expr, including all children, to 'container'. protected void treeToThriftHelper(TExpr container) { TExprNode msg = new TExprNode(); @@ -911,6 +917,7 @@ protected void treeToThriftHelper(TExpr container) { } } msg.output_scale = getOutputScale(); + msg.setIsNullable(isNullable()); toThrift(msg); container.addToNodes(msg); for (Expr child : children) { @@ -1555,7 +1562,8 @@ protected Function getBuiltinFunction( Analyzer analyzer, String name, Type[] argTypes, Function.CompareMode mode) throws AnalysisException { FunctionName fnName = new FunctionName(name); - Function searchDesc = new Function(fnName, argTypes, Type.INVALID, false); + Function searchDesc = new Function(fnName, Arrays.asList(argTypes), Type.INVALID, false, + VectorizedUtil.isVectorized()); Function f = Catalog.getCurrentCatalog().getFunction(searchDesc, mode); if (f != null && fnName.getFunction().equalsIgnoreCase("rand")) { if (this.children.size() == 1 @@ -1820,4 +1828,64 @@ public boolean isContainsClass(String className) { return false; } + protected boolean hasNullableChild() { + for (Expr expr : children) { + if (expr.isNullable()) return true; + } + return false; + } + + /** + * For excute expr the result is nullable + * TODO: Now only SlotRef and LiteralExpr overwrite the method, each child of Expr should + * overwrite this method to plan correct + */ + public boolean isNullable() { + if (fn == null) { + return true; + } + switch (fn.getNullableMode()) { + case DEPEND_ON_ARGUMENT: + return hasNullableChild(); + case ALWAYS_NOT_NULLABLE: + return false; + case CUSTOM: + return customNullableAlgorithm(); + case ALWAYS_NULLABLE: + default: + return true; + } + } + + private boolean customNullableAlgorithm() { + Preconditions.checkState(fn.getNullableMode() == Function.NullableMode.CUSTOM); + if (fn.functionName().equalsIgnoreCase("if")) { + Preconditions.checkState(children.size() == 3); + for (int i = 1; i < children.size(); i++) { + if (children.get(i).isNullable()) { + return true; + } + } + return false; + } + if (fn.functionName().equalsIgnoreCase("ifnull")) { + Preconditions.checkState(children.size() == 2); + if (children.get(0).isNullable()) { + return children.get(1).isNullable(); + } + return false; + } + if (fn.functionName().equalsIgnoreCase("coalesce")) { + for (Expr expr : children) { + if (!expr.isNullable()) { + return false; + } + } + return true; + } + if (fn.functionName().equalsIgnoreCase("concat_ws")) { + return children.get(0).isNullable(); + } + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java index 55f0f64565cbe0..2d2708ac19e5e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java @@ -76,7 +76,8 @@ public Expr evalExpr(Expr constExpr) { // 1. Not UDF // 2. Not in NonNullResultWithNullParamFunctions // 3. Has null parameter - if (!Catalog.getCurrentCatalog().isNonNullResultWithNullParamFunction(fn.getFunctionName().getFunction()) + if ((fn.getNullableMode() == Function.NullableMode.DEPEND_ON_ARGUMENT + || Catalog.getCurrentCatalog().isNullResultWithOneNullParamFunction(fn.getFunctionName().getFunction())) && !fn.isUdf()) { for (Expr e : constExpr.getChildren()) { if (e instanceof NullLiteral) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index a04cdb105b62af..5ee8fd8de18681 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -29,6 +29,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TAggregateExpr; @@ -49,6 +50,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Arrays; import java.util.List; // TODO: for aggregations, we need to unify the code paths for builtins and UDAs. @@ -563,7 +565,11 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { if (this.children.isEmpty()) { throw new AnalysisException("The " + fnName + " function must has one input param"); } - Type type = getChild(0).type.getMaxResolutionType(); + // Prevent the cast type in vector exec engine + Type type = getChild(0).type; + if (!VectorizedUtil.isVectorized()) { + type = getChild(0).type.getMaxResolutionType(); + } fn = getBuiltinFunction(analyzer, fnName.getFunction(), new Type[]{type}, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); } else if (fnName.getFunction().equalsIgnoreCase("count_distinct")) { @@ -604,7 +610,7 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { Database db = Catalog.getCurrentCatalog().getDb(dbName); if (db != null) { Function searchDesc = new Function( - fnName, collectChildReturnTypes(), Type.INVALID, false); + fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false); fn = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java index e684aa27bb617d..0edd1a083419a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java @@ -68,19 +68,19 @@ public static void initBuiltins(FunctionSet functionSet) { String typeString = Function.getUdfTypeName(t.getPrimitiveType()); functionSet.addBuiltin(ScalarFunction.createBuiltin(IN_ITERATE, - Lists.newArrayList(t, t), true, Type.BOOLEAN, + Type.BOOLEAN, Lists.newArrayList(t, t), true, "doris::InPredicate::in_iterate", null, null, false)); functionSet.addBuiltin(ScalarFunction.createBuiltin(NOT_IN_ITERATE, - Lists.newArrayList(t, t), true, Type.BOOLEAN, + Type.BOOLEAN, Lists.newArrayList(t, t), true, "doris::InPredicate::not_in_iterate", null, null, false)); String prepareFn = "doris::InPredicate::set_lookup_prepare_" + typeString; String closeFn = "doris::InPredicate::set_lookup_close_" + typeString; functionSet.addBuiltin(ScalarFunction.createBuiltin(IN_SET_LOOKUP, - Lists.newArrayList(t, t), true, Type.BOOLEAN, + Type.BOOLEAN, Lists.newArrayList(t, t), true, "doris::InPredicate::in_set_lookup", prepareFn, closeFn, false)); functionSet.addBuiltin(ScalarFunction.createBuiltin(NOT_IN_SET_LOOKUP, - Lists.newArrayList(t, t), true, Type.BOOLEAN, + Type.BOOLEAN, Lists.newArrayList(t, t), true, "doris::InPredicate::not_in_set_lookup", prepareFn, closeFn, false)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java index 4e3cb88e6e2c26..a498e36b5e5277 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java @@ -266,9 +266,8 @@ public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisE } columnSet.add(colAlias); - // TODO: inlineView threat all column is nullable to make sure query results are correct - // we should judge column whether is nullable by selectItemExpr in the future - columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType(), true)); + columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType(), + selectItemExpr.isNullable())); } InlineView inlineView = (view != null) ? new InlineView(view, columnList) : new InlineView(getExplicitAlias(), columnList); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java index 740ea361b49bd4..862ac9afd78ceb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java @@ -48,11 +48,11 @@ public static void initBuiltins(FunctionSet functionSet) { "EEENS2_10BooleanValEPNS2_15FunctionContextERKT_"; } - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( IS_NULL, isNullSymbol, Lists.newArrayList(t), Type.BOOLEAN)); String isNotNullSymbol = isNullSymbol.replace("7is_null", "11is_not_null"); - functionSet.addBuiltin(ScalarFunction.createBuiltinOperator( + functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator( IS_NOT_NULL, isNotNullSymbol, Lists.newArrayList(t), Type.BOOLEAN)); } } @@ -128,4 +128,8 @@ public Expr negate() { return new IsNullPredicate(getChild(0), !isNotNull); } + @Override + public boolean isNullable() { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java index c4803235b30bfe..f25b3ea70349db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java @@ -17,10 +17,6 @@ package org.apache.doris.analysis; -import java.util.Objects; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.ScalarFunction; @@ -32,6 +28,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + public class LikePredicate extends Predicate { public enum Operator { @@ -52,14 +52,14 @@ public String toString() { public static void initBuiltins(FunctionSet functionSet) { functionSet.addBuiltin(ScalarFunction.createBuiltin( - Operator.LIKE.name(), Lists.newArrayList(Type.VARCHAR, Type.VARCHAR), - false, Type.BOOLEAN, + Operator.LIKE.name(), Type.BOOLEAN, Lists.newArrayList(Type.VARCHAR, Type.VARCHAR), + false, "_ZN5doris13LikePredicate4likeEPN9doris_udf15FunctionContextERKNS1_9StringValES6_", "_ZN5doris13LikePredicate12like_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE", "_ZN5doris13LikePredicate10like_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE", true)); functionSet.addBuiltin(ScalarFunction.createBuiltin( - Operator.REGEXP.name(), Lists.newArrayList(Type.VARCHAR, Type.VARCHAR), - false, Type.BOOLEAN, + Operator.REGEXP.name(), Type.BOOLEAN, Lists.newArrayList(Type.VARCHAR, Type.VARCHAR), + false, "_ZN5doris13LikePredicate5regexEPN9doris_udf15FunctionContextERKNS1_9StringValES6_", "_ZN5doris13LikePredicate13regex_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE", "_ZN5doris13LikePredicate11regex_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE", true)); @@ -140,4 +140,5 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(op); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index f6ab8f669e0195..40b89b3c14c4ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -194,5 +194,10 @@ public boolean equals(Object obj) { } return this.compareLiteral(((LiteralExpr) obj)) == 0; } + + @Override + public boolean isNullable() { + return this instanceof NullLiteral; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 8803bd05ba640a..8081ff57e08c9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -222,6 +222,7 @@ public int getSlotOffset() { * Initializes a slot by setting its source expression information */ public void initFromExpr(Expr expr) { + setIsNullable(expr.isNullable()); setLabel(expr.toSql()); Preconditions.checkState(sourceExprs_.isEmpty()); setSourceExpr(expr); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index cd52ef6de07808..006beffe2d840a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -401,4 +401,9 @@ public static SlotRef read(DataInput in) throws IOException { slotRef.readFields(in); return slotRef; } + + @Override + public boolean isNullable() { + return desc.getIsNullable(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java index ff3cae0c1d2b70..04fcff25fffadd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java @@ -20,6 +20,7 @@ import static org.apache.doris.common.io.IOUtils.readOptionStringOrNull; import static org.apache.doris.common.io.IOUtils.writeOptionString; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gson.Gson; import org.apache.doris.analysis.CreateFunctionStmt; @@ -35,19 +36,21 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; -// import org.apache.doris.analysis.String; - /** * Internal representation of an aggregate function. * TODO: Create separate AnalyticFunction class */ public class AggregateFunction extends Function { - + private static final Logger LOG = LogManager.getLogger(AggregateFunction.class); + + public static ImmutableSet NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET = + ImmutableSet.of(FunctionSet.COUNT, "ndv", FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT, "ndv_no_finalize"); + // Set if different from retType_, null otherwise. private Type intermediateType; @@ -90,30 +93,22 @@ public class AggregateFunction extends Function { protected AggregateFunction() { } - public AggregateFunction(FunctionName fnName, ArrayList argTypes, Type retType, - boolean hasVarArgs) { - super(fnName, argTypes, retType, hasVarArgs); + public AggregateFunction(FunctionName fnName, List argTypes, + Type retType, Type intermediateType, + HdfsURI location, String updateFnSymbol, String initFnSymbol, + String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, + String removeFnSymbol, String finalizeFnSymbol) { + this(fnName, argTypes, retType, intermediateType, location, updateFnSymbol, initFnSymbol, serializeFnSymbol, + mergeFnSymbol, getValueFnSymbol, removeFnSymbol, finalizeFnSymbol, false); } public AggregateFunction(FunctionName fnName, List argTypes, - Type retType, Type intermediateType, - HdfsURI location, String updateFnSymbol, String initFnSymbol, - String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, - String removeFnSymbol, String finalizeFnSymbol) { - super(fnName, argTypes, retType, false); - setLocation(location); - this.intermediateType = (intermediateType.equals(retType)) ? null : intermediateType; - this.updateFnSymbol = updateFnSymbol; - this.initFnSymbol = initFnSymbol; - this.serializeFnSymbol = serializeFnSymbol; - this.mergeFnSymbol = mergeFnSymbol; - this.getValueFnSymbol = getValueFnSymbol; - this.removeFnSymbol = removeFnSymbol; - this.finalizeFnSymbol = finalizeFnSymbol; - ignoresDistinct = false; - isAnalyticFn = false; - isAggregateFn = true; - returnsNonNullOnEmpty = false; + Type retType, Type intermediateType, + HdfsURI location, String updateFnSymbol, String initFnSymbol, + String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, + String removeFnSymbol, String finalizeFnSymbol, boolean vectorized) { + this(fnName, argTypes, retType, intermediateType, false, location, updateFnSymbol, initFnSymbol, serializeFnSymbol, + mergeFnSymbol, getValueFnSymbol, removeFnSymbol, finalizeFnSymbol, vectorized); } public AggregateFunction(FunctionName fnName, List argTypes, @@ -121,7 +116,18 @@ public AggregateFunction(FunctionName fnName, List argTypes, HdfsURI location, String updateFnSymbol, String initFnSymbol, String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, String removeFnSymbol, String finalizeFnSymbol) { - super(fnName, argTypes, retType, hasVarArgs); + this(fnName, argTypes, retType, intermediateType, hasVarArgs, location, updateFnSymbol, initFnSymbol, serializeFnSymbol, + mergeFnSymbol, getValueFnSymbol, removeFnSymbol, finalizeFnSymbol, false); + } + + public AggregateFunction(FunctionName fnName, List argTypes, + Type retType, Type intermediateType, boolean hasVarArgs, + HdfsURI location, String updateFnSymbol, String initFnSymbol, + String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, + String removeFnSymbol, String finalizeFnSymbol, boolean vectorized) { + // only `count` is always not nullable, other aggregate function is always nullable + super(fnName, argTypes, retType, hasVarArgs, vectorized, + AggregateFunction.NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET.contains(fnName.getFunction()) ? NullableMode.ALWAYS_NOT_NULLABLE : NullableMode.DEPEND_ON_ARGUMENT); setLocation(location); this.intermediateType = (intermediateType.equals(retType)) ? null : intermediateType; this.updateFnSymbol = updateFnSymbol; @@ -138,25 +144,46 @@ public AggregateFunction(FunctionName fnName, List argTypes, } public static AggregateFunction createBuiltin(String name, - List argTypes, Type retType, Type intermediateType, - String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, - String serializeFnSymbol, String finalizeFnSymbol, boolean ignoresDistinct, - boolean isAnalyticFn, boolean returnsNonNullOnEmpty) { + List argTypes, Type retType, Type intermediateType, + String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, + String serializeFnSymbol, String finalizeFnSymbol, boolean ignoresDistinct, + boolean isAnalyticFn, boolean returnsNonNullOnEmpty) { + return createBuiltin(name, argTypes, retType, intermediateType, + initFnSymbol, updateFnSymbol, mergeFnSymbol, + serializeFnSymbol, finalizeFnSymbol, ignoresDistinct, isAnalyticFn, returnsNonNullOnEmpty, false); + } + public static AggregateFunction createBuiltin(String name, + List argTypes, Type retType, Type intermediateType, + String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, + String serializeFnSymbol, String finalizeFnSymbol, boolean ignoresDistinct, + boolean isAnalyticFn, boolean returnsNonNullOnEmpty, boolean vectorized) { return createBuiltin(name, argTypes, retType, intermediateType, initFnSymbol, - updateFnSymbol, mergeFnSymbol, serializeFnSymbol, null, null, finalizeFnSymbol, - ignoresDistinct, isAnalyticFn, returnsNonNullOnEmpty); + updateFnSymbol, mergeFnSymbol, serializeFnSymbol, null, null, finalizeFnSymbol, + ignoresDistinct, isAnalyticFn, returnsNonNullOnEmpty, vectorized); } public static AggregateFunction createBuiltin(String name, - List argTypes, Type retType, Type intermediateType, - String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, - String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, - String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, - boolean returnsNonNullOnEmpty) { + List argTypes, Type retType, Type intermediateType, + String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, + String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, + String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, + boolean returnsNonNullOnEmpty) { + return createBuiltin(name, argTypes, retType, intermediateType, + initFnSymbol, updateFnSymbol, mergeFnSymbol, + serializeFnSymbol, getValueFnSymbol, removeFnSymbol, + finalizeFnSymbol, ignoresDistinct, isAnalyticFn, returnsNonNullOnEmpty, false); + } + + public static AggregateFunction createBuiltin(String name, + List argTypes, Type retType, Type intermediateType, + String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, + String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, + String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, + boolean returnsNonNullOnEmpty, boolean vectorized) { return createBuiltin(name, argTypes, retType, intermediateType, false, initFnSymbol, updateFnSymbol, mergeFnSymbol, serializeFnSymbol, getValueFnSymbol, removeFnSymbol, - finalizeFnSymbol, ignoresDistinct, isAnalyticFn,returnsNonNullOnEmpty); + finalizeFnSymbol, ignoresDistinct, isAnalyticFn, returnsNonNullOnEmpty, vectorized); } public static AggregateFunction createBuiltin(String name, @@ -165,10 +192,21 @@ public static AggregateFunction createBuiltin(String name, String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, boolean returnsNonNullOnEmpty) { + return createBuiltin(name, argTypes, retType, intermediateType, hasVarArgs, initFnSymbol, updateFnSymbol, + mergeFnSymbol, serializeFnSymbol, getValueFnSymbol, removeFnSymbol, finalizeFnSymbol, ignoresDistinct, + isAnalyticFn, returnsNonNullOnEmpty, false); + } + + public static AggregateFunction createBuiltin(String name, + List argTypes, Type retType, Type intermediateType, boolean hasVarArgs, + String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, + String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, + String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, + boolean returnsNonNullOnEmpty, boolean vectorized) { AggregateFunction fn = new AggregateFunction(new FunctionName(name), argTypes, retType, intermediateType, hasVarArgs, null, updateFnSymbol, initFnSymbol, serializeFnSymbol, mergeFnSymbol, getValueFnSymbol, removeFnSymbol, - finalizeFnSymbol); + finalizeFnSymbol, vectorized); fn.setBinaryType(TFunctionBinaryType.BUILTIN); fn.ignoresDistinct = ignoresDistinct; fn.isAnalyticFn = isAnalyticFn; @@ -214,7 +252,7 @@ public AggregateFunction(FunctionName fnName, Type[] argTypes, String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, String serializeFnSymbol, String finalizeFnSymbol, String getValueFnSymbol, String removeFnSymbol) { - super(fnName, argTypes, retType, hasVarArgs); + super(fnName, Arrays.asList(argTypes), retType, hasVarArgs); this.setLocation(new HdfsURI(location)); this.intermediateType = (intermediateType.equals(retType)) ? null : intermediateType; this.updateFnSymbol = updateFnSymbol; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 63ab00000a1781..6dc90b73fb82f1 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -233,14 +233,10 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Queues; import com.google.common.collect.Sets; -import com.sleepycat.je.rep.InsufficientLogException; -import com.sleepycat.je.rep.NetworkRestore; -import com.sleepycat.je.rep.NetworkRestoreConfig; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.codehaus.jackson.map.ObjectMapper; import java.io.BufferedReader; import java.io.DataInputStream; @@ -267,6 +263,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import com.sleepycat.je.rep.InsufficientLogException; +import com.sleepycat.je.rep.NetworkRestore; +import com.sleepycat.je.rep.NetworkRestoreConfig; +import org.codehaus.jackson.map.ObjectMapper; + public class Catalog { private static final Logger LOG = LogManager.getLogger(Catalog.class); // 0 ~ 9999 used for qe @@ -5727,8 +5728,8 @@ public List getBuiltinFunctions() { return functionSet.getBulitinFunctions(); } - public boolean isNonNullResultWithNullParamFunction(String funcName) { - return functionSet.isNonNullResultWithNullParamFunctions(funcName); + public boolean isNullResultWithOneNullParamFunction(String funcName) { + return functionSet.isNullResultWithOneNullParamFunctions(funcName); } public boolean isNondeterministicFunction(String funcName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 6931b19c0379d2..7802c0dbb8672f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -82,6 +82,18 @@ public enum CompareMode { IS_MATCHABLE } + public enum NullableMode { + // Whether output column is nullable is depend on the input column is nullable + DEPEND_ON_ARGUMENT, + // like 'str_to_date', 'cast', 'date_format' etc, the output column is nullable + // depend on input content + ALWAYS_NULLABLE, + // like 'count', the output column is always not nullable + ALWAYS_NOT_NULLABLE, + // Whether output column is nullable is depend on custom algorithm by @Expr.isNullable() + CUSTOM + } + public static final long UNIQUE_FUNCTION_ID = 0; // Function id, every function has a unique id. Now all built-in functions' id is 0 private long id = 0; @@ -104,6 +116,10 @@ public enum CompareMode { private HdfsURI location; private TFunctionBinaryType binaryType; + protected NullableMode nullableMode = NullableMode.DEPEND_ON_ARGUMENT; + + private boolean vectorized = false; + // library's checksum to make sure all backends use one library to serve user's request protected String checksum = ""; @@ -111,33 +127,38 @@ public enum CompareMode { protected Function() { } - public Function(FunctionName name, Type[] argTypes, Type retType, boolean varArgs) { - this(0, name, argTypes, retType, varArgs); + public Function(FunctionName name, List args, Type retType, boolean varArgs) { + this(0, name, args, retType, varArgs, false, NullableMode.DEPEND_ON_ARGUMENT); } - public Function(FunctionName name, List args, Type retType, boolean varArgs) { - this(0, name, args, retType, varArgs); + public Function(FunctionName name, List args, Type retType, boolean varArgs, boolean vectorized) { + this(0, name, args, retType, varArgs, vectorized, NullableMode.DEPEND_ON_ARGUMENT); } - public Function(long id, FunctionName name, Type[] argTypes, Type retType, boolean hasVarArgs) { + public Function(FunctionName name, List args, Type retType, boolean varArgs, boolean vectorized, NullableMode mode) { + this(0, name, args, retType, varArgs, vectorized, mode); + } + + public Function(long id, FunctionName name, List argTypes, Type retType, boolean hasVarArgs, + TFunctionBinaryType binaryType, boolean userVisible, boolean vectorized, NullableMode mode) { this.id = id; this.name = name; this.hasVarArgs = hasVarArgs; - if (argTypes == null) { - this.argTypes = new Type[0]; - } else { - this.argTypes = argTypes; - } - this.retType = retType; - } - - public Function(long id, FunctionName name, List argTypes, Type retType, boolean hasVarArgs) { - this(id, name, (Type[]) null, retType, hasVarArgs); if (argTypes.size() > 0) { this.argTypes = argTypes.toArray(new Type[argTypes.size()]); } else { this.argTypes = new Type[0]; } + this.retType = retType; + this.binaryType = binaryType; + this.userVisible = userVisible; + this.vectorized = vectorized; + this.nullableMode = mode; + } + + public Function(long id, FunctionName name, List argTypes, Type retType, + boolean hasVarArgs, boolean vectorized, NullableMode mode) { + this(id, name, argTypes, retType, hasVarArgs, TFunctionBinaryType.BUILTIN, true, vectorized, mode); } public FunctionName getFunctionName() { @@ -438,6 +459,7 @@ public TFunction toThrift() { if (!checksum.isEmpty()) { fn.setChecksum(checksum); } + fn.setVectorized(vectorized); return fn; } @@ -612,6 +634,7 @@ protected void writeFields(DataOutput output) throws IOException { } writeOptionString(output, libUrl); writeOptionString(output, checksum); + output.writeBoolean(vectorized); } @Override @@ -692,4 +715,12 @@ public List getInfo(boolean isVerbose) { } return row; } + + boolean isVectorized() { + return vectorized; + } + + public NullableMode getNullableMode() { + return nullableMode; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 2abaf6b3086839..1c196c7c6c8fb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -17,9 +17,11 @@ package org.apache.doris.catalog; +import com.google.common.base.Preconditions; import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.LikePredicate; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class FunctionSet { +public class FunctionSet { private static final Logger LOG = LogManager.getLogger(FunctionSet.class); // All of the registered user functions. The key is the user facing name (e.g. "myUdf"), @@ -48,7 +50,7 @@ public class FunctionSet { // on this map. Functions are sorted in a canonical order defined by // FunctionResolutionOrder. private final HashMap> functions; - + private final HashMap> vectorizedFunctions; // For most build-in functions, it will return NullLiteral when params contain NullLiteral. // But a few functions need to handle NullLiteral differently, such as "if". It need to add // an attribute to LiteralExpr to mark null and check the attribute to decide whether to @@ -56,14 +58,14 @@ public class FunctionSet { // Functions in this set is defined in `gensrc/script/doris_builtins_functions.py`, // and will be built automatically. - // cmy: This does not contain any user defined functions. All UDFs handle null values by themselves. - private ImmutableSet nonNullResultWithNullParamFunctions; + private ImmutableSet nullResultWithOneNullParamFunctions; // Including now(), curdate(), etc.. private ImmutableSet nondeterministicFunctions; public FunctionSet() { functions = Maps.newHashMap(); + vectorizedFunctions = Maps.newHashMap(); } public void init() { @@ -72,19 +74,21 @@ public void init() { ArithmeticExpr.initBuiltins(this); BinaryPredicate.initBuiltins(this); + CompoundPredicate.initBuiltins(this); CastExpr.initBuiltins(this); + IsNullPredicate.initBuiltins(this); ScalarBuiltins.initBuiltins(this); LikePredicate.initBuiltins(this); InPredicate.initBuiltins(this); } - public void buildNonNullResultWithNullParamFunction(Set funcNames) { + public void buildNullResultWithOneNullParamFunction(Set funcNames) { ImmutableSet.Builder setBuilder = new ImmutableSet.Builder(); for (String funcName : funcNames) { setBuilder.add(funcName); } - this.nonNullResultWithNullParamFunctions = setBuilder.build(); + this.nullResultWithOneNullParamFunctions = setBuilder.build(); } public void buildNondeterministicFunctions(Set funcNames) { @@ -95,15 +99,74 @@ public void buildNondeterministicFunctions(Set funcNames) { this.nondeterministicFunctions = setBuilder.build(); } - - public boolean isNonNullResultWithNullParamFunctions(String funcName) { - return nonNullResultWithNullParamFunctions.contains(funcName); - } - public boolean isNondeterministicFunction(String funcName) { return nondeterministicFunctions.contains(funcName); } + public boolean isNullResultWithOneNullParamFunctions(String funcName) { + return nullResultWithOneNullParamFunctions.contains(funcName); + } + + private static final Map MIN_INIT_SYMBOL = + ImmutableMap.builder() + .put(Type.BOOLEAN, + "8min_initIN9doris_udf10BooleanValEEEvPNS2_15FunctionContextEPT_") + .put(Type.TINYINT, + "8min_initIN9doris_udf10TinyIntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.SMALLINT, + "8min_initIN9doris_udf11SmallIntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.INT, + "8min_initIN9doris_udf6IntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.BIGINT, + "8min_initIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.FLOAT, + "8min_initIN9doris_udf8FloatValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DOUBLE, + "8min_initIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextEPT_") + // .put(Type.CHAR, + // "3minIN9doris_udf9StringValEEEvPNS2_15FunctionContextERKT_PS6_") + .put(Type.VARCHAR, + "8min_initIN9doris_udf9StringValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DATE, + "8min_initIN9doris_udf11DateTimeValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DATETIME, + "8min_initIN9doris_udf11DateTimeValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DECIMALV2, + "8min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionContextEPT_") + .put(Type.LARGEINT, + "8min_initIN9doris_udf11LargeIntValEEEvPNS2_15FunctionContextEPT_") + .build(); + + private static final Map MAX_INIT_SYMBOL = + ImmutableMap.builder() + .put(Type.BOOLEAN, + "8max_initIN9doris_udf10BooleanValEEEvPNS2_15FunctionContextEPT_") + .put(Type.TINYINT, + "8max_initIN9doris_udf10TinyIntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.SMALLINT, + "8max_initIN9doris_udf11SmallIntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.INT, + "8max_initIN9doris_udf6IntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.BIGINT, + "8max_initIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_") + .put(Type.FLOAT, + "8max_initIN9doris_udf8FloatValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DOUBLE, + "8max_initIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextEPT_") + // .put(Type.CHAR, + // "3minIN9doris_udf9StringValEEEvPNS2_15FunctionContextERKT_PS6_") + .put(Type.VARCHAR, + "8max_initIN9doris_udf9StringValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DATE, + "8max_initIN9doris_udf11DateTimeValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DATETIME, + "8max_initIN9doris_udf11DateTimeValEEEvPNS2_15FunctionContextEPT_") + .put(Type.DECIMALV2, + "8max_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionContextEPT_") + .put(Type.LARGEINT, + "8max_initIN9doris_udf11LargeIntValEEEvPNS2_15FunctionContextEPT_") + .build(); + private static final Map MIN_UPDATE_SYMBOL = ImmutableMap.builder() .put(Type.BOOLEAN, @@ -874,7 +937,7 @@ public boolean isNondeterministicFunction(String funcName) { .build(); public Function getFunction(Function desc, Function.CompareMode mode) { - List fns = functions.get(desc.functionName()); + List fns = desc.isVectorized() ? vectorizedFunctions.get(desc.functionName()) : functions.get(desc.functionName()); if (fns == null) { return null; } @@ -944,8 +1007,8 @@ public static boolean isCastMatchAllowed(Function desc, Function candicate) { return true; } - public Function getFunction(String signatureString) { - for (List fns : functions.values()) { + public Function getFunction(String signatureString, boolean vectorized) { + for (List fns : vectorized ? vectorizedFunctions.values() : functions.values()) { for (Function f : fns) { if (f.signatureString().equals(signatureString)) { return f; @@ -960,38 +1023,45 @@ private boolean addFunction(Function fn, boolean isBuiltin) { if (getFunction(fn, Function.CompareMode.IS_INDISTINGUISHABLE) != null) { return false; } - List fns = functions.get(fn.functionName()); + List fns = fn.isVectorized() ? vectorizedFunctions.get(fn.functionName()) : functions.get(fn.functionName()); if (fns == null) { fns = Lists.newArrayList(); - functions.put(fn.functionName(), fns); - } - if (fns.add(fn)) { - return true; + if (fn.isVectorized()) { + vectorizedFunctions.put(fn.functionName(), fns); + } else { + functions.put(fn.functionName(), fns); + } } - return false; - } - - /** - * Add a builtin with the specified name and signatures to this - * This defaults to not using a Prepare/Close function. - */ - public void addScalarBuiltin(String fnName, String symbol, boolean userVisible, - boolean varArgs, PrimitiveType retType, PrimitiveType ... args) { - addScalarBuiltin(fnName, symbol, userVisible, null, null, varArgs, retType, args); + fns.add(fn); + return true; } /** * Add a builtin with the specified name and signatures to this db. */ public void addScalarBuiltin(String fnName, String symbol, boolean userVisible, - String prepareFnSymbol, String closeFnSymbol, boolean varArgs, - PrimitiveType retType, PrimitiveType ... args) { + String prepareFnSymbol, String closeFnSymbol, + Function.NullableMode nullableMode, PrimitiveType retType, + boolean varArgs, PrimitiveType ... args) { ArrayList argsType = new ArrayList(); for (PrimitiveType type : args) { argsType.add(Type.fromPrimitiveType(type)); } addBuiltin(ScalarFunction.createBuiltin( - fnName, argsType, varArgs, Type.fromPrimitiveType(retType), + fnName, Type.fromPrimitiveType(retType), nullableMode, argsType, varArgs, + symbol, prepareFnSymbol, closeFnSymbol, userVisible)); + } + + public void addScalarAndVectorizedBuiltin(String fnName, String symbol, boolean userVisible, + String prepareFnSymbol, String closeFnSymbol, + Function.NullableMode nullableMode, PrimitiveType retType, + boolean varArgs, PrimitiveType ... args) { + ArrayList argsType = new ArrayList(); + for (PrimitiveType type : args) { + argsType.add(Type.fromPrimitiveType(type)); + } + addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltin( + fnName, Type.fromPrimitiveType(retType), nullableMode, argsType, varArgs, symbol, prepareFnSymbol, closeFnSymbol, userVisible)); } @@ -1002,6 +1072,37 @@ public void addBuiltin(Function fn) { addFunction(fn, true); } + /** + * Adds a function both in FunctionSet and VecFunctionSet to this database. + * The function must not already exist and need to be not vectorized + */ + public void addBuiltinBothScalaAndVectorized(Function fn) { + if (getFunction(fn, Function.CompareMode.IS_INDISTINGUISHABLE) != null) { + return; + } + Preconditions.checkState(!fn.isVectorized()); + + // add scala function + List fns = functions.get(fn.functionName()); + if (fns == null) { + fns = Lists.newArrayList(); + functions.put(fn.functionName(), fns); + } + fns.add(fn); + + // add vectorized function + List vecFns = vectorizedFunctions.get(fn.functionName()); + if (vecFns == null) { + vecFns = Lists.newArrayList(); + vectorizedFunctions.put(fn.functionName(), vecFns); + } + ScalarFunction scalarFunction = (ScalarFunction)fn; + vecFns.add(ScalarFunction.createVecBuiltin(scalarFunction.functionName(), scalarFunction.getSymbolName(), + Lists.newArrayList(scalarFunction.getArgs()), scalarFunction.hasVarArgs(), + scalarFunction.getReturnType(), scalarFunction.isUserVisible(), scalarFunction.getNullableMode())); + } + + public static final String COUNT = "count"; // Populate all the aggregate builtins in the catalog. // null symbols indicate the function does not need that step of the evaluation. @@ -1020,12 +1121,21 @@ private void initAggregateBuiltins() { // count(*) addBuiltin(AggregateFunction.createBuiltin(FunctionSet.COUNT, new ArrayList(), Type.BIGINT, Type.BIGINT, - prefix + "9init_zeroIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", + prefix + "18init_zero_not_nullIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", prefix + "17count_star_updateEPN9doris_udf15FunctionContextEPNS1_9BigIntValE", prefix + "11count_mergeEPN9doris_udf15FunctionContextERKNS1_9BigIntValEPS4_", null, null, prefix + "17count_star_removeEPN9doris_udf15FunctionContextEPNS1_9BigIntValE", null, false, true, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin(FunctionSet.COUNT, + new ArrayList(), Type.BIGINT, Type.BIGINT, + prefix + "18init_zero_not_nullIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", + prefix + "17count_star_updateEPN9doris_udf15FunctionContextEPNS1_9BigIntValE", + prefix + "11count_mergeEPN9doris_udf15FunctionContextERKNS1_9BigIntValEPS4_", + null, null, + prefix + "17count_star_removeEPN9doris_udf15FunctionContextEPNS1_9BigIntValE", + null, false, true, true, true)); for (Type t : Type.getSupportedTypes()) { if (t.isNull()) { @@ -1037,12 +1147,21 @@ private void initAggregateBuiltins() { // Count addBuiltin(AggregateFunction.createBuiltin(FunctionSet.COUNT, Lists.newArrayList(t), Type.BIGINT, Type.BIGINT, - prefix + "9init_zeroIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", + prefix + "18init_zero_not_nullIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", prefix + "12count_updateEPN9doris_udf15FunctionContextERKNS1_6AnyValEPNS1_9BigIntValE", prefix + "11count_mergeEPN9doris_udf15FunctionContextERKNS1_9BigIntValEPS4_", null, null, prefix + "12count_removeEPN9doris_udf15FunctionContextERKNS1_6AnyValEPNS1_9BigIntValE", null, false, true, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin(FunctionSet.COUNT, + Lists.newArrayList(t), Type.BIGINT, Type.BIGINT, + prefix + "18init_zero_not_nullIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", + prefix + "12count_updateEPN9doris_udf15FunctionContextERKNS1_6AnyValEPNS1_9BigIntValE", + prefix + "11count_mergeEPN9doris_udf15FunctionContextERKNS1_9BigIntValEPS4_", + null, null, + prefix + "12count_removeEPN9doris_udf15FunctionContextERKNS1_6AnyValEPNS1_9BigIntValE", + null, false, true, true, true)); // count in multi distinct if (t.equals(Type.CHAR) || t.equals(Type.VARCHAR)) { @@ -1057,6 +1176,18 @@ private void initAggregateBuiltins() { null, prefix + "30count_distinct_string_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), + Type.BIGINT, + Type.VARCHAR, + prefix + "", + prefix + "", + prefix + "", + prefix + "", + null, + null, + prefix + "", + false, true, true, true)); } else if (t.equals(Type.TINYINT) || t.equals(Type.SMALLINT) || t.equals(Type.INT) || t.equals(Type.BIGINT) || t.equals(Type.LARGEINT) || t.equals(Type.DOUBLE)) { addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), @@ -1070,6 +1201,20 @@ private void initAggregateBuiltins() { null, prefix + MULTI_DISTINCT_COUNT_FINALIZE_SYMBOL.get(t), false, true, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), + Type.BIGINT, + t, + prefix + MULTI_DISTINCT_INIT_SYMBOL.get(t), + prefix + MULTI_DISTINCT_UPDATE_SYMBOL.get(t), + prefix + MULTI_DISTINCT_MERGE_SYMBOL.get(t), + prefix + MULTI_DISTINCT_SERIALIZE_SYMBOL.get(t), + null, + null, + prefix + MULTI_DISTINCT_COUNT_FINALIZE_SYMBOL.get(t), + false, true, true, true)); + + } else if (t.equals(Type.DATE) || t.equals(Type.DATETIME)) { addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), Type.BIGINT, @@ -1082,6 +1227,8 @@ private void initAggregateBuiltins() { null, prefix + "28count_distinct_date_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, true)); + // vectorized + // now we don't support datetime distinct } else if (t.equals(Type.DECIMALV2)) { addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), Type.BIGINT, @@ -1094,6 +1241,18 @@ private void initAggregateBuiltins() { null, prefix + "33count_distinct_decimalv2_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), + Type.BIGINT, + Type.DECIMALV2, + prefix + "", + prefix + "", + prefix + "", + prefix + "", + null, + null, + prefix + "", + false, true, true, true)); } // sum in multi distinct @@ -1109,7 +1268,20 @@ private void initAggregateBuiltins() { null, prefix + MULTI_DISTINCT_SUM_FINALIZE_SYMBOL.get(t), false, true, true)); - } else if (t.equals(Type.DECIMALV2)) { + + // vectorized + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_sum", Lists.newArrayList(t), + t, + Type.BIGINT, + prefix + MULTI_DISTINCT_INIT_SYMBOL.get(t), + prefix + MULTI_DISTINCT_UPDATE_SYMBOL.get(t), + prefix + MULTI_DISTINCT_MERGE_SYMBOL.get(t), + prefix + MULTI_DISTINCT_SERIALIZE_SYMBOL.get(t), + null, + null, + prefix + MULTI_DISTINCT_SUM_FINALIZE_SYMBOL.get(t), + false, true, true, true)); + } else if (t.equals(Type.DECIMALV2)) { addBuiltin(AggregateFunction.createBuiltin("multi_distinct_sum", Lists.newArrayList(t), MULTI_DISTINCT_SUM_RETURN_TYPE.get(t), Type.VARCHAR, @@ -1121,25 +1293,50 @@ private void initAggregateBuiltins() { null, prefix + "31sum_distinct_decimalv2_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_sum", Lists.newArrayList(t), + MULTI_DISTINCT_SUM_RETURN_TYPE.get(t), + Type.DECIMALV2, + prefix + "", + prefix + "", + prefix + "", + prefix + "", + null, + null, + prefix + "", + false, true, true, true)); } // Min - String minMaxInit = t.isStringType() ? initNullString : initNull; String minMaxSerializeOrFinalize = t.isStringType() ? stringValSerializeOrFinalize : null; String minMaxGetValue = t.isStringType() ? stringValGetValue : null; addBuiltin(AggregateFunction.createBuiltin("min", - Lists.newArrayList(t), t, t, minMaxInit, + Lists.newArrayList(t), t, t, prefix + MIN_INIT_SYMBOL.get(t), prefix + MIN_UPDATE_SYMBOL.get(t), prefix + MIN_UPDATE_SYMBOL.get(t), minMaxSerializeOrFinalize, minMaxGetValue, null, minMaxSerializeOrFinalize, true, true, false)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin("min", + Lists.newArrayList(t), t, t, prefix + MIN_INIT_SYMBOL.get(t), + prefix + MIN_UPDATE_SYMBOL.get(t), + prefix + MIN_UPDATE_SYMBOL.get(t), + minMaxSerializeOrFinalize, minMaxGetValue, + null, minMaxSerializeOrFinalize, true, true, false, true)); // Max addBuiltin(AggregateFunction.createBuiltin("max", - Lists.newArrayList(t), t, t, minMaxInit, + Lists.newArrayList(t), t, t, prefix + MAX_INIT_SYMBOL.get(t), prefix + MAX_UPDATE_SYMBOL.get(t), prefix + MAX_UPDATE_SYMBOL.get(t), minMaxSerializeOrFinalize, minMaxGetValue, null, minMaxSerializeOrFinalize, true, true, false)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin("max", + Lists.newArrayList(t), t, t, prefix + MAX_INIT_SYMBOL.get(t), + prefix + MAX_UPDATE_SYMBOL.get(t), + prefix + MAX_UPDATE_SYMBOL.get(t), + minMaxSerializeOrFinalize, minMaxGetValue, + null, minMaxSerializeOrFinalize, true, true, false, true)); // NDV // ndv return string @@ -1172,6 +1369,16 @@ private void initAggregateBuiltins() { "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", "_ZN5doris15BitmapFunctions15bitmap_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, false, true)); + // vectorized + addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION_INT, + Lists.newArrayList(t), Type.BIGINT, t, + "", + BITMAP_UNION_INT_SYMBOL.get(t), + "", + "", + "", + true, false, true, true)); + // INTERSECT_COUNT addBuiltin(AggregateFunction.createBuiltin(INTERSECT_COUNT, @@ -1197,6 +1404,18 @@ private void initAggregateBuiltins() { "_ZN5doris12HllFunctions12hll_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, true, true)); + // HLL_UNION_AGG vectorized + addBuiltin(AggregateFunction.createBuiltin("hll_union_agg", + Lists.newArrayList(t), Type.BIGINT, Type.VARCHAR, + "_ZN5doris12HllFunctions8hll_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + HLL_UNION_AGG_UPDATE_SYMBOL.get(t), + "_ZN5doris12HllFunctions9hll_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + "_ZN5doris12HllFunctions13hll_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE", + null, + "_ZN5doris12HllFunctions12hll_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + true, true, true, true)); + // HLL_UNION addBuiltin(AggregateFunction.createBuiltin(HLL_UNION, Lists.newArrayList(t), Type.HLL, Type.HLL, @@ -1207,6 +1426,16 @@ private void initAggregateBuiltins() { "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, false, true)); + // HLL_UNION vectorized + addBuiltin(AggregateFunction.createBuiltin(HLL_UNION, + Lists.newArrayList(t), Type.HLL, Type.HLL, + "_ZN5doris12HllFunctions8hll_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + "_ZN5doris12HllFunctions9hll_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris12HllFunctions9hll_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + true, false, true, true)); + // HLL_RAW_AGG is alias of HLL_UNION addBuiltin(AggregateFunction.createBuiltin("hll_raw_agg", Lists.newArrayList(t), Type.HLL, Type.HLL, @@ -1217,6 +1446,16 @@ private void initAggregateBuiltins() { "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, false, true)); + // HLL_RAW_AGG is alias of HLL_UNION vectorized + addBuiltin(AggregateFunction.createBuiltin("hll_raw_agg", + Lists.newArrayList(t), Type.HLL, Type.HLL, + "_ZN5doris12HllFunctions8hll_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + "_ZN5doris12HllFunctions9hll_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris12HllFunctions9hll_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + "_ZN5doris12HllFunctions13hll_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + true, false, true, true)); + // TopN if (TOPN_UPDATE_SYMBOL.containsKey(t)) { addBuiltin(AggregateFunction.createBuiltin("topn", @@ -1310,33 +1549,84 @@ private void initAggregateBuiltins() { String []sumNames = {"sum", "sum_distinct"}; for (String name : sumNames) { addBuiltin(AggregateFunction.createBuiltin(name, - Lists.newArrayList(Type.BIGINT), Type.BIGINT, Type.BIGINT, initNull, + Lists.newArrayList(Type.BIGINT), Type.BIGINT, Type.BIGINT, prefix + "14init_zero_nullIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, null, prefix + "10sum_removeIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, false, true, false)); addBuiltin(AggregateFunction.createBuiltin(name, - Lists.newArrayList(Type.DOUBLE), Type.DOUBLE, Type.DOUBLE, initNull, + Lists.newArrayList(Type.DOUBLE), Type.DOUBLE, Type.DOUBLE, prefix + "14init_zero_nullIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextEPT_", prefix + "3sumIN9doris_udf9DoubleValES3_EEvPNS2_15FunctionContextERKT_PT0_", prefix + "3sumIN9doris_udf9DoubleValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, null, prefix + "10sum_removeIN9doris_udf9DoubleValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, false, true, false)); addBuiltin(AggregateFunction.createBuiltin(name, - Lists.newArrayList(Type.DECIMALV2), Type.DECIMALV2, Type.DECIMALV2, initNull, + Lists.newArrayList(Type.DECIMALV2), Type.DECIMALV2, Type.DECIMALV2, prefix + "14init_zero_nullIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionContextEPT_", prefix + "3sumIN9doris_udf12DecimalV2ValES3_EEvPNS2_15FunctionContextERKT_PT0_", prefix + "3sumIN9doris_udf12DecimalV2ValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, null, prefix + "10sum_removeIN9doris_udf12DecimalV2ValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, false, true, false)); addBuiltin(AggregateFunction.createBuiltin(name, - Lists.newArrayList(Type.LARGEINT), Type.LARGEINT, Type.LARGEINT, initNull, + Lists.newArrayList(Type.LARGEINT), Type.LARGEINT, Type.LARGEINT, prefix + "14init_zero_nullIN9doris_udf11LargeIntValEEEvPNS2_15FunctionContextEPT_", prefix + "3sumIN9doris_udf11LargeIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", prefix + "3sumIN9doris_udf11LargeIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, null, prefix + "10sum_removeIN9doris_udf11LargeIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", null, false, true, false)); + + // vectorized + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.TINYINT), Type.BIGINT, Type.BIGINT, initNull, + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.SMALLINT), Type.BIGINT, Type.BIGINT, initNull, + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.INT), Type.BIGINT, Type.BIGINT, initNull, + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.BIGINT), Type.BIGINT, Type.BIGINT, initNull, + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf9BigIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.DOUBLE), Type.DOUBLE, Type.DOUBLE, initNull, + prefix + "3sumIN9doris_udf9DoubleValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf9DoubleValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf9DoubleValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.DECIMALV2), Type.DECIMALV2, Type.DECIMALV2, initNull, + prefix + "3sumIN9doris_udf12DecimalV2ValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf12DecimalV2ValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf12DecimalV2ValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(name, + Lists.newArrayList(Type.LARGEINT), Type.LARGEINT, Type.LARGEINT, initNull, + prefix + "3sumIN9doris_udf11LargeIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + prefix + "3sumIN9doris_udf11LargeIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, null, + prefix + "10sum_removeIN9doris_udf11LargeIntValES3_EEvPNS2_15FunctionContextERKT_PT0_", + null, false, true, false, true)); } // bitmap @@ -1350,6 +1640,17 @@ private void initAggregateBuiltins() { "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION, Lists.newArrayList(Type.BITMAP), + Type.BITMAP, + Type.BITMAP, + "", + "", + "", + "", + "", + true, false, true, true)); + + addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION_COUNT, Lists.newArrayList(Type.BITMAP), Type.BIGINT, Type.VARCHAR, @@ -1361,6 +1662,19 @@ private void initAggregateBuiltins() { null, "_ZN5doris15BitmapFunctions15bitmap_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, true, true)); + + addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION_COUNT, Lists.newArrayList(Type.BITMAP), + Type.BIGINT, + Type.BITMAP, + "", + "", + "", + "", + "", + null, + "", + true, true, true, true)); + // TODO(ml): supply function symbol addBuiltin(AggregateFunction.createBuiltin(BITMAP_INTERSECT, Lists.newArrayList(Type.BITMAP), Type.BITMAP, Type.VARCHAR, @@ -1371,6 +1685,15 @@ private void initAggregateBuiltins() { "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(BITMAP_INTERSECT, Lists.newArrayList(Type.BITMAP), + Type.BITMAP, Type.BITMAP, + "", + "", + "", + "", + "", + true, false, true, true)); + //PercentileApprox addBuiltin(AggregateFunction.createBuiltin("percentile_approx", Lists.newArrayList(Type.DOUBLE, Type.DOUBLE), Type.DOUBLE, Type.VARCHAR, @@ -1411,6 +1734,7 @@ private void initAggregateBuiltins() { prefix + "10avg_removeIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE", prefix + "12avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, false)); + addBuiltin(AggregateFunction.createBuiltin("avg", Lists.newArrayList(Type.DECIMALV2), Type.DECIMALV2, Type.VARCHAR, prefix + "18decimalv2_avg_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", @@ -1421,6 +1745,7 @@ private void initAggregateBuiltins() { prefix + "20decimalv2_avg_removeEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValEPNS1_9StringValE", prefix + "22decimalv2_avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, false)); + // Avg(Timestamp) addBuiltin(AggregateFunction.createBuiltin("avg", Lists.newArrayList(Type.DATE), Type.DATE, Type.VARCHAR, @@ -1442,6 +1767,47 @@ private void initAggregateBuiltins() { prefix + "20timestamp_avg_removeEPN9doris_udf15FunctionContextERKNS1_11DateTimeValEPNS1_9StringValE", prefix + "22timestamp_avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, false)); + // vectorized avg + addBuiltin(AggregateFunction.createBuiltin("avg", + Lists.newArrayList(Type.BIGINT), Type.DOUBLE, Type.BIGINT, + prefix + "8avg_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + prefix + "10avg_updateIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE", + prefix + "9avg_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + stringValSerializeOrFinalize, + prefix + "13avg_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE", + prefix + "10avg_removeIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE", + prefix + "12avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin("avg", + Lists.newArrayList(Type.DOUBLE), Type.DOUBLE, Type.DOUBLE, + prefix + "8avg_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + prefix + "10avg_updateIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE", + prefix + "9avg_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + stringValSerializeOrFinalize, + prefix + "13avg_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE", + prefix + "10avg_removeIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE", + prefix + "12avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin("avg", + Lists.newArrayList(Type.DECIMALV2), Type.DECIMALV2, Type.DECIMALV2, + prefix + "18decimalv2_avg_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + prefix + "20decimalv2_avg_updateEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValEPNS1_9StringValE", + prefix + "19decimalv2_avg_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + prefix + "23decimalv2_avg_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + prefix + "23decimalv2_avg_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE", + prefix + "20decimalv2_avg_removeEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValEPNS1_9StringValE", + prefix + "22decimalv2_avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin("avg", + Lists.newArrayList(Type.DATE), Type.DATE, Type.DATE, + prefix + "8avg_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + prefix + "20timestamp_avg_updateEPN9doris_udf15FunctionContextERKNS1_11DateTimeValEPNS1_9StringValE", + prefix + "9avg_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + stringValSerializeOrFinalize, + prefix + "23timestamp_avg_get_valueEPN9doris_udf15FunctionContextERKNS1_9StringValE", + prefix + "20timestamp_avg_removeEPN9doris_udf15FunctionContextERKNS1_11DateTimeValEPNS1_9StringValE", + prefix + "22timestamp_avg_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + false, true, false, true)); // Group_concat(string) addBuiltin(AggregateFunction.createBuiltin("group_concat", Lists.newArrayList(Type.VARCHAR), Type.VARCHAR, Type.VARCHAR, initNullString, @@ -1479,7 +1845,7 @@ private void initAggregateBuiltins() { prefix + "13rank_finalizeEPN9doris_udf15FunctionContextERNS1_9StringValE")); addBuiltin(AggregateFunction.createAnalyticBuiltin( "row_number", new ArrayList(), Type.BIGINT, Type.BIGINT, - prefix + "9init_zeroIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", + prefix + "18init_zero_not_nullIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", prefix + "17count_star_updateEPN9doris_udf15FunctionContextEPNS1_9BigIntValE", prefix + "11count_mergeEPN9doris_udf15FunctionContextERKNS1_9BigIntValEPS4_", null, null)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java index b00febfc0605bc..3f60a594abb1a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java @@ -17,12 +17,6 @@ package org.apache.doris.catalog; -import static org.apache.doris.common.io.IOUtils.writeOptionString; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.gson.Gson; import org.apache.doris.analysis.CreateFunctionStmt; import org.apache.doris.analysis.FunctionName; import org.apache.doris.analysis.HdfsURI; @@ -30,6 +24,11 @@ import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TFunctionBinaryType; import org.apache.doris.thrift.TScalarFunction; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -37,9 +36,14 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; +import com.google.gson.Gson; + +import static org.apache.doris.common.io.IOUtils.writeOptionString; + // import org.apache.doris.thrift.TSymbolType; /** @@ -57,14 +61,19 @@ public class ScalarFunction extends Function { protected ScalarFunction() { } - public ScalarFunction( - FunctionName fnName, Type[] argTypes, Type retType, boolean hasVarArgs) { - super(fnName, argTypes, retType, hasVarArgs); + public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, + boolean userVisible) { + this(fnName, argTypes, retType, hasVarArgs, TFunctionBinaryType.BUILTIN, userVisible, false); } - public ScalarFunction( - FunctionName fnName, ArrayList argTypes, Type retType, boolean hasVarArgs) { - super(fnName, argTypes, retType, hasVarArgs); + public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, + boolean userVisible, boolean isVec) { + this(fnName, argTypes, retType, hasVarArgs, TFunctionBinaryType.BUILTIN, userVisible, isVec); + } + + public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, + TFunctionBinaryType binaryType, boolean userVisible, boolean isVec) { + super(0, fnName, argTypes, retType, hasVarArgs, binaryType, userVisible, isVec, NullableMode.DEPEND_ON_ARGUMENT); } public ScalarFunction(FunctionName fnName, List argTypes, @@ -81,48 +90,33 @@ public ScalarFunction(FunctionName fnName, List argTypes, * Creates a builtin scalar function. This is a helper that wraps a few steps * into one call. */ + public static ScalarFunction createBuiltin(String name, Type retType, + ArrayList argTypes, boolean hasVarArgs, + String symbol, String prepareFnSymbol, String closeFnSymbol, + boolean userVisible) { + return createBuiltin(name, retType, NullableMode.DEPEND_ON_ARGUMENT, argTypes, hasVarArgs, + symbol, prepareFnSymbol, closeFnSymbol, userVisible); + } + public static ScalarFunction createBuiltin( - String name, ArrayList argTypes, - boolean hasVarArgs, Type retType, String symbol, - String prepareFnSymbol, String closeFnSymbol, boolean userVisible) { + String name, Type retType, NullableMode nullableMode, + ArrayList argTypes, boolean hasVarArgs, + String symbol, String prepareFnSymbol, String closeFnSymbol, boolean userVisible) { Preconditions.checkNotNull(symbol); ScalarFunction fn = new ScalarFunction( - new FunctionName(name), argTypes, retType, hasVarArgs); - fn.setBinaryType(TFunctionBinaryType.BUILTIN); - fn.setUserVisible(userVisible); + new FunctionName(name), argTypes, retType, hasVarArgs, userVisible); fn.symbolName = symbol; fn.prepareFnSymbol = prepareFnSymbol; fn.closeFnSymbol = closeFnSymbol; - -///* try { -// fn.symbolName = fn.lookupSymbol(symbol, TSymbolType.UDF_EVALUATE, null, -// fn.hasVarArgs(), fn.getArgs()); -// } catch (AnalysisException e) { -// // This should never happen -// throw new RuntimeException("Builtin symbol '" + symbol + "'" + argTypes -// + " not found!", e); -// } -// if (prepareFnSymbol != null) { -// try { -// fn.prepareFnSymbol = fn.lookupSymbol(prepareFnSymbol, TSymbolType.UDF_PREPARE); -// } catch (AnalysisException e) { -// // This should never happen -// throw new RuntimeException( -// "Builtin symbol '" + prepareFnSymbol + "' not found!", e); -// } -// } -// if (closeFnSymbol != null) { -// try { -// fn.closeFnSymbol = fn.lookupSymbol(closeFnSymbol, TSymbolType.UDF_CLOSE); -// } catch (AnalysisException e) { -// // This should never happen -// throw new RuntimeException( -// "Builtin symbol '" + closeFnSymbol + "' not found!", e); -// } -// }*/ + fn.nullableMode = nullableMode; return fn; } + public static ScalarFunction createBuiltinOperator( + String name, ArrayList argTypes, Type retType) { + return createBuiltinOperator(name, argTypes, retType, NullableMode.DEPEND_ON_ARGUMENT); + } + /** * Creates a builtin scalar operator function. This is a helper that wraps a few steps * into one call. @@ -130,7 +124,7 @@ public static ScalarFunction createBuiltin( * implementations. (gen_functions.py). Is there a better way to coordinate this. */ public static ScalarFunction createBuiltinOperator( - String name, ArrayList argTypes, Type retType) { + String name, ArrayList argTypes, Type retType, NullableMode nullableMode) { // Operators have a well defined symbol based on the function name and type. // Convert Add(TINYINT, TINYINT) --> Add_TinyIntVal_TinyIntVal String beFn = name; @@ -184,22 +178,96 @@ public static ScalarFunction createBuiltinOperator( String beClass = usesDecimal ? "DecimalOperators" : "Operators"; if (usesDecimalV2) beClass = "DecimalV2Operators"; String symbol = "doris::" + beClass + "::" + beFn; - return createBuiltinOperator(name, symbol, argTypes, retType); + return createBuiltinOperator(name, symbol, argTypes, retType, nullableMode); + } + + public static ScalarFunction createVecBuiltinOperator( + String name, ArrayList argTypes, Type retType) { + return createVecBuiltinOperator(name, argTypes, retType, NullableMode.DEPEND_ON_ARGUMENT); + } + + /** + * Creates a builtin scala vec operator function. This is a helper that wraps a few steps + * into one call. + * TODO: this needs to be kept in sync with what generates the be operator + * implementations. (gen_functions.py). Is there a better way to coordinate this. + */ + public static ScalarFunction createVecBuiltinOperator( + String name, ArrayList argTypes, Type retType, NullableMode nullableMode) { + StringBuilder beFn = new StringBuilder(name); + boolean usesDecimal = false; + boolean usesDecimalV2 = false; + + // just mock a fake symbol for vec function, we treat + // all argument is same as first argument + for (int i = 0; i < argTypes.size(); ++i) { + switch (argTypes.get(0).getPrimitiveType()) { + case BOOLEAN: + beFn.append("_boolean_val"); + break; + case TINYINT: + beFn.append("_tiny_int_val"); + break; + case SMALLINT: + beFn.append("_small_int_val"); + break; + case INT: + beFn.append("_int_val"); + break; + case BIGINT: + beFn.append("_big_int_val"); + break; + case LARGEINT: + beFn.append("_large_int_val"); + break; + case FLOAT: + beFn.append("_float_val"); + break; + case DOUBLE: + case TIME: + beFn.append("_double_val"); + break; + case CHAR: + case VARCHAR: + case HLL: + case BITMAP: + beFn.append("_string_val"); + break; + case DATE: + case DATETIME: + beFn.append("_datetime_val"); + break; + case DECIMALV2: + beFn.append("_decimalv2_val"); + usesDecimalV2 = true; + break; + default: + Preconditions.checkState(false, "Argument type not supported: " + argTypes.get(i)); + } + } + String beClass = usesDecimal ? "DecimalOperators" : "Operators"; + if (usesDecimalV2) beClass = "DecimalV2Operators"; + String symbol = "doris::" + beClass + "::" + beFn; + return createVecBuiltinOperator(name, symbol, argTypes, retType, nullableMode); } public static ScalarFunction createBuiltinOperator( String name, String symbol, ArrayList argTypes, Type retType) { - return createBuiltin(name, symbol, argTypes, false, retType, false); + return createBuiltinOperator(name, symbol, argTypes, retType, NullableMode.DEPEND_ON_ARGUMENT); + } + + public static ScalarFunction createBuiltinOperator( + String name, String symbol, ArrayList argTypes, Type retType, NullableMode nullableMode) { + return createBuiltin(name, symbol, argTypes, false, retType, false, nullableMode); } public static ScalarFunction createBuiltin( String name, String symbol, ArrayList argTypes, - boolean hasVarArgs, Type retType, boolean userVisible) { + boolean hasVarArgs, Type retType, boolean userVisible, NullableMode nullableMode) { ScalarFunction fn = new ScalarFunction( - new FunctionName(name), argTypes, retType, hasVarArgs); - fn.setBinaryType(TFunctionBinaryType.BUILTIN); - fn.setUserVisible(userVisible); + new FunctionName(name), argTypes, retType, hasVarArgs, userVisible); fn.symbolName = symbol; + fn.nullableMode = nullableMode; // try { // fn.symbolName_ = fn.lookupSymbol(symbol, TSymbolType.UDF_EVALUATE, null, @@ -213,6 +281,22 @@ public static ScalarFunction createBuiltin( return fn; } + public static ScalarFunction createVecBuiltinOperator( + String name, String symbol, ArrayList argTypes, Type retType, NullableMode nullableMode) { + return createVecBuiltin(name, symbol, argTypes, false, retType, false, nullableMode); + } + + //TODO: This method should not be here, move to other place in the future + public static ScalarFunction createVecBuiltin( + String name, String symbol, ArrayList argTypes, + boolean hasVarArgs, Type retType, boolean userVisible, NullableMode nullableMode) { + ScalarFunction fn = new ScalarFunction( + new FunctionName(name), argTypes, retType, hasVarArgs, userVisible, true); + fn.symbolName = symbol; + fn.nullableMode = nullableMode; + return fn; + } + /** * Create a function that is used to search the catalog for a matching builtin. Only * the fields necessary for matching function prototypes are specified. @@ -222,8 +306,7 @@ public static ScalarFunction createBuiltinSearchDesc( ArrayList fnArgs = (argTypes == null) ? new ArrayList() : Lists.newArrayList(argTypes); ScalarFunction fn = new ScalarFunction( - new FunctionName(name), fnArgs, Type.INVALID, hasVarArgs); - fn.setBinaryType(TFunctionBinaryType.BUILTIN); + new FunctionName(name), fnArgs, Type.INVALID, hasVarArgs, true); return fn; } @@ -231,9 +314,8 @@ public static ScalarFunction createUdf( FunctionName name, Type[] args, Type returnType, boolean isVariadic, String objectFile, String symbol, String prepareFnSymbol, String closeFnSymbol) { - ScalarFunction fn = new ScalarFunction(name, args, returnType, isVariadic); - fn.setBinaryType(TFunctionBinaryType.NATIVE); - fn.setUserVisible(true); + ScalarFunction fn = new ScalarFunction(name, Arrays.asList(args), returnType, isVariadic, + TFunctionBinaryType.NATIVE, true, false); fn.symbolName = symbol; fn.prepareFnSymbol = prepareFnSymbol; fn.closeFnSymbol = closeFnSymbol; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java index eea7574b8e651c..d29098647f0b72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java @@ -53,6 +53,8 @@ public class ProfileTreeBuilder { private static final String PROFILE_NAME_BUFFER_POOL = "Buffer pool"; private static final String PROFILE_NAME_EXCHANGE_NODE = "EXCHANGE_NODE"; public static final String FINAL_SENDER_ID = "-1"; + private static final String PROFILE_NAME_VEXCHANGE_NODE = "VEXCHANGE_NODE"; + public static final String DATA_BUFFER_SENDER_ID = "-1"; public static final String UNKNOWN_ID = "-2"; private RuntimeProfile profileRoot; @@ -253,7 +255,8 @@ private ProfileTreeNode buildTreeNode(RuntimeProfile profile, ProfileTreeNode ro node.setParentNode(root); } - if (node.name.equals(PROFILE_NAME_EXCHANGE_NODE) && instanceId == null) { + if ((node.name.equals(PROFILE_NAME_EXCHANGE_NODE) || + node.name.equals(PROFILE_NAME_VEXCHANGE_NODE)) && instanceId == null) { exchangeNodes.add(node); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java new file mode 100644 index 00000000000000..349cf1e2360b7f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/VectorizedUtil.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.qe.ConnectContext; + +public class VectorizedUtil { + public static boolean isVectorized() { + if (ConnectContext.get() == null) { + return false; + } + return ConnectContext.get().getSessionVariable().enableVectorizedEngine(); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index 667e34e4acec58..4b94a96bdb4614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -263,7 +263,6 @@ protected String debugString() { @Override protected void toThrift(TPlanNode msg) { msg.node_type = TPlanNodeType.AGGREGATION_NODE; - List aggregateFunctions = Lists.newArrayList(); // only serialize agg exprs that are being materialized for (FunctionCallExpr e: aggInfo.getMaterializedAggregateExprs()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index e7b82c025919aa..8b30ad4fa48409 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -18,16 +18,22 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprId; import org.apache.doris.analysis.ExprSubstitutionMap; +import org.apache.doris.analysis.FunctionName; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.TreeNode; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TFunctionBinaryType; import org.apache.doris.thrift.TPlan; import org.apache.doris.thrift.TPlanNode; @@ -85,6 +91,8 @@ abstract public class PlanNode extends TreeNode { protected List conjuncts = Lists.newArrayList(); + protected Expr vconjunct = null; + // Conjuncts used to filter the original load file. // In the load execution plan, the difference between "preFilterConjuncts" and "conjuncts" is that // conjuncts are used to filter the data after column conversion and mapping, @@ -128,7 +136,8 @@ protected PlanNode(PlanNodeId id, ArrayList tupleIds, String planNodeNa this.tupleIds = Lists.newArrayList(tupleIds); this.tblRefIds = Lists.newArrayList(tupleIds); this.cardinality = -1; - this.planNodeName = planNodeName; + this.planNodeName = VectorizedUtil.isVectorized() ? + "V" + planNodeName : planNodeName; this.numInstances = 1; } @@ -138,7 +147,8 @@ protected PlanNode(PlanNodeId id, String planNodeName) { this.tupleIds = Lists.newArrayList(); this.tblRefIds = Lists.newArrayList(); this.cardinality = -1; - this.planNodeName = planNodeName; + this.planNodeName = VectorizedUtil.isVectorized() ? + "V" + planNodeName : planNodeName; this.numInstances = 1; } @@ -154,7 +164,8 @@ protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName) { this.conjuncts = Expr.cloneList(node.conjuncts, null); this.cardinality = -1; this.compactData = node.compactData; - this.planNodeName = planNodeName; + this.planNodeName = VectorizedUtil.isVectorized() ? + "V" + planNodeName : planNodeName; this.numInstances = 1; } @@ -286,6 +297,39 @@ public List getConjuncts() { return conjuncts; } + private void initCompoundPredicate(Expr expr) { + if (expr instanceof CompoundPredicate) { + CompoundPredicate compoundPredicate = (CompoundPredicate) expr; + compoundPredicate.setType(Type.BOOLEAN); + List args = new ArrayList<>(); + args.add(Type.BOOLEAN); + args.add(Type.BOOLEAN); + Function function = new Function(new FunctionName("", compoundPredicate.getOp().toString()), args, Type.BOOLEAN, false); + function.setBinaryType(TFunctionBinaryType.BUILTIN); + expr.setFn(function); + } + + for (Expr child : expr.getChildren()) { + initCompoundPredicate(child); + } + } + + private Expr convertConjunctsToAndCompoundPredicate() { + List targetConjuncts = Lists.newArrayList(conjuncts); + while (targetConjuncts.size() > 1) { + List newTargetConjuncts = Lists.newArrayList(); + for (int i = 0; i < targetConjuncts.size(); i+= 2) { + Expr expr = i + 1 < targetConjuncts.size() ? new CompoundPredicate(CompoundPredicate.Operator.AND, targetConjuncts.get(i), + targetConjuncts.get(i + 1)) : targetConjuncts.get(i); + newTargetConjuncts.add(expr); + } + targetConjuncts = newTargetConjuncts; + } + + Preconditions.checkArgument(targetConjuncts.size() == 1); + return targetConjuncts.get(0); + } + public void addConjuncts(List conjuncts) { if (conjuncts == null) { return; @@ -429,10 +473,16 @@ private void treeToThriftHelper(TPlan container) { for (Expr e : conjuncts) { msg.addToConjuncts(e.treeToThrift()); } + // Serialize any runtime filters for (RuntimeFilter filter : runtimeFilters) { msg.addToRuntimeFilters(filter.toThrift()); } + + if (vconjunct != null) { + msg.vconjunct = vconjunct.treeToThrift(); + } + msg.compact_data = compactData; toThrift(msg); container.addToNodes(msg); @@ -802,4 +852,15 @@ public String toString() { sb.append("\n").append(getNodeExplainString("", TExplainLevel.BRIEF)); return sb.toString(); } + + void convertToVectoriezd() { + if (!conjuncts.isEmpty()) { + vconjunct = convertConjunctsToAndCompoundPredicate(); + initCompoundPredicate(vconjunct); + } + + for (PlanNode child : children) { + child.convertToVectoriezd(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index bb2efa4e7d1eb1..30303516b80b68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -28,6 +28,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.common.UserException; import org.apache.doris.common.profile.PlanTreeBuilder; import org.apache.doris.common.profile.PlanTreePrinter; @@ -167,6 +168,10 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue singleNodePlanner = new SingleNodePlanner(plannerContext); PlanNode singleNodePlan = singleNodePlanner.createSingleNodePlan(); + if (VectorizedUtil.isVectorized()) { + singleNodePlan.convertToVectoriezd(); + } + if (statement instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) statement; insertStmt.prepareExpressions(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java index 547dfb3d04f050..ea915830c5890e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ResultSink.java @@ -18,6 +18,7 @@ package org.apache.doris.planner; import org.apache.doris.analysis.OutFileClause; +import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TExplainLevel; @@ -49,7 +50,11 @@ public ResultSink(PlanNodeId exchNodeId, TResultSinkType sinkType) { @Override public String getExplainString(String prefix, TExplainLevel explainLevel) { StringBuilder strBuilder = new StringBuilder(); - strBuilder.append(prefix + "RESULT SINK\n"); + strBuilder.append(prefix); + if (VectorizedUtil.isVectorized()) { + strBuilder.append("V"); + } + strBuilder.append("RESULT SINK\n"); return strBuilder.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 801ba0a0b47e37..f10e67644122f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1670,8 +1670,8 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s switch (tblRef.getTable().getType()) { case OLAP: - OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode"); - + OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), + "OlapScanNode"); olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened()); scanNode = olapNode; break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 9cefa7d111d888..a3095bd615013b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -452,6 +452,7 @@ public void exec() throws Exception { } else { // This is a load process. this.queryOptions.setIsReportSuccess(true); + this.queryOptions.setEnableVectorizedEngine(false); deltaUrls = Lists.newArrayList(); loadCounters = Maps.newHashMap(); List relatedBackendIds = Lists.newArrayList(addressToBackendID.values()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 9818ab13facc6a..78698fb12d8091 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -141,11 +141,14 @@ public class SessionVariable implements Serializable, Writable { public static final String DELETE_WITHOUT_PARTITION = "delete_without_partition"; - public static final String EXTRACT_WIDE_RANGE_EXPR = "extract_wide_range_expr"; - public static final long DEFAULT_INSERT_VISIBLE_TIMEOUT_MS = 10_000; + + public static final String EXTRACT_WIDE_RANGE_EXPR = "extract_wide_range_expr"; + public static final long MIN_INSERT_VISIBLE_TIMEOUT_MS = 1000; // If user set a very small value, use this value instead. + public static final String ENABLE_VECTORIZED_ENGINE = "enable_vectorized_engine"; + // session origin value public Map sessionOriginValue = new HashMap(); // check stmt is or not [select /*+ SET_VAR(...)*/ ...] @@ -355,6 +358,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = RUNTIME_FILTER_MAX_IN_NUM) private int runtimeFilterMaxInNum = 1024; + @VariableMgr.VarAttr(name = ENABLE_VECTORIZED_ENGINE) + public boolean enableVectorizedEngine = false; + public long getMaxExecMemByte() { return maxExecMemByte; } @@ -699,6 +705,14 @@ public void setRuntimeFilterMaxInNum(int runtimeFilterMaxInNum) { this.runtimeFilterMaxInNum = runtimeFilterMaxInNum; } + public boolean enableVectorizedEngine() { + return enableVectorizedEngine; + } + + public void setEnableVectorizedEngine(boolean enableVectorizedEngine) { + this.enableVectorizedEngine = enableVectorizedEngine; + } + public long getInsertVisibleTimeoutMs() { if (insertVisibleTimeoutMs < MIN_INSERT_VISIBLE_TIMEOUT_MS) { return MIN_INSERT_VISIBLE_TIMEOUT_MS; @@ -758,6 +772,7 @@ public TQueryOptions toThrift() { tResult.setQueryTimeout(queryTimeoutS); tResult.setIsReportSuccess(isReportSucc); tResult.setCodegenLevel(codegenLevel); + tResult.setEnableVectorizedEngine(enableVectorizedEngine); tResult.setBatchSize(batchSize); tResult.setDisableStreamPreaggregations(disableStreamPreaggregations); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 4cf5033d1cef56..df4eec80ead046 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -54,6 +54,7 @@ import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.AuditLog; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; @@ -324,7 +325,7 @@ public void execute(TUniqueId queryId) throws Exception { if (i > 0) { UUID uuid = UUID.randomUUID(); TUniqueId newQueryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); - LOG.warn("Query {} {} times with new query id: {}", DebugUtil.printId(queryId), i, DebugUtil.printId(newQueryId)); + AuditLog.getQueryAudit().log("Query {} {} times with new query id: {}", DebugUtil.printId(queryId), i, DebugUtil.printId(newQueryId)); context.setQueryId(newQueryId); } handleQueryStmt(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CaseExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CaseExprTest.java new file mode 100644 index 00000000000000..0d991b77c4e285 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CaseExprTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.doris.analysis; + +import java.util.List; + +import com.clearspring.analytics.util.Lists; +import mockit.Expectations; +import mockit.Injectable; +import org.junit.Assert; +import org.junit.Test; + +public class CaseExprTest { + + @Test + public void testIsNullable(@Injectable Expr caseExpr, + @Injectable Expr whenExpr, + @Injectable Expr thenExpr, + @Injectable Expr elseExpr) { + // without case without else + CaseWhenClause caseWhenClause = new CaseWhenClause(whenExpr, thenExpr); + List caseWhenClauseList = Lists.newArrayList(); + caseWhenClauseList.add(caseWhenClause); + CaseExpr caseExpr1 = new CaseExpr(null, caseWhenClauseList, null); + new Expectations() { + { + thenExpr.isNullable(); + result = false; + } + }; + Assert.assertTrue(caseExpr1.isNullable()); + // with case without else + CaseExpr caseExpr2 = new CaseExpr(caseExpr, caseWhenClauseList, null); + new Expectations() { + { + thenExpr.isNullable(); + result = false; + } + }; + Assert.assertTrue(caseExpr2.isNullable()); + // with case with else + CaseExpr caseExpr3 = new CaseExpr(caseExpr, caseWhenClauseList, elseExpr); + new Expectations() { + { + thenExpr.isNullable(); + result = false; + elseExpr.isNullable(); + result = true; + } + }; + Assert.assertTrue(caseExpr3.isNullable()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionSetTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionSetTest.java index 2301a12e505cde..fbdb75e83b8cfd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionSetTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionSetTest.java @@ -23,6 +23,8 @@ import org.junit.Before; import org.junit.Test; +import java.util.Arrays; + public class FunctionSetTest { private FunctionSet functionSet; @@ -36,18 +38,18 @@ public void setUp() { @Test public void testGetLagFunction() { Type[] argTypes1 = {ScalarType.DECIMALV2, ScalarType.TINYINT, ScalarType.TINYINT}; - Function lagDesc1 = new Function(new FunctionName("lag"), argTypes1, (Type) ScalarType.INVALID, false); + Function lagDesc1 = new Function(new FunctionName("lag"), Arrays.asList(argTypes1), (Type) ScalarType.INVALID, false); Function newFunction = functionSet.getFunction(lagDesc1, Function.CompareMode.IS_SUPERTYPE_OF); Type[] newArgTypes = newFunction.getArgs(); Assert.assertTrue(newArgTypes[0].matchesType(newArgTypes[2])); Assert.assertTrue(newArgTypes[0].matchesType(ScalarType.DECIMALV2)); Type[] argTypes2 = {ScalarType.VARCHAR, ScalarType.TINYINT, ScalarType.TINYINT}; - Function lagDesc2 = new Function(new FunctionName("lag"), argTypes2, (Type) ScalarType.INVALID, false); + Function lagDesc2 = new Function(new FunctionName("lag"), Arrays.asList(argTypes2), (Type) ScalarType.INVALID, false); newFunction = functionSet.getFunction(lagDesc2, Function.CompareMode.IS_SUPERTYPE_OF); newArgTypes = newFunction.getArgs(); Assert.assertTrue(newArgTypes[0].matchesType(newArgTypes[2])); Assert.assertTrue(newArgTypes[0].matchesType(ScalarType.VARCHAR)); } -} \ No newline at end of file +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java index 135a50b46a89e2..b145917588dd77 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java @@ -327,7 +327,7 @@ public void testWithStmtSlotIsAllowNull() throws Exception { Planner planner1 = stmtExecutor1.planner(); List fragments1 = planner1.getFragments(); String plan1 = planner1.getExplainString(fragments1, new ExplainOptions(true, false)); - Assert.assertEquals(3, StringUtils.countMatches(plan1, "nullIndicatorBit=0")); + Assert.assertEquals(2, StringUtils.countMatches(plan1, "nullIndicatorBit=0")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java index a7a07d0d18e1f8..28aafaf631a382 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java @@ -340,7 +340,7 @@ public void testHllColumnsNormal() throws UserException { new Expectations() {{ catalog.getFunction((Function) any, (Function.CompareMode) any); - result = new ScalarFunction(new FunctionName(FunctionSet.HLL_HASH), Lists.newArrayList(), Type.BIGINT, false); + result = new ScalarFunction(new FunctionName(FunctionSet.HLL_HASH), Lists.newArrayList(), Type.BIGINT, false, true); }}; new Expectations() { @@ -390,7 +390,7 @@ public void testHllColumnsNoHllHash() throws UserException { new Expectations() { { catalog.getFunction((Function) any, (Function.CompareMode) any); - result = new ScalarFunction(new FunctionName("hll_hash1"), Lists.newArrayList(), Type.BIGINT, false); + result = new ScalarFunction(new FunctionName("hll_hash1"), Lists.newArrayList(), Type.BIGINT, false, true); minTimes = 0; } }; diff --git a/fe/fe-core/src/test/resources/log4j2.xml b/fe/fe-core/src/test/resources/log4j2.xml index d0e6cbaced4356..7e0957b1a926d0 100644 --- a/fe/fe-core/src/test/resources/log4j2.xml +++ b/fe/fe-core/src/test/resources/log4j2.xml @@ -16,7 +16,7 @@ - + diff --git a/gensrc/proto/data.proto b/gensrc/proto/data.proto index d380031dfbc397..ca62757b777fcc 100644 --- a/gensrc/proto/data.proto +++ b/gensrc/proto/data.proto @@ -33,5 +33,57 @@ message PRowBatch { repeated int32 tuple_offsets = 3; required bytes tuple_data = 4; required bool is_compressed = 5; -}; +} + +message PColumn { + enum DataType { + UINT8 = 0; + UINT16 = 1; + UINT32 = 2; + UINT64 = 3; + UINT128 = 4; + UINT256 = 5; + INT16 = 6; + INT8 = 7; + INT32 = 8; + INT64 = 9; + INT128 = 10; + INT256 = 11; + FLOAT32 = 12; + FLOAT64 = 13; + BOOLEAN = 14; + DATE = 15; + DATETIME = 16; + HLL = 17; + BITMAP = 18; + ARRAY = 19; + MAP = 20; + STRUCT =21; + STRING = 22; + DECIMAL32 = 23; + DECIMAL64 = 24; + DECIMAL128 = 25; + BYTES = 26; + NOTHING = 27; + UNKNOWN = 999; + } + message Decimal { + optional uint32 precision = 1; + optional uint32 scale = 2; + } + required string name = 1; + // origin column data_type + required DataType type = 2; + // serialized data type + repeated bool is_null = 3; + // store in plain text or in binary data + optional bytes binary = 4; + optional bool compressed = 5 [default = false]; + optional Decimal decimal_param = 6; +} +message PBlock { + optional uint64 num_rows = 1; + optional uint32 num_columns = 2; + repeated PColumn columns = 3; +} diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index b792fc0c5d222b..381987e875e61b 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -43,6 +43,8 @@ message PTransmitDataParams { // different per packet required int64 packet_seq = 7; optional PQueryStatistics query_statistics = 8; + + optional PBlock block = 9; }; message PTransmitDataResult { @@ -395,5 +397,6 @@ service PBackendService { rpc merge_filter(PMergeFilterRequest) returns (PMergeFilterResponse); rpc apply_filter(PPublishFilterRequest) returns (PPublishFilterResponse); rpc fold_constant_expr(PConstantExprRequest) returns (PConstantExprResult); + rpc transmit_block(PTransmitDataParams) returns (PTransmitDataResult); }; diff --git a/gensrc/proto/palo_internal_service.proto b/gensrc/proto/palo_internal_service.proto new file mode 100644 index 00000000000000..222fdf72515d6d --- /dev/null +++ b/gensrc/proto/palo_internal_service.proto @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// NOTE(XXX): DEPRECATED, just use to compatiple with old version. +// Make system can grayscale upgrade +syntax="proto2"; + +import "internal_service.proto"; + +package palo; +option java_package = "org.apache.doris.proto"; + +option cc_generic_services = true; + +service PInternalService { + rpc transmit_data(doris.PTransmitDataParams) returns (doris.PTransmitDataResult); + rpc exec_plan_fragment(doris.PExecPlanFragmentRequest) returns (doris.PExecPlanFragmentResult); + rpc cancel_plan_fragment(doris.PCancelPlanFragmentRequest) returns (doris.PCancelPlanFragmentResult); + rpc fetch_data(doris.PFetchDataRequest) returns (doris.PFetchDataResult); + rpc tablet_writer_open(doris.PTabletWriterOpenRequest) returns (doris.PTabletWriterOpenResult); + rpc tablet_writer_add_batch(doris.PTabletWriterAddBatchRequest) returns (doris.PTabletWriterAddBatchResult); + rpc tablet_writer_cancel(doris.PTabletWriterCancelRequest) returns (doris.PTabletWriterCancelResult); + rpc get_info(doris.PProxyRequest) returns (doris.PProxyResult); + rpc update_cache(doris.PUpdateCacheRequest) returns (doris.PCacheResponse); + rpc fetch_cache(doris.PFetchCacheRequest) returns (doris.PFetchCacheResult); + rpc clear_cache(doris.PClearCacheRequest) returns (doris.PCacheResponse); + + rpc merge_filter(doris.PMergeFilterRequest) returns (doris.PMergeFilterResponse); + rpc apply_filter(doris.PPublishFilterRequest) returns (doris.PPublishFilterResponse); + + rpc fold_constant_expr(doris.PConstantExprRequest) returns (doris.PConstantExprResult); + rpc transmit_block(doris.PTransmitDataParams) returns (doris.PTransmitDataResult); +}; diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 7979224b50b349..82536b3fd5bbaf 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -24,11 +24,18 @@ # The format is: # [sql aliases], , [], , # With an optional -# , +# , , , # # 'sql aliases' are the function names that can be used from sql. There must be at least # one per function. # +# 'vec' means that whether the vec engine support this function. There are two values of +# this params: 'vec', ''. If the value is 'vec', it means both normal and vec engine support +# this function. If the value is '', it means only normal engine support this function. +# +# 'nullable mode' reflects whether the return value of the function is null. See @Function.NullableMode +# for the specific mode and meaning. +# # The symbol can be empty for functions that are not yet implemented or are special-cased # in Expr::CreateExpr() (i.e., functions that are implemented via a custom Expr class # rather than a single function). @@ -37,856 +44,1031 @@ # For functions corresponding to builtin operators, we can reuse the implementations [['bitand'], 'TINYINT', ['TINYINT', 'TINYINT'], '_ZN5doris9Operators32bitand_tiny_int_val_tiny_int_valEPN9doris_udf' - '15FunctionContextERKNS1_10TinyIntValES6_'], + '15FunctionContextERKNS1_10TinyIntValES6_', '', '', '', ''], [['bitand'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], '_ZN5doris9Operators34bitand_small_int_val_small_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11SmallIntValES6_'], + '15FunctionContextERKNS1_11SmallIntValES6_', '', '', '', ''], [['bitand'], 'INT', ['INT', 'INT'], '_ZN5doris9Operators22bitand_int_val_int_valEPN9doris_udf' - '15FunctionContextERKNS1_6IntValES6_'], + '15FunctionContextERKNS1_6IntValES6_', '', '', '', ''], [['bitand'], 'BIGINT', ['BIGINT', 'BIGINT'], '_ZN5doris9Operators30bitand_big_int_val_big_int_valEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValES6_'], + '15FunctionContextERKNS1_9BigIntValES6_', '', '', '', ''], [['bitand'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], '_ZN5doris9Operators34bitand_large_int_val_large_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11LargeIntValES6_'], + '15FunctionContextERKNS1_11LargeIntValES6_', '', '', '', ''], [['bitor'], 'TINYINT', ['TINYINT', 'TINYINT'], '_ZN5doris9Operators31bitor_tiny_int_val_tiny_int_valEPN9doris_udf' - '15FunctionContextERKNS1_10TinyIntValES6_'], + '15FunctionContextERKNS1_10TinyIntValES6_', '', '', '', ''], [['bitor'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], '_ZN5doris9Operators33bitor_small_int_val_small_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11SmallIntValES6_'], + '15FunctionContextERKNS1_11SmallIntValES6_', '', '', '', ''], [['bitor'], 'INT', ['INT', 'INT'], '_ZN5doris9Operators21bitor_int_val_int_valEPN9doris_udf' - '15FunctionContextERKNS1_6IntValES6_'], + '15FunctionContextERKNS1_6IntValES6_', '', '', '', ''], [['bitor'], 'BIGINT', ['BIGINT', 'BIGINT'], '_ZN5doris9Operators29bitor_big_int_val_big_int_valEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValES6_'], + '15FunctionContextERKNS1_9BigIntValES6_', '', '', '', ''], [['bitor'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], '_ZN5doris9Operators33bitor_large_int_val_large_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11LargeIntValES6_'], + '15FunctionContextERKNS1_11LargeIntValES6_', '', '', '', ''], [['bitxor'], 'TINYINT', ['TINYINT', 'TINYINT'], '_ZN5doris9Operators32bitxor_tiny_int_val_tiny_int_valEPN9doris_udf' - '15FunctionContextERKNS1_10TinyIntValES6_'], + '15FunctionContextERKNS1_10TinyIntValES6_', '', '', '', ''], [['bitxor'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], '_ZN5doris9Operators34bitxor_small_int_val_small_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11SmallIntValES6_'], + '15FunctionContextERKNS1_11SmallIntValES6_', '', '', '', ''], [['bitxor'], 'INT', ['INT', 'INT'], '_ZN5doris9Operators22bitxor_int_val_int_valEPN9doris_udf' - '15FunctionContextERKNS1_6IntValES6_'], + '15FunctionContextERKNS1_6IntValES6_', '', '', '', ''], [['bitxor'], 'BIGINT', ['BIGINT', 'BIGINT'], '_ZN5doris9Operators30bitxor_big_int_val_big_int_valEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValES6_'], + '15FunctionContextERKNS1_9BigIntValES6_', '', '', '', ''], [['bitxor'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], '_ZN5doris9Operators34bitxor_large_int_val_large_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11LargeIntValES6_'], + '15FunctionContextERKNS1_11LargeIntValES6_', '', '', '', ''], [['bitnot'], 'TINYINT', ['TINYINT'], '_ZN5doris9Operators19bitnot_tiny_int_valEPN9doris_udf' - '15FunctionContextERKNS1_10TinyIntValE'], + '15FunctionContextERKNS1_10TinyIntValE', '', '', '', ''], [['bitnot'], 'SMALLINT', ['SMALLINT'], '_ZN5doris9Operators20bitnot_small_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11SmallIntValE'], + '15FunctionContextERKNS1_11SmallIntValE', '', '', '', ''], [['bitnot'], 'INT', ['INT'], '_ZN5doris9Operators14bitnot_int_valEPN9doris_udf' - '15FunctionContextERKNS1_6IntValE'], + '15FunctionContextERKNS1_6IntValE', '', '', '', ''], [['bitnot'], 'BIGINT', ['BIGINT'], '_ZN5doris9Operators18bitnot_big_int_valEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValE'], + '15FunctionContextERKNS1_9BigIntValE', '', '', '', ''], [['bitnot'], 'LARGEINT', ['LARGEINT'], '_ZN5doris9Operators20bitnot_large_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11LargeIntValE'], + '15FunctionContextERKNS1_11LargeIntValE', '', '', '', ''], # array functions [['array'], 'ARRAY', ['INT', '...'], - '_ZN5doris14ArrayFunctions5arrayEPN9doris_udf15FunctionContextEiPKNS1_6IntValE'], + '_ZN5doris14ArrayFunctions5arrayEPN9doris_udf15FunctionContextEiPKNS1_6IntValE', '', '', '', ''], [['array'], 'ARRAY', ['VARCHAR', '...'], - '_ZN5doris14ArrayFunctions5arrayEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], - [['array'], 'ARRAY', ['ARRAY', '...'], '', ''], - [['array'], 'ARRAY', ['MAP', '...'], '', ''], - [['array'], 'ARRAY', ['STRUCT', '...'], '', ''], - [['%element_extract%'], 'VARCHAR', ['ARRAY', 'INT'], '', ''], - [['%element_extract%'], 'VARCHAR', ['ARRAY', 'VARCHAR'], '', ''], - [['%element_extract%'], 'VARCHAR', ['MAP', 'VARCHAR'], '', ''], - [['%element_extract%'], 'VARCHAR', ['MAP', 'INT'], '', ''], - [['%element_extract%'], 'VARCHAR', ['STRUCT', 'INT'], '', ''], - [['%element_extract%'], 'VARCHAR', ['STRUCT', 'VARCHAR'], '', ''], + '_ZN5doris14ArrayFunctions5arrayEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', '', '', '', ''], + [['array'], 'ARRAY', ['ARRAY', '...'], '', '', '', '', ''], + [['array'], 'ARRAY', ['MAP', '...'], '', '', '', '', ''], + [['array'], 'ARRAY', ['STRUCT', '...'], '', '', '', '', ''], + [['%element_extract%'], 'VARCHAR', ['ARRAY', 'INT'], '', '', '', '', ''], + [['%element_extract%'], 'VARCHAR', ['ARRAY', 'VARCHAR'], '', '', '', '', ''], + [['%element_extract%'], 'VARCHAR', ['MAP', 'VARCHAR'], '', '', '', '', ''], + [['%element_extract%'], 'VARCHAR', ['MAP', 'INT'], '', '', '', '', ''], + [['%element_extract%'], 'VARCHAR', ['STRUCT', 'INT'], '', '', '', '', ''], + [['%element_extract%'], 'VARCHAR', ['STRUCT', 'VARCHAR'], '', '', '', '', ''], # Timestamp functions [['unix_timestamp'], 'INT', [], - '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextE'], + '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], [['unix_timestamp'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['unix_timestamp'], 'INT', ['DATE'], - '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['unix_timestamp'], 'INT', ['VARCHAR', 'VARCHAR'], - '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris18TimestampFunctions7to_unixEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', '', 'ALWAYS_NULLABLE'], [['from_unixtime'], 'VARCHAR', ['INT'], - '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf15FunctionContextERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf15FunctionContextERKNS1_6IntValE', + '', '', 'vec', ''], [['from_unixtime'], 'VARCHAR', ['INT', 'VARCHAR'], '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf' '15FunctionContextERKNS1_6IntValERKNS1_9StringValE', '_ZN5doris18TimestampFunctions14format_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris18TimestampFunctions12format_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', 'vec', ''], [['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIME', [], - '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE'], + '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], [['curtime', 'current_time'], 'TIME', [], - '_ZN5doris18TimestampFunctions7curtimeEPN9doris_udf15FunctionContextE'], + '_ZN5doris18TimestampFunctions7curtimeEPN9doris_udf15FunctionContextE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], [['curdate', 'current_date'], 'DATE', [], - '_ZN5doris18TimestampFunctions7curdateEPN9doris_udf15FunctionContextE'], + '_ZN5doris18TimestampFunctions7curdateEPN9doris_udf15FunctionContextE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], [['utc_timestamp'], 'DATETIME', [], - '_ZN5doris18TimestampFunctions13utc_timestampEPN9doris_udf15FunctionContextE'], + '_ZN5doris18TimestampFunctions13utc_timestampEPN9doris_udf15FunctionContextE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], [['timestamp'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions9timestampEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9timestampEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['from_days'], 'DATE', ['INT'], - '_ZN5doris18TimestampFunctions9from_daysEPN9doris_udf15FunctionContextERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9from_daysEPN9doris_udf15FunctionContextERKNS1_6IntValE', + '', '', '', ''], [['to_days'], 'INT', ['DATE'], - '_ZN5doris18TimestampFunctions7to_daysEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_daysEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['year'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions4yearEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions4yearEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['month'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions5monthEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions5monthEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['quarter'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions7quarterEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7quarterEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['dayofweek'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions11day_of_weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions11day_of_weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['day', 'dayofmonth'], 'INT', ['DATETIME'], '_ZN5doris18TimestampFunctions12day_of_monthEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', ''], [['dayofyear'], 'INT', ['DATETIME'], '_ZN5doris18TimestampFunctions11day_of_yearEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', ''], [['weekofyear'], 'INT', ['DATETIME'], '_ZN5doris18TimestampFunctions12week_of_yearEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', ''], [['yearweek'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions9year_weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9year_weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['yearweek'], 'INT', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions9year_weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9year_weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['week'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions4weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions4weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['week'], 'INT', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions4weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions4weekEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['hour'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions4hourEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions4hourEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['minute'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions6minuteEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions6minuteEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['second'], 'INT', ['DATETIME'], - '_ZN5doris18TimestampFunctions6secondEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions6secondEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['makedate'], 'DATETIME', ['INT', 'INT'], - '_ZN5doris18TimestampFunctions9make_dateEPN9doris_udf15FunctionContextERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions9make_dateEPN9doris_udf15FunctionContextERKNS1_6IntValES6_', + '', '', '', ''], [['years_add'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions9years_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['years_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions9years_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['months_add', 'add_months'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions10months_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['months_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions10months_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['weeks_add'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions9weeks_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['weeks_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions9weeks_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['days_add', 'date_add', 'adddate'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions8days_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['days_sub', 'date_sub', 'subdate'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions8days_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['hours_add'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions9hours_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['hours_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions9hours_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['minutes_add'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions11minutes_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['minutes_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions11minutes_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['seconds_add'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions11seconds_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['seconds_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions11seconds_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', 'vec', ''], [['microseconds_add'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions10micros_addEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['microseconds_sub'], 'DATETIME', ['DATETIME', 'INT'], '_ZN5doris18TimestampFunctions10micros_subEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['datediff'], 'INT', ['DATETIME', 'DATETIME'], '_ZN5doris18TimestampFunctions9date_diffEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValES6_'], + '15FunctionContextERKNS1_11DateTimeValES6_', '', '', 'vec', ''], [['timediff'], 'TIME', ['DATETIME', 'DATETIME'], '_ZN5doris18TimestampFunctions9time_diffEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValES6_'], + '15FunctionContextERKNS1_11DateTimeValES6_', '', '', 'vec', ''], [['str_to_date'], 'DATETIME', ['VARCHAR', 'VARCHAR'], '_ZN5doris18TimestampFunctions11str_to_dateEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_'], + '15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'], [['date_format'], 'VARCHAR', ['DATETIME', 'VARCHAR'], '_ZN5doris18TimestampFunctions11date_formatEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValE', '_ZN5doris18TimestampFunctions14format_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris18TimestampFunctions12format_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', 'vec', ''], [['date_format'], 'VARCHAR', ['DATE', 'VARCHAR'], '_ZN5doris18TimestampFunctions11date_formatEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValE', '_ZN5doris18TimestampFunctions14format_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris18TimestampFunctions12format_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', 'vec', ''], [['date', 'to_date'], 'DATE', ['DATETIME'], - '_ZN5doris18TimestampFunctions7to_dateEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions7to_dateEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', 'vec', ''], [['dayname'], 'VARCHAR', ['DATETIME'], '_ZN5doris18TimestampFunctions8day_nameEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', ''], [['monthname'], 'VARCHAR', ['DATETIME'], '_ZN5doris18TimestampFunctions10month_nameEPN9doris_udf' - '15FunctionContextERKNS1_11DateTimeValE'], + '15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', ''], [['convert_tz'], 'DATETIME', ['DATETIME', 'VARCHAR', 'VARCHAR'], '_ZN5doris18TimestampFunctions10convert_tzEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValES9_', '_ZN5doris18TimestampFunctions18convert_tz_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris18TimestampFunctions16convert_tz_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris18TimestampFunctions16convert_tz_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + 'vec', 'ALWAYS_NULLABLE'], [['years_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10years_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10years_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['months_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions11months_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions11months_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['weeks_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10weeks_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10weeks_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['days_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions9days_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions9days_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['hours_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10hours_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10hours_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['minutes_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions12minutes_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions12minutes_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['seconds_diff'], 'BIGINT', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions12seconds_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions12seconds_diffEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', 'vec', ''], [['year_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['year_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['year_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['year_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions10year_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['year_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['year_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['year_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['year_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions9year_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['month_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['month_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['month_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['month_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions11month_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['month_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['month_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['month_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['month_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions10month_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['week_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['week_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['week_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['week_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions10week_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['week_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['week_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['week_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['week_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions9week_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['day_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['day_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['day_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['day_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions9day_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['day_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['day_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['day_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['day_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions8day_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['hour_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['hour_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['hour_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['hour_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions10hour_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['hour_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['hour_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['hour_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['hour_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions9hour_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['minute_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['minute_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['minute_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['minute_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions12minute_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['minute_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['minute_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['minute_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['minute_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions11minute_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['second_floor'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['second_floor'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['second_floor'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['second_floor'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions12second_floorEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], [['second_ceil'], 'DATETIME', ['DATETIME'], - '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE'], + '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '', '', '', ''], [['second_ceil'], 'DATETIME', ['DATETIME', 'DATETIME'], - '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_'], + '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValES6_', + '', '', '', ''], [['second_ceil'], 'DATETIME', ['DATETIME', 'INT'], - '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE'], + '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValE', + '', '', '', ''], [['second_ceil'], 'DATETIME', ['DATETIME', 'INT', 'DATETIME'], - '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_'], + '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', + '', '', '', ''], # Math builtin functions [['pi'], 'DOUBLE', [], - '_ZN5doris13MathFunctions2piEPN9doris_udf15FunctionContextE'], + '_ZN5doris13MathFunctions2piEPN9doris_udf15FunctionContextE', '', '', 'vec', 'ALWAYS_NULLABLE'], [['e'], 'DOUBLE', [], - '_ZN5doris13MathFunctions1eEPN9doris_udf15FunctionContextE'], + '_ZN5doris13MathFunctions1eEPN9doris_udf15FunctionContextE', '', '', 'vec', 'ALWAYS_NULLABLE'], [['abs'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['abs'], 'FLOAT', ['FLOAT'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_8FloatValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_8FloatValE', '', '', 'vec', ''], [['abs'], 'LARGEINT', ['LARGEINT'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_11LargeIntValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_11LargeIntValE', '', '', 'vec', ''], [['abs'], 'LARGEINT', ['BIGINT'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_9BigIntValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', '', '', 'vec', ''], [['abs'], 'INT', ['SMALLINT'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_11SmallIntValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_11SmallIntValE', '', '', 'vec', ''], [['abs'], 'BIGINT', ['INT'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_6IntValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_6IntValE', '', '', 'vec', ''], [['abs'], 'SMALLINT', ['TINYINT'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_10TinyIntValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_10TinyIntValE', '', '', 'vec', ''], [['abs'], 'DECIMALV2', ['DECIMALV2'], - '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValE'], + '_ZN5doris13MathFunctions3absEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValE', '', '', 'vec', ''], [['sign'], 'FLOAT', ['DOUBLE'], - '_ZN5doris13MathFunctions4signEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4signEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['sin'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions3sinEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions3sinEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['asin'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions4asinEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4asinEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['cos'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions3cosEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions3cosEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['acos'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions4acosEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4acosEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['tan'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions3tanEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions3tanEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['atan'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions4atanEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4atanEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['ceil', 'ceiling', 'dceil'], 'BIGINT', ['DOUBLE'], - '_ZN5doris13MathFunctions4ceilEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4ceilEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['floor', 'dfloor'], 'BIGINT', ['DOUBLE'], - '_ZN5doris13MathFunctions5floorEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions5floorEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['round', 'dround'], 'BIGINT', ['DOUBLE'], - '_ZN5doris13MathFunctions5roundEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions5roundEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', '', ''], [['round', 'dround'], 'DOUBLE', ['DOUBLE', 'INT'], '_ZN5doris13MathFunctions11round_up_toEPN9doris_udf' - '15FunctionContextERKNS1_9DoubleValERKNS1_6IntValE'], + '15FunctionContextERKNS1_9DoubleValERKNS1_6IntValE', '', '', '', ''], [['truncate'], 'DOUBLE', ['DOUBLE', 'INT'], '_ZN5doris13MathFunctions8truncateEPN9doris_udf' - '15FunctionContextERKNS1_9DoubleValERKNS1_6IntValE'], + '15FunctionContextERKNS1_9DoubleValERKNS1_6IntValE', '', '', 'vec', ''], [['ln', 'dlog1'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions2lnEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions2lnEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['log'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], - '_ZN5doris13MathFunctions3logEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_'], + '_ZN5doris13MathFunctions3logEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_', '', '', 'vec', ''], [['log2'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions4log2EPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4log2EPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['log10', 'dlog10'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions5log10EPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions5log10EPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['exp', 'dexp'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions3expEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions3expEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['radians'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions7radiansEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions7radiansEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['degrees'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions7degreesEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions7degreesEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['sqrt', 'dsqrt'], 'DOUBLE', ['DOUBLE'], - '_ZN5doris13MathFunctions4sqrtEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions4sqrtEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['pow', 'power', 'dpow', 'fpow'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], - '_ZN5doris13MathFunctions3powEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_'], + '_ZN5doris13MathFunctions3powEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_', '', '', 'vec', ''], [['rand', 'random'], 'DOUBLE', [], '_ZN5doris13MathFunctions4randEPN9doris_udf15FunctionContextE', '_ZN5doris13MathFunctions12rand_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris13MathFunctions10rand_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', '', 'ALWAYS_NOT_NULLABLE'], [['rand', 'random'], 'DOUBLE', ['BIGINT'], '_ZN5doris13MathFunctions9rand_seedEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', '_ZN5doris13MathFunctions12rand_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris13MathFunctions10rand_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', '', ''], [['bin'], 'VARCHAR', ['BIGINT'], - '_ZN5doris13MathFunctions3binEPN9doris_udf15FunctionContextERKNS1_9BigIntValE'], + '_ZN5doris13MathFunctions3binEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', '', '', '', ''], [['hex'], 'VARCHAR', ['BIGINT'], - '_ZN5doris13MathFunctions7hex_intEPN9doris_udf15FunctionContextERKNS1_9BigIntValE'], + '_ZN5doris13MathFunctions7hex_intEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', '', '', '', ''], [['hex'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris13MathFunctions10hex_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris13MathFunctions10hex_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', '', ''], [['unhex'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris13MathFunctions5unhexEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris13MathFunctions5unhexEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', '', ''], [['conv'], 'VARCHAR', ['BIGINT', 'TINYINT', 'TINYINT'], '_ZN5doris13MathFunctions8conv_intEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValERKNS1_10TinyIntValES9_'], + '15FunctionContextERKNS1_9BigIntValERKNS1_10TinyIntValES9_', '', '', '', ''], [['conv'], 'VARCHAR', ['VARCHAR', 'TINYINT', 'TINYINT'], '_ZN5doris13MathFunctions11conv_stringEPN9doris_udf' - '15FunctionContextERKNS1_9StringValERKNS1_10TinyIntValES9_'], + '15FunctionContextERKNS1_9StringValERKNS1_10TinyIntValES9_', '', '', '', ''], [['pmod'], 'BIGINT', ['BIGINT', 'BIGINT'], '_ZN5doris13MathFunctions11pmod_bigintEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValES6_'], + '15FunctionContextERKNS1_9BigIntValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['pmod'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], '_ZN5doris13MathFunctions11pmod_doubleEPN9doris_udf' - '15FunctionContextERKNS1_9DoubleValES6_'], + '15FunctionContextERKNS1_9DoubleValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod'], 'TINYINT', ['TINYINT', 'TINYINT'], '_ZN5doris9Operators29mod_tiny_int_val_tiny_int_valEPN9doris_udf' - '15FunctionContextERKNS1_10TinyIntValES6_'], + '15FunctionContextERKNS1_10TinyIntValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], '_ZN5doris9Operators31mod_small_int_val_small_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11SmallIntValES6_'], + '15FunctionContextERKNS1_11SmallIntValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod'], 'INT', ['INT', 'INT'], '_ZN5doris9Operators19mod_int_val_int_valEPN9doris_udf' - '15FunctionContextERKNS1_6IntValES6_'], + '15FunctionContextERKNS1_6IntValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod'], 'BIGINT', ['BIGINT', 'BIGINT'], '_ZN5doris9Operators27mod_big_int_val_big_int_valEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValES6_'], + '15FunctionContextERKNS1_9BigIntValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], '_ZN5doris9Operators31mod_large_int_val_large_int_valEPN9doris_udf' - '15FunctionContextERKNS1_11LargeIntValES6_'], + '15FunctionContextERKNS1_11LargeIntValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod'], 'DECIMALV2', ['DECIMALV2', 'DECIMALV2'], '_ZN5doris18DecimalV2Operators31mod_decimalv2_val_decimalv2_valEPN9doris_udf' - '15FunctionContextERKNS1_12DecimalV2ValES6_'], + '15FunctionContextERKNS1_12DecimalV2ValES6_', '', '', '', 'ALWAYS_NULLABLE'], [['mod', 'fmod'], 'FLOAT', ['FLOAT', 'FLOAT'], - '_ZN5doris13MathFunctions10fmod_floatEPN9doris_udf15FunctionContextERKNS1_8FloatValES6_'], + '_ZN5doris13MathFunctions10fmod_floatEPN9doris_udf15FunctionContextERKNS1_8FloatValES6_', + '', '', '', 'ALWAYS_NULLABLE'], [['mod', 'fmod'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], - '_ZN5doris13MathFunctions11fmod_doubleEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_'], + '_ZN5doris13MathFunctions11fmod_doubleEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_', + '', '', '', 'ALWAYS_NULLABLE'], [['positive'], 'BIGINT', ['BIGINT'], '_ZN5doris13MathFunctions15positive_bigintEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValE'], + '15FunctionContextERKNS1_9BigIntValE', '', '', 'vec', ''], [['positive'], 'DOUBLE', ['DOUBLE'], '_ZN5doris13MathFunctions15positive_doubleEPN9doris_udf' - '15FunctionContextERKNS1_9DoubleValE'], + '15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['positive'], 'DECIMALV2', ['DECIMALV2'], '_ZN5doris13MathFunctions16positive_decimalEPN9doris_udf' - '15FunctionContextERKNS1_12DecimalV2ValE'], + '15FunctionContextERKNS1_12DecimalV2ValE', '', '', 'vec', ''], [['negative'], 'BIGINT', ['BIGINT'], '_ZN5doris13MathFunctions15negative_bigintEPN9doris_udf' - '15FunctionContextERKNS1_9BigIntValE'], + '15FunctionContextERKNS1_9BigIntValE', '', '', 'vec', ''], [['negative'], 'DOUBLE', ['DOUBLE'], '_ZN5doris13MathFunctions15negative_doubleEPN9doris_udf' - '15FunctionContextERKNS1_9DoubleValE'], + '15FunctionContextERKNS1_9DoubleValE', '', '', 'vec', ''], [['negative'], 'DECIMALV2', ['DECIMALV2'], '_ZN5doris13MathFunctions16negative_decimalEPN9doris_udf' - '15FunctionContextERKNS1_12DecimalV2ValE'], + '15FunctionContextERKNS1_12DecimalV2ValE', '', '', 'vec', ''], [['least'], 'TINYINT', ['TINYINT', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_10TinyIntValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_10TinyIntValE', + '', '', '', ''], [['least'], 'SMALLINT', ['SMALLINT', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_11SmallIntValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_11SmallIntValE', + '', '', '', ''], [['least'], 'INT', ['INT', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_6IntValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_6IntValE', + '', '', '', ''], [['least'], 'BIGINT', ['BIGINT', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_9BigIntValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_9BigIntValE', + '', '', '', ''], [['least'], 'LARGEINT', ['LARGEINT', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_11LargeIntValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_11LargeIntValE', + '', '', '', ''], [['least'], 'FLOAT', ['FLOAT', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_8FloatValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_8FloatValE', + '', '', '', ''], [['least'], 'DOUBLE', ['DOUBLE', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_9DoubleValE', + '', '', '', ''], [['least'], 'DATETIME', ['DATETIME', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_11DateTimeValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_11DateTimeValE', + '', '', '', ''], [['least'], 'DECIMALV2', ['DECIMALV2', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_12DecimalV2ValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_12DecimalV2ValE', + '', '', '', ''], [['least'], 'VARCHAR', ['VARCHAR', '...'], - '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + '_ZN5doris13MathFunctions5leastEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', + '', '', '', ''], [['greatest'], 'TINYINT', ['TINYINT', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_10TinyIntValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_10TinyIntValE', + '', '', '', ''], [['greatest'], 'SMALLINT', ['SMALLINT', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_11SmallIntValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_11SmallIntValE', + '', '', '', ''], [['greatest'], 'INT', ['INT', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_6IntValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_6IntValE', + '', '', '', ''], [['greatest'], 'BIGINT', ['BIGINT', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_9BigIntValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_9BigIntValE', + '', '', '', ''], [['greatest'], 'LARGEINT', ['LARGEINT', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_11LargeIntValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_11LargeIntValE', + '', '', '', ''], [['greatest'], 'FLOAT', ['FLOAT', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_8FloatValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_8FloatValE', + '', '', '', ''], [['greatest'], 'DOUBLE', ['DOUBLE', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_9DoubleValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_9DoubleValE', + '', '', '', ''], [['greatest'], 'DECIMALV2', ['DECIMALV2', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_12DecimalV2ValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_12DecimalV2ValE', + '', '', '', ''], [['greatest'], 'DATETIME', ['DATETIME', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_11DateTimeValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_11DateTimeValE', + '', '', '', ''], [['greatest'], 'VARCHAR', ['VARCHAR', '...'], - '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + '_ZN5doris13MathFunctions8greatestEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', + '', '', '', ''], # Conditional Functions # Some of these have empty symbols because the BE special-cases them based on the # function name - [['if'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN', 'BOOLEAN'], ''], - [['if'], 'TINYINT', ['BOOLEAN', 'TINYINT', 'TINYINT'], ''], - [['if'], 'SMALLINT', ['BOOLEAN', 'SMALLINT', 'SMALLINT'], ''], - [['if'], 'INT', ['BOOLEAN', 'INT', 'INT'], ''], - [['if'], 'BIGINT', ['BOOLEAN', 'BIGINT', 'BIGINT'], ''], - [['if'], 'LARGEINT', ['BOOLEAN', 'LARGEINT', 'LARGEINT'], ''], - [['if'], 'FLOAT', ['BOOLEAN', 'FLOAT', 'FLOAT'], ''], - [['if'], 'DOUBLE', ['BOOLEAN', 'DOUBLE', 'DOUBLE'], ''], - [['if'], 'DATETIME', ['BOOLEAN', 'DATETIME', 'DATETIME'], ''], - [['if'], 'DATE', ['BOOLEAN', 'DATE', 'DATE'], ''], - [['if'], 'DECIMALV2', ['BOOLEAN', 'DECIMALV2', 'DECIMALV2'], ''], - [['if'], 'BITMAP', ['BOOLEAN', 'BITMAP', 'BITMAP'], ''], + [['if'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN', 'BOOLEAN'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'TINYINT', ['BOOLEAN', 'TINYINT', 'TINYINT'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'SMALLINT', ['BOOLEAN', 'SMALLINT', 'SMALLINT'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'INT', ['BOOLEAN', 'INT', 'INT'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'BIGINT', ['BOOLEAN', 'BIGINT', 'BIGINT'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'LARGEINT', ['BOOLEAN', 'LARGEINT', 'LARGEINT'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'FLOAT', ['BOOLEAN', 'FLOAT', 'FLOAT'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'DOUBLE', ['BOOLEAN', 'DOUBLE', 'DOUBLE'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'DATETIME', ['BOOLEAN', 'DATETIME', 'DATETIME'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'DATE', ['BOOLEAN', 'DATE', 'DATE'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'DECIMALV2', ['BOOLEAN', 'DECIMALV2', 'DECIMALV2'], '', '', '', 'vec', 'CUSTOM'], + [['if'], 'BITMAP', ['BOOLEAN', 'BITMAP', 'BITMAP'], '', '', '', 'vec', 'CUSTOM'], # The priority of varchar should be lower than decimal in IS_SUPERTYPE_OF mode. - [['if'], 'VARCHAR', ['BOOLEAN', 'VARCHAR', 'VARCHAR'], ''], - - [['nullif'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], ''], - [['nullif'], 'TINYINT', ['TINYINT', 'TINYINT'], ''], - [['nullif'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], ''], - [['nullif'], 'INT', ['INT', 'INT'], ''], - [['nullif'], 'BIGINT', ['BIGINT', 'BIGINT'], ''], - [['nullif'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], ''], - [['nullif'], 'FLOAT', ['FLOAT', 'FLOAT'], ''], - [['nullif'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], ''], - [['nullif'], 'DATETIME', ['DATETIME', 'DATETIME'], ''], - [['nullif'], 'DATE', ['DATE', 'DATE'], ''], - [['nullif'], 'DECIMALV2', ['DECIMALV2', 'DECIMALV2'], ''], + [['if'], 'VARCHAR', ['BOOLEAN', 'VARCHAR', 'VARCHAR'], '', '', '', 'vec', 'CUSTOM'], + + [['nullif'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'TINYINT', ['TINYINT', 'TINYINT'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'INT', ['INT', 'INT'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'BIGINT', ['BIGINT', 'BIGINT'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'FLOAT', ['FLOAT', 'FLOAT'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'DATETIME', ['DATETIME', 'DATETIME'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'DATE', ['DATE', 'DATE'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['nullif'], 'DECIMALV2', ['DECIMALV2', 'DECIMALV2'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], # The priority of varchar should be lower than decimal in IS_SUPERTYPE_OF mode. - [['nullif'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], ''], - - [['ifnull'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], ''], - [['ifnull'], 'TINYINT', ['TINYINT', 'TINYINT'], ''], - [['ifnull'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], ''], - [['ifnull'], 'INT', ['INT', 'INT'], ''], - [['ifnull'], 'BIGINT', ['BIGINT', 'BIGINT'], ''], - [['ifnull'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], ''], - [['ifnull'], 'FLOAT', ['FLOAT', 'FLOAT'], ''], - [['ifnull'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], ''], - [['ifnull'], 'DATE', ['DATE', 'DATE'], ''], - [['ifnull'], 'DATETIME', ['DATETIME', 'DATETIME'], ''], - [['ifnull'], 'DATETIME', ['DATE', 'DATETIME'], ''], - [['ifnull'], 'DATETIME', ['DATETIME', 'DATE'], ''], - [['ifnull'], 'DECIMALV2', ['DECIMALV2', 'DECIMALV2'], ''], - [['ifnull'], 'BITMAP', ['BITMAP', 'BITMAP'], ''], + [['nullif'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '', '', '', 'vec', 'ALWAYS_NULLABLE'], + + [['ifnull'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'TINYINT', ['TINYINT', 'TINYINT'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'INT', ['INT', 'INT'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'BIGINT', ['BIGINT', 'BIGINT'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'LARGEINT', ['LARGEINT', 'LARGEINT'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'FLOAT', ['FLOAT', 'FLOAT'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'DOUBLE', ['DOUBLE', 'DOUBLE'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'DATE', ['DATE', 'DATE'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'DATETIME', ['DATETIME', 'DATETIME'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'DATETIME', ['DATE', 'DATETIME'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'DATETIME', ['DATETIME', 'DATE'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'DECIMALV2', ['DECIMALV2', 'DECIMALV2'], '', '', '', 'vec', 'CUSTOM'], + [['ifnull'], 'BITMAP', ['BITMAP', 'BITMAP'], '', '', '', 'vec', 'CUSTOM'], # The priority of varchar should be lower than decimal in IS_SUPERTYPE_OF mode. - [['ifnull'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], ''], - - [['coalesce'], 'BOOLEAN', ['BOOLEAN', '...'], ''], - [['coalesce'], 'TINYINT', ['TINYINT', '...'], ''], - [['coalesce'], 'SMALLINT', ['SMALLINT', '...'], ''], - [['coalesce'], 'INT', ['INT', '...'], ''], - [['coalesce'], 'BIGINT', ['BIGINT', '...'], ''], - [['coalesce'], 'LARGEINT', ['LARGEINT', '...'], ''], - [['coalesce'], 'FLOAT', ['FLOAT', '...'], ''], - [['coalesce'], 'DOUBLE', ['DOUBLE', '...'], ''], - [['coalesce'], 'DATETIME', ['DATETIME', '...'], ''], - [['coalesce'], 'DATE', ['DATE', '...'], ''], - [['coalesce'], 'DECIMALV2', ['DECIMALV2', '...'], ''], - [['coalesce'], 'BITMAP', ['BITMAP', '...'], ''], + [['ifnull'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '', '', '', 'vec', 'CUSTOM'], + + [['coalesce'], 'BOOLEAN', ['BOOLEAN', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'TINYINT', ['TINYINT', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'SMALLINT', ['SMALLINT', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'INT', ['INT', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'BIGINT', ['BIGINT', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'LARGEINT', ['LARGEINT', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'FLOAT', ['FLOAT', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'DOUBLE', ['DOUBLE', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'DATETIME', ['DATETIME', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'DATE', ['DATE', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'DECIMALV2', ['DECIMALV2', '...'], '', '', '', '', 'CUSTOM'], + [['coalesce'], 'BITMAP', ['BITMAP', '...'], '', '', '', '', 'CUSTOM'], # The priority of varchar should be lower than decimal in IS_SUPERTYPE_OF mode. - [['coalesce'], 'VARCHAR', ['VARCHAR', '...'], ''], + [['coalesce'], 'VARCHAR', ['VARCHAR', '...'], '', '', '', '', 'CUSTOM'], [['esquery'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], '_ZN5doris11ESFunctions5matchEPN' - '9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], # String builtin functions [['substr', 'substring'], 'VARCHAR', ['VARCHAR', 'INT'], '_ZN5doris15StringFunctions9substringEPN' - '9doris_udf15FunctionContextERKNS1_9StringValERKNS1_6IntValE'], + '9doris_udf15FunctionContextERKNS1_9StringValERKNS1_6IntValE', '', '', '', 'ALWAYS_NULLABLE'], [['substr', 'substring'], 'VARCHAR', ['VARCHAR', 'INT', 'INT'], '_ZN5doris15StringFunctions9substringEPN' - '9doris_udf15FunctionContextERKNS1_9StringValERKNS1_6IntValES9_'], + '9doris_udf15FunctionContextERKNS1_9StringValERKNS1_6IntValES9_', '', '', 'vec', 'ALWAYS_NULLABLE'], [['strleft', 'left'], 'VARCHAR', ['VARCHAR', 'INT'], '_ZN5doris15StringFunctions4leftEPN9doris_udf' - '15FunctionContextERKNS1_9StringValERKNS1_6IntValE'], + '15FunctionContextERKNS1_9StringValERKNS1_6IntValE', '', '', 'vec', ''], [['strright', 'right'], 'VARCHAR', ['VARCHAR', 'INT'], '_ZN5doris15StringFunctions5rightEPN9doris_udf' - '15FunctionContextERKNS1_9StringValERKNS1_6IntValE'], + '15FunctionContextERKNS1_9StringValERKNS1_6IntValE', '', '', 'vec', ''], [['ends_with'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions9ends_withEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15StringFunctions9ends_withEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['starts_with'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions11starts_withEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15StringFunctions11starts_withEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['null_or_empty'], 'BOOLEAN', ['VARCHAR'], - '_ZN5doris15StringFunctions13null_or_emptyEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions13null_or_emptyEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['space'], 'VARCHAR', ['INT'], - '_ZN5doris15StringFunctions5spaceEPN9doris_udf15FunctionContextERKNS1_6IntValE'], + '_ZN5doris15StringFunctions5spaceEPN9doris_udf15FunctionContextERKNS1_6IntValE', '', '', 'vec', ''], [['repeat'], 'VARCHAR', ['VARCHAR', 'INT'], '_ZN5doris15StringFunctions6repeatEPN9doris_udf' - '15FunctionContextERKNS1_9StringValERKNS1_6IntValE'], + '15FunctionContextERKNS1_9StringValERKNS1_6IntValE', '', '', 'vec', ''], [['lpad'], 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], '_ZN5doris15StringFunctions4lpadEPN9doris_udf' - '15FunctionContextERKNS1_9StringValERKNS1_6IntValES6_'], + '15FunctionContextERKNS1_9StringValERKNS1_6IntValES6_', '', '', 'vec', ''], [['rpad'], 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], '_ZN5doris15StringFunctions4rpadEPN9doris_udf' - '15FunctionContextERKNS1_9StringValERKNS1_6IntValES6_'], + '15FunctionContextERKNS1_9StringValERKNS1_6IntValES6_', '', '', 'vec', ''], [['append_trailing_char_if_absent'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions30append_trailing_char_if_absentEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15StringFunctions30append_trailing_char_if_absentEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', 'ALWAYS_NULLABLE'], [['length'], 'INT', ['VARCHAR'], - '_ZN5doris15StringFunctions6lengthEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions6lengthEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], [['bit_length'], 'INT', ['VARCHAR'], - '_ZN5doris15StringFunctions10bit_lengthEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions10bit_lengthEPN9doris_udf15FunctionContextERKNS1_9StringValE','', '', 'vec', ''], + [['char_length', 'character_length'], 'INT', ['VARCHAR'], - '_ZN5doris15StringFunctions16char_utf8_lengthEPN9doris_udf15FunctionContextERKNS1_9StringValE'], - [['lower', 'lcase'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris15StringFunctions5lowerEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions16char_utf8_lengthEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], + [['lower', 'lcase'], 'VARCHAR', ['VARCHAR'], + '_ZN5doris15StringFunctions5lowerEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['upper', 'ucase'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris15StringFunctions5upperEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions5upperEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['reverse'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris15StringFunctions7reverseEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions7reverseEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['trim'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris15StringFunctions4trimEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions4trimEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['ltrim'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris15StringFunctions5ltrimEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions5ltrimEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['rtrim'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris15StringFunctions5rtrimEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions5rtrimEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['ascii'], 'INT', ['VARCHAR'], - '_ZN5doris15StringFunctions5asciiEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15StringFunctions5asciiEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', ''], [['instr'], 'INT', ['VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions5instrEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15StringFunctions5instrEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], [['locate'], 'INT', ['VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions6locateEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15StringFunctions6locateEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], [['locate'], 'INT', ['VARCHAR', 'VARCHAR', 'INT'], '_ZN5doris15StringFunctions10locate_posEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_RKNS1_6IntValE'], + '15FunctionContextERKNS1_9StringValES6_RKNS1_6IntValE', '', '', 'vec', ''], [['regexp_extract'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'BIGINT'], '_ZN5doris15StringFunctions14regexp_extractEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_RKNS1_9BigIntValE', '_ZN5doris15StringFunctions14regexp_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris15StringFunctions12regexp_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', '', ''], [['regexp_replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions14regexp_replaceEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_S6_', '_ZN5doris15StringFunctions14regexp_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris15StringFunctions12regexp_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', '', ''], [['concat'], 'VARCHAR', ['VARCHAR', '...'], - '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + '_ZN5doris15StringFunctions6concatEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', + '', '', 'vec', ''], [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], - '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_'], + '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_', + '', '', 'vec', ''], [['concat_ws'], 'VARCHAR', ['VARCHAR', 'VARCHAR', '...'], '_ZN5doris15StringFunctions9concat_wsEPN9doris_udf' - '15FunctionContextERKNS1_9StringValEiPS5_'], + '15FunctionContextERKNS1_9StringValEiPS5_', '', '', 'vec', 'CUSTOM'], [['find_in_set'], 'INT', ['VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions11find_in_setEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_'], + '15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], [['parse_url'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions9parse_urlEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_', '_ZN5doris15StringFunctions17parse_url_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris15StringFunctions15parse_url_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', '', ''], [['parse_url'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions13parse_url_keyEPN9doris_udf' '15FunctionContextERKNS1_9StringValES6_S6_', '_ZN5doris15StringFunctions17parse_url_prepareEPN9doris_udf' '15FunctionContextENS2_18FunctionStateScopeE', '_ZN5doris15StringFunctions15parse_url_closeEPN9doris_udf' - '15FunctionContextENS2_18FunctionStateScopeE'], + '15FunctionContextENS2_18FunctionStateScopeE', '', ''], [['money_format'], 'VARCHAR', ['BIGINT'], - '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_9BigIntValE'], + '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', + '', '', '', ''], [['money_format'], 'VARCHAR', ['LARGEINT'], - '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_11LargeIntValE'], + '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_11LargeIntValE', + '', '', '', ''], [['money_format'], 'VARCHAR', ['DOUBLE'], - '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_9DoubleValE'], + '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_9DoubleValE', + '', '', '', ''], [['money_format'], 'VARCHAR', ['DECIMALV2'], - '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValE'], - [['split_part'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'INT'], - '_ZN5doris15StringFunctions10split_partEPN9doris_udf15FunctionContextERKNS1_9StringValES6_RKNS1_6IntValE'], + '_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_12DecimalV2ValE', + '', '', '', ''], + [['split_part'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'INT'], + '_ZN5doris15StringFunctions10split_partEPN9doris_udf15FunctionContextERKNS1_9StringValES6_RKNS1_6IntValE', + '', '', '', ''], # Utility functions [['sleep'], 'BOOLEAN', ['INT'], - '_ZN5doris16UtilityFunctions5sleepEPN9doris_udf15FunctionContextERKNS1_6IntValE'], + '_ZN5doris16UtilityFunctions5sleepEPN9doris_udf15FunctionContextERKNS1_6IntValE', + '', '', '', ''], [['version'], 'VARCHAR', [], - '_ZN5doris16UtilityFunctions7versionEPN9doris_udf15FunctionContextE'], + '_ZN5doris16UtilityFunctions7versionEPN9doris_udf15FunctionContextE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], # Json functions [['get_json_int'], 'INT', ['VARCHAR', 'VARCHAR'], '_ZN5doris13JsonFunctions12get_json_intEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', '_ZN5doris13JsonFunctions17json_path_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris13JsonFunctions15json_path_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris13JsonFunctions15json_path_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + 'vec', ''], [['get_json_double'], 'DOUBLE', ['VARCHAR', 'VARCHAR'], '_ZN5doris13JsonFunctions15get_json_doubleEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', '_ZN5doris13JsonFunctions17json_path_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris13JsonFunctions15json_path_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris13JsonFunctions15json_path_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + 'vec', ''], [['get_json_string'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '_ZN5doris13JsonFunctions15get_json_stringEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', '_ZN5doris13JsonFunctions17json_path_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris13JsonFunctions15json_path_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris13JsonFunctions15json_path_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + 'vec', ''], #hll function [['hll_cardinality'], 'BIGINT', ['VARCHAR'], - '_ZN5doris12HllFunctions15hll_cardinalityEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris12HllFunctions15hll_cardinalityEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['hll_hash'], 'HLL', ['VARCHAR'], - '_ZN5doris12HllFunctions8hll_hashEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris12HllFunctions8hll_hashEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['hll_empty'], 'HLL', [], - '_ZN5doris12HllFunctions9hll_emptyEPN9doris_udf15FunctionContextE'], + '_ZN5doris12HllFunctions9hll_emptyEPN9doris_udf15FunctionContextE', + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], #bitmap function [['to_bitmap'], 'BITMAP', ['VARCHAR'], - '_ZN5doris15BitmapFunctions9to_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15BitmapFunctions9to_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], [['bitmap_hash'], 'BITMAP', ['VARCHAR'], - '_ZN5doris15BitmapFunctions11bitmap_hashEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15BitmapFunctions11bitmap_hashEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], [['bitmap_count'], 'BIGINT', ['BITMAP'], - '_ZN5doris15BitmapFunctions12bitmap_countEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15BitmapFunctions12bitmap_countEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], [['bitmap_empty'], 'BITMAP', [], - '_ZN5doris15BitmapFunctions12bitmap_emptyEPN9doris_udf15FunctionContextE'], + '_ZN5doris15BitmapFunctions12bitmap_emptyEPN9doris_udf15FunctionContextE', + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['bitmap_or'], 'BITMAP', ['BITMAP','BITMAP'], - '_ZN5doris15BitmapFunctions9bitmap_orEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15BitmapFunctions9bitmap_orEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['bitmap_xor'], 'BITMAP', ['BITMAP','BITMAP'], - '_ZN5doris15BitmapFunctions10bitmap_xorEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15BitmapFunctions10bitmap_xorEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['bitmap_not'], 'BITMAP', ['BITMAP','BITMAP'], - '_ZN5doris15BitmapFunctions10bitmap_notEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15BitmapFunctions10bitmap_notEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['bitmap_and'], 'BITMAP', ['BITMAP','BITMAP'], - '_ZN5doris15BitmapFunctions10bitmap_andEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15BitmapFunctions10bitmap_andEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['bitmap_to_string'], 'VARCHAR', ['BITMAP'], - '_ZN5doris15BitmapFunctions16bitmap_to_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15BitmapFunctions16bitmap_to_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], [['bitmap_from_string'], 'BITMAP', ['VARCHAR'], - '_ZN5doris15BitmapFunctions18bitmap_from_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15BitmapFunctions18bitmap_from_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], [['bitmap_contains'], 'BOOLEAN', ['BITMAP','BIGINT'], - '_ZN5doris15BitmapFunctions15bitmap_containsEPN9doris_udf15FunctionContextERKNS1_9StringValERKNS1_9BigIntValE'], + '_ZN5doris15BitmapFunctions15bitmap_containsEPN9doris_udf15FunctionContextERKNS1_9StringValERKNS1_9BigIntValE', + '', '', 'vec', ''], [['bitmap_has_any'], 'BOOLEAN', ['BITMAP','BITMAP'], - '_ZN5doris15BitmapFunctions14bitmap_has_anyEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + '_ZN5doris15BitmapFunctions14bitmap_has_anyEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', + '', '', 'vec', ''], [['bitmap_min'], 'BIGINT', ['BITMAP'], - '_ZN5doris15BitmapFunctions10bitmap_minEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris15BitmapFunctions10bitmap_minEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', 'vec', ''], # hash functions [['murmur_hash3_32'], 'INT', ['VARCHAR', '...'], - '_ZN5doris13HashFunctions15murmur_hash3_32EPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + '_ZN5doris13HashFunctions15murmur_hash3_32EPN9doris_udf15FunctionContextEiPKNS1_9StringValE', + '', '', '', ''], # aes and base64 function [['aes_encrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '_ZN5doris19EncryptionFunctions11aes_encryptEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_'], + '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], [['aes_decrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], '_ZN5doris19EncryptionFunctions11aes_decryptEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_'], + '15FunctionContextERKNS1_9StringValES6_', '', '', '', ''], [['from_base64'], 'VARCHAR', ['VARCHAR'], '_ZN5doris19EncryptionFunctions11from_base64EPN9doris_udf' - '15FunctionContextERKNS1_9StringValE'], + '15FunctionContextERKNS1_9StringValE', '', '', '', ''], [['to_base64'], 'VARCHAR', ['VARCHAR'], '_ZN5doris19EncryptionFunctions9to_base64EPN9doris_udf' - '15FunctionContextERKNS1_9StringValE'], + '15FunctionContextERKNS1_9StringValE', '', '', '', ''], # for compatable with MySQL [['md5'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris19EncryptionFunctions3md5EPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris19EncryptionFunctions3md5EPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', '', ''], [['md5sum'], 'VARCHAR', ['VARCHAR', '...'], - '_ZN5doris19EncryptionFunctions6md5sumEPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], + '_ZN5doris19EncryptionFunctions6md5sumEPN9doris_udf15FunctionContextEiPKNS1_9StringValE', '', '', '', ''], # geo functions [['ST_Point'], 'VARCHAR', ['DOUBLE', 'DOUBLE'], - '_ZN5doris12GeoFunctions8st_pointEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_'], + '_ZN5doris12GeoFunctions8st_pointEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_', '', '', '', ''], [['ST_X'], 'DOUBLE', ['VARCHAR'], - '_ZN5doris12GeoFunctions4st_xEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris12GeoFunctions4st_xEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', '', 'ALWAYS_NULLABLE'], [['ST_Y'], 'DOUBLE', ['VARCHAR'], - '_ZN5doris12GeoFunctions4st_yEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris12GeoFunctions4st_yEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', '', 'ALWAYS_NULLABLE'], [['ST_Distance_Sphere'], 'DOUBLE', ['DOUBLE', 'DOUBLE', 'DOUBLE', 'DOUBLE'], - '_ZN5doris12GeoFunctions18st_distance_sphereEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_S6_S6_'], + '_ZN5doris12GeoFunctions18st_distance_sphereEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_S6_S6_', + '', '', '', ''], [['ST_AsText', 'ST_AsWKT'], 'VARCHAR', ['VARCHAR'], - '_ZN5doris12GeoFunctions9st_as_wktEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + '_ZN5doris12GeoFunctions9st_as_wktEPN9doris_udf15FunctionContextERKNS1_9StringValE', + '', '', '', 'ALWAYS_NULLABLE'], [['ST_GeometryFromText', 'ST_GeomFromText'], 'VARCHAR', ['VARCHAR'], '_ZN5doris12GeoFunctions11st_from_wktEPN9doris_udf15FunctionContextERKNS1_9StringValE', '_ZN5doris12GeoFunctions19st_from_wkt_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + '', 'ALWAYS_NULLABLE'], [['ST_LineFromText', 'ST_LineStringFromText'], 'VARCHAR', ['VARCHAR'], '_ZN5doris12GeoFunctions7st_lineEPN9doris_udf15FunctionContextERKNS1_9StringValE', '_ZN5doris12GeoFunctions15st_line_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + '', 'ALWAYS_NULLABLE'], [['ST_Polygon', 'ST_PolyFromText', 'ST_PolygonFromText'], 'VARCHAR', ['VARCHAR'], '_ZN5doris12GeoFunctions10st_polygonEPN9doris_udf15FunctionContextERKNS1_9StringValE', '_ZN5doris12GeoFunctions18st_polygon_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + '', 'ALWAYS_NULLABLE'], [['ST_Circle'], 'VARCHAR', ['DOUBLE', 'DOUBLE', 'DOUBLE'], '_ZN5doris12GeoFunctions9st_circleEPN9doris_udf15FunctionContextERKNS1_9DoubleValES6_S6_', '_ZN5doris12GeoFunctions17st_circle_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris12GeoFunctions17st_from_wkt_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + '', ''], [['ST_Contains'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], '_ZN5doris12GeoFunctions11st_containsEPN9doris_udf15FunctionContextERKNS1_9StringValES6_', '_ZN5doris12GeoFunctions19st_contains_prepareEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', - '_ZN5doris12GeoFunctions17st_contains_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE'], + '_ZN5doris12GeoFunctions17st_contains_closeEPN9doris_udf15FunctionContextENS2_18FunctionStateScopeE', + '', 'ALWAYS_NULLABLE'], # grouping sets functions [['grouping_id'], 'BIGINT', ['BIGINT'], - '_ZN5doris21GroupingSetsFunctions11grouping_idEPN9doris_udf15FunctionContextERKNS1_9BigIntValE'], - [['grouping'], 'BIGINT', ['BIGINT'], '_ZN5doris21GroupingSetsFunctions8groupingEPN9doris_udf15FunctionContextERKNS1_9BigIntValE'], + '_ZN5doris21GroupingSetsFunctions11grouping_idEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', + '', '', '', 'ALWAYS_NOT_NULLABLE'], + [['grouping'], 'BIGINT', ['BIGINT'], + '_ZN5doris21GroupingSetsFunctions8groupingEPN9doris_udf15FunctionContextERKNS1_9BigIntValE', + '' ,'', '', 'ALWAYS_NOT_NULLABLE'], ] # Except the following functions, other function will directly return @@ -917,6 +1099,31 @@ 'current_time', 'utc_timestamp' ] +# This is the subset of ALWAYS_NULLABLE +# The function belongs to @null_result_with_one_null_param_functions, +# as long as one parameter is null, the function must return null. +null_result_with_one_null_param_functions = [ + 'unix_timestamp', + 'str_to_date', + 'convert_tz', + 'pi', + 'e', + 'divide', + 'int_divide', + 'pmod', + 'mod', + 'fmod', + 'substr', + 'substring', + 'append_trailing_char_if_absent', + 'ST_X', + 'ST_Y', + 'ST_AsText', + 'ST_GeometryFromText', + 'ST_LineFromText', + 'ST_Polygon', + 'ST_Contains' +] invisible_functions = [ ] diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 7e45eed53cc2c7..26ee29f862fec2 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -35,6 +35,7 @@ package org.apache.doris.builtins;\n\ \n\ import org.apache.doris.catalog.PrimitiveType;\n\ +import org.apache.doris.catalog.Function;\n\ import org.apache.doris.catalog.FunctionSet;\n\ import com.google.common.collect.Sets;\n\ import java.util.Set;\n\ @@ -59,21 +60,32 @@ def add_function(fn_meta_data, user_visible): """add function """ - assert 4 <= len(fn_meta_data) <= 6, \ + assert len(fn_meta_data) == 8, \ "Invalid function entry in doris_builtins_functions.py:\n\t" + repr(fn_meta_data) entry = {} entry["sql_names"] = fn_meta_data[0] entry["ret_type"] = fn_meta_data[1] entry["args"] = fn_meta_data[2] entry["symbol"] = fn_meta_data[3] - if len(fn_meta_data) >= 5: + if fn_meta_data[4] != '': entry["prepare"] = fn_meta_data[4] - if len(fn_meta_data) >= 6: + if fn_meta_data[5] != '': entry["close"] = fn_meta_data[5] + if fn_meta_data[6] != '': + entry['vec'] = True + if fn_meta_data[7] != '': + entry['nullable_mode'] = fn_meta_data[7] + else: + entry['nullable_mode'] = 'DEPEND_ON_ARGUMENT' + entry["user_visible"] = user_visible meta_data_entries.append(entry) +""" +Order of params: +name, symbol, user_visible, prepare, close, nullable_mode, ret_type, has_var_args, args +""" def generate_fe_entry(entry, name): """add function """ @@ -84,13 +96,17 @@ def generate_fe_entry(entry, name): java_output += ", true" else: java_output += ", false" - if 'prepare' in entry: java_output += ', "%s"' % entry["prepare"] - if 'close' in entry: - java_output += ', "%s"' % entry["close"] - else: - java_output += ', null' + else: + java_output += ', null' + if 'close' in entry: + java_output += ', "%s"' % entry["close"] + else: + java_output += ', null' + + java_output += ", Function.NullableMode." + entry["nullable_mode"] + java_output += ", PrimitiveType." + entry["ret_type"] # Check the last entry for varargs indicator. if entry["args"] and entry["args"][-1] == "...": @@ -98,8 +114,6 @@ def generate_fe_entry(entry, name): java_output += ", true" else: java_output += ", false" - - java_output += ", PrimitiveType." + entry["ret_type"] for arg in entry["args"]: java_output += ", PrimitiveType." + arg return java_output @@ -114,15 +128,19 @@ def generate_fe_registry_init(filename): for entry in meta_data_entries: for name in entry["sql_names"]: java_output = generate_fe_entry(entry, name) - java_registry_file.write(" functionSet.addScalarBuiltin(%s);\n" % java_output) + if ("vec" not in entry): + java_registry_file.write(" functionSet.addScalarBuiltin(%s);\n" % java_output) + else: + java_registry_file.write(" functionSet.addScalarAndVectorizedBuiltin(%s);\n" % java_output) + java_registry_file.write("\n") # add non_null_result_with_null_param_functions java_registry_file.write(" Set funcNames = Sets.newHashSet();\n") - for entry in doris_builtins_functions.non_null_result_with_null_param_functions: + for entry in doris_builtins_functions.null_result_with_one_null_param_functions: java_registry_file.write(" funcNames.add(\"%s\");\n" % entry) - java_registry_file.write(" functionSet.buildNonNullResultWithNullParamFunction(funcNames);\n"); + java_registry_file.write(" functionSet.buildNullResultWithOneNullParamFunction(funcNames);\n"); # add nondeterministic functions java_registry_file.write(" Set nondeterministicFuncNames = Sets.newHashSet();\n") @@ -133,6 +151,7 @@ def generate_fe_registry_init(filename): java_registry_file.write(java_registry_epilogue) java_registry_file.close() + # Read the function metadata inputs for function in doris_builtins_functions.visible_functions: add_function(function, True) diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index b4ff8e1d993c9d..3aa1d106b92a04 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -32,7 +32,9 @@ enum TDataSinkType { EXPORT_SINK, OLAP_TABLE_SINK, MEMORY_SCRATCH_SINK, - ODBC_TABLE_SINK + ODBC_TABLE_SINK, + VRESULT_SINK, + VDATA_STREAM_SINK, } enum TResultSinkType { diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index 584706ae7dc3b0..450148f381d507 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -182,6 +182,9 @@ struct TExprNode { // If set, child[vararg_start_idx] is the first vararg child. 27: optional i32 vararg_start_idx 28: optional Types.TPrimitiveType child_type + + // For vectorized engine + 29: optional bool is_nullable } // A flattened representation of a tree of Expr nodes, obtained by depth-first diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 434c661296a839..c89ca86770f50b 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -146,6 +146,9 @@ struct TQueryOptions { // if the right table is greater than this value in the hash join, we will ignore IN filter 34: optional i32 runtime_filter_max_in_num = 1024; + + // whether enable vectorized engine + 41: optional bool enable_vectorized_engine = false } @@ -306,7 +309,7 @@ struct TExecPlanFragmentParams { 14: optional TLoadErrorHubInfo load_error_hub_info // The total number of fragments on same BE host - 15: optional i32 fragment_num_on_host; + 15: optional i32 fragment_num_on_host // If true, all @Common components is unset and should be got from BE's cache // If this field is unset or it set to false, all @Common components is set. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 029f5c4bc3658f..8576d579690da0 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -50,7 +50,7 @@ enum TPlanNodeType { ASSERT_NUM_ROWS_NODE, INTERSECT_NODE, EXCEPT_NODE, - ODBC_SCAN_NODE + ODBC_SCAN_NODE, } // phases of an execution node @@ -769,6 +769,8 @@ struct TPlanNode { 35: optional TOdbcScanNode odbc_scan_node // Runtime filters assigned to this plan node, exist in HashJoinNode and ScanNode 36: optional list runtime_filters + + 40: optional Exprs.TExpr vconjunct } // A flattened representation of a tree of PlanNodes, obtained by depth-first diff --git a/gensrc/thrift/Status.thrift b/gensrc/thrift/Status.thrift index bd3ceac9f03f6d..d0bbe8a2727c03 100644 --- a/gensrc/thrift/Status.thrift +++ b/gensrc/thrift/Status.thrift @@ -74,7 +74,17 @@ enum TStatusCode { CONFIGURATION_ERROR = 43, INCOMPLETE = 44, OLAP_ERR_VERSION_ALREADY_MERGED = 45, - DATA_QUALITY_ERROR = 46 + DATA_QUALITY_ERROR = 46, + + VEC_EXCEPTION = 50, + VEC_LOGIC_ERROR = 51, + VEC_ILLEGAL_DIVISION = 52, + VEC_BAD_CAST = 53, + VEC_CANNOT_ALLOCATE_MEMORY = 54, + VEC_CANNOT_MUNMAP = 55, + VEC_CANNOT_MREMAP = 56, + VEC_BAD_ARGUMENTS = 57, + } struct TStatus { diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index a526ec33a2a35c..1cd6598def9766 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -325,6 +325,7 @@ struct TFunction { 11: optional i64 id 12: optional string checksum + 13: optional bool vectorized = false } enum TLoadJobState { diff --git a/run-be-ut.sh b/run-be-ut.sh index 4ca5c274e1a600..590b4c4cfea369 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -73,7 +73,7 @@ fi eval set -- "$OPTS" -PARALLEL=$[$(nproc)/4+1] +PARALLEL=$[$(nproc)/2+1] CLEAN= RUN= if [ $# == 1 ] ; then diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 4afa6a9757139c..cdd028408fc8fb 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -736,6 +736,20 @@ build_parallel_hashmap() { cp -r parallel_hashmap $TP_INSTALL_DIR/include/ } +# pdqsort +build_pdqsort() { + check_if_source_exist $PDQSORT_SOURCE + cd $TP_SOURCE_DIR/$PDQSORT_SOURCE + cp -r pdqsort.h $TP_INSTALL_DIR/include/ +} + +# libdivide +build_libdivide() { + check_if_source_exist $LIBDIVIDE_SOURCE + cd $TP_SOURCE_DIR/$LIBDIVIDE_SOURCE + cp -r libdivide.h $TP_INSTALL_DIR/include/ +} + #orc build_orc() { check_if_source_exist $ORC_SOURCE @@ -936,6 +950,8 @@ build_bitshuffle build_croaringbitmap build_fmt build_parallel_hashmap +build_pdqosrt +build_libdivide build_orc build_cctz build_tsan_header diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 3e07ad96d9d3ea..6088e7e0756874 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -379,6 +379,17 @@ HDFS3_NAME="libhdfs3-master.zip" HDFS3_SOURCE="libhdfs3-master" HDFS3_MD5SUM="9e46a16009cf86b5e187d302b3582628" +#libdivide +LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/refs/tags/v4.0.0.tar.gz" +LIBDIVIDE_NAME="libdivide-4.0.0.tar.gz" +LIBDIVIDE_SOURCE="libdivide-4.0.0" +LIBDIVIDE_MD5SUM="71b99d499b9a5effd0fb2e249622b017" + +#pdqsort +PDQSORT_DOWNLOAD="http://ftp.cise.ufl.edu/ubuntu/pool/universe/p/pdqsort/pdqsort_0.0.0+git20180419.orig.tar.gz" +PDQSORT_NAME="pdqsort.tar.gz" +PDQSORT_SOURCE="pdqsort-0.0.0+git20180419" +PDQSORT_MD5SUM="39261c3e7b40aa7505662fac29f22d20" # all thirdparties which need to be downloaded is set in array TP_ARCHIVES export TP_ARCHIVES="LIBEVENT OPENSSL @@ -434,4 +445,7 @@ AWS_SDK LZMA XML2 GSASL -HDFS3" +HDFS3 +LIBDIVIDE +PDQSORT" + From 36279c0e5aec6248bcde5f9e68470c5d1f0f838b Mon Sep 17 00:00:00 2001 From: HappenLee Date: Mon, 9 Aug 2021 17:10:23 +0800 Subject: [PATCH 2/2] fix error result of agg function --- be/src/exprs/aggregate_functions.cpp | 4 ++-- .../src/main/java/org/apache/doris/catalog/Function.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp index dc066f504a244a..04b413ead710f2 100644 --- a/be/src/exprs/aggregate_functions.cpp +++ b/be/src/exprs/aggregate_functions.cpp @@ -71,13 +71,13 @@ void AggregateFunctions::init_zero_not_null(FunctionContext*, DecimalV2Val* dst) template void AggregateFunctions::init_zero(FunctionContext*, T* dst) { - dst->is_null = true; + dst->is_null = false; dst->val = 0; } template <> void AggregateFunctions::init_zero(FunctionContext*, DecimalV2Val* dst) { - dst->is_null = true; + dst->is_null = false; dst->set_to_zero(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 7802c0dbb8672f..3496f795a8bf3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -634,7 +634,6 @@ protected void writeFields(DataOutput output) throws IOException { } writeOptionString(output, libUrl); writeOptionString(output, checksum); - output.writeBoolean(vectorized); } @Override