diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index dad805a3d51d..0b3a336c8686 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -597,7 +597,6 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.EnableSequences = serviceConfig.GetEnableSequences(); kqpConfig.EnableColumnsWithDefault = serviceConfig.GetEnableColumnsWithDefault(); kqpConfig.BindingsMode = RemapBindingsMode(serviceConfig.GetBindingsMode()); - kqpConfig.PredicateExtract20 = serviceConfig.GetPredicateExtract20(); kqpConfig.IndexAutoChooserMode = serviceConfig.GetIndexAutoChooseMode(); kqpConfig.EnablePgConstsToParams = serviceConfig.GetEnablePgConstsToParams() && serviceConfig.GetEnableAstCache(); kqpConfig.ExtractPredicateRangesLimit = serviceConfig.GetExtractPredicateRangesLimit(); @@ -607,7 +606,6 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.EnableOltpSink = serviceConfig.GetEnableOltpSink(); kqpConfig.BlockChannelsMode = serviceConfig.GetBlockChannelsMode(); kqpConfig.IdxLookupJoinsPrefixPointLimit = serviceConfig.GetIdxLookupJoinPointsLimit(); - kqpConfig.OldLookupJoinBehaviour = serviceConfig.GetOldLookupJoinBehaviour(); kqpConfig.EnableSpillingGenericQuery = serviceConfig.GetEnableQueryServiceSpilling(); kqpConfig.DefaultCostBasedOptimizationLevel = serviceConfig.GetDefaultCostBasedOptimizationLevel(); diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index 8259e15f1abd..6d703f5c80a2 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -509,15 +509,12 @@ class TKqpCompileService : public TActorBootstrapped { bool enableKqpScanQuerySourceRead = TableServiceConfig.GetEnableKqpScanQuerySourceRead(); - bool predicateExtract20 = TableServiceConfig.GetPredicateExtract20(); - bool defaultSyntaxVersion = TableServiceConfig.GetSqlVersion(); auto indexAutoChooser = TableServiceConfig.GetIndexAutoChooseMode(); ui64 rangesLimit = TableServiceConfig.GetExtractPredicateRangesLimit(); ui64 idxLookupPointsLimit = TableServiceConfig.GetIdxLookupJoinPointsLimit(); - bool oldLookupJoinBehaviour = TableServiceConfig.GetOldLookupJoinBehaviour(); bool enableSequences = TableServiceConfig.GetEnableSequences(); bool enableColumnsWithDefault = TableServiceConfig.GetEnableColumnsWithDefault(); @@ -543,7 +540,6 @@ class TKqpCompileService : public TActorBootstrapped { TableServiceConfig.GetEnableKqpScanQueryStreamIdxLookupJoin() != enableKqpScanQueryStreamIdxLookupJoin || TableServiceConfig.GetEnableKqpDataQueryStreamIdxLookupJoin() != enableKqpDataQueryStreamIdxLookupJoin || TableServiceConfig.GetEnableKqpScanQuerySourceRead() != enableKqpScanQuerySourceRead || - TableServiceConfig.GetPredicateExtract20() != predicateExtract20 || TableServiceConfig.GetIndexAutoChooseMode() != indexAutoChooser || TableServiceConfig.GetEnableSequences() != enableSequences || TableServiceConfig.GetEnableColumnsWithDefault() != enableColumnsWithDefault || @@ -551,7 +547,6 @@ class TKqpCompileService : public TActorBootstrapped { TableServiceConfig.GetEnableOltpSink() != enableOltpSink || TableServiceConfig.GetEnableCreateTableAs() != enableCreateTableAs || TableServiceConfig.GetBlockChannelsMode() != blockChannelsMode || - TableServiceConfig.GetOldLookupJoinBehaviour() != oldLookupJoinBehaviour || TableServiceConfig.GetExtractPredicateRangesLimit() != rangesLimit || TableServiceConfig.GetResourceManager().GetMkqlHeavyProgramMemoryLimit() != mkqlHeavyLimit || TableServiceConfig.GetIdxLookupJoinPointsLimit() != idxLookupPointsLimit || diff --git a/ydb/core/kqp/opt/kqp_opt_impl.h b/ydb/core/kqp/opt/kqp_opt_impl.h index 8370b0994b9b..4ffce6b5ec24 100644 --- a/ydb/core/kqp/opt/kqp_opt_impl.h +++ b/ydb/core/kqp/opt/kqp_opt_impl.h @@ -4,7 +4,6 @@ #include #include -#include namespace NKikimr::NKqp::NOpt { @@ -48,12 +47,6 @@ NYql::NNodes::TKqpTable BuildTableMeta(const NYql::TKikimrTableMetadata& tableMe TIntrusivePtr GetIndexMetadata(const NYql::NNodes::TKqlReadTableIndex& index, const NYql::TKikimrTablesData& tables, TStringBuf cluster); -bool KqpTableLookupCanCompare(NYql::NNodes::TExprBase node); -NYql::NNodes::TMaybeNode KqpTableLookupGetValue(NYql::NNodes::TExprBase node, - const NYql::TTypeAnnotationNode* type, NYql::TExprContext& ctx); -NYql::NCommon::TTableLookup::TCompareResult KqpTableLookupCompare(NYql::NNodes::TExprBase left, - NYql::NNodes::TExprBase right); - TVector> BuildSecondaryIndexVector( const NYql::TKikimrTableDescription& table, NYql::TPositionHandle pos, NYql::TExprContext& ctx, const THashSet* filter, diff --git a/ydb/core/kqp/opt/kqp_opt_range_legacy.cpp b/ydb/core/kqp/opt/kqp_opt_range_legacy.cpp deleted file mode 100644 index 163bce1e4bc8..000000000000 --- a/ydb/core/kqp/opt/kqp_opt_range_legacy.cpp +++ /dev/null @@ -1,256 +0,0 @@ -#include "kqp_opt_impl.h" - -#include - -namespace NKikimr::NKqp::NOpt { - -using namespace NYql; -using namespace NYql::NNodes; -using namespace NYql::NCommon; - -namespace { - -template -TTableLookup::TCompareResult::TResult CompareValues(const T& left, const T& right) { - if (left == right) { - return TTableLookup::TCompareResult::Equal; - } else { - return left > right - ? TTableLookup::TCompareResult::Greater - : TTableLookup::TCompareResult::Less; - } -} - -template -TTableLookup::TCompareResult CompareIntegralNodes(TCoAtom left, TCoAtom right, NKikimr::NUdf::EDataSlot slot) { - T leftValue = FromString(left.Ref(), slot); - T rightValue = FromString(right.Ref(), slot); - auto compareResult = CompareValues(leftValue, rightValue); - - TMaybe adjacent; - switch (compareResult) { - case TTableLookup::TCompareResult::Equal: - break; - - case TTableLookup::TCompareResult::Greater: - adjacent = leftValue == rightValue + 1; - break; - - case TTableLookup::TCompareResult::Less: - adjacent = rightValue == leftValue + 1; - break; - } - - return TTableLookup::TCompareResult(compareResult, adjacent); -} - -template -TTableLookup::TCompareResult CompareNodes(TCoAtom left, TCoAtom right, NKikimr::NUdf::EDataSlot slot) { - T leftValue = FromString(left.Ref(), slot); - T rightValue = FromString(right.Ref(), slot); - return TTableLookup::TCompareResult(CompareValues(leftValue, rightValue)); -} - -template<> -TTableLookup::TCompareResult CompareNodes(TCoAtom left, TCoAtom right, NKikimr::NUdf::EDataSlot slot) { - bool leftValue = FromString(left.Ref(), slot); - bool rightValue = FromString(right.Ref(), slot); - auto compareResult = CompareValues(leftValue, rightValue); - - return TTableLookup::TCompareResult(compareResult); -} - -template<> -TTableLookup::TCompareResult CompareNodes(TCoAtom left, TCoAtom right, NKikimr::NUdf::EDataSlot slot) { - return CompareIntegralNodes(left, right, slot); -} - -template<> -TTableLookup::TCompareResult CompareNodes(TCoAtom left, TCoAtom right, NKikimr::NUdf::EDataSlot slot) { - return CompareIntegralNodes(left, right, slot); -} - -template<> -TTableLookup::TCompareResult CompareNodes(TCoAtom left, TCoAtom right, NKikimr::NUdf::EDataSlot slot) { - Y_UNUSED(slot); - - const auto& leftValue = left.Value(); - const auto& rightValue = right.Value(); - return TTableLookup::TCompareResult(CompareValues(leftValue, rightValue)); -} - -} // namespace - -bool KqpTableLookupCanCompare(TExprBase node) { - if (node.Maybe()) { - return true; - } - - if (node.Maybe()) { - return true; - } - - if (node.Maybe()) { - return true; - } - - if (node.Maybe()) { - return true; - } - - return false; -} - -TMaybeNode KqpTableLookupGetValue(TExprBase node, const TTypeAnnotationNode* type, - TExprContext& ctx) -{ - const TTypeAnnotationNode* targetType = type; - bool isTargetOptional = false; - if (type->GetKind() == ETypeAnnotationKind::Optional) { - targetType = type->Cast()->GetItemType(); - isTargetOptional = true; - } - - if (targetType->GetKind() != ETypeAnnotationKind::Data) { - return TMaybeNode(); - } - - THashSet knownArgs; - bool canPush = true; - VisitExpr(node.Ptr(), [&knownArgs, &canPush] (const TExprNode::TPtr& exprNode) { - auto node = TExprBase(exprNode); - - if (!canPush) { - return false; - } - - if (auto maybeLambda = node.Maybe()) { - for (const auto& arg : maybeLambda.Cast().Args()) { - knownArgs.emplace(arg.Raw()); - } - } - - if (auto maybeArg = node.Maybe()) { - if (!knownArgs.contains(maybeArg.Cast().Raw())) { - canPush = false; - return false; - } - } - - return true; - }); - - if (!canPush) { - return TMaybeNode(); - } - - const auto& dataTypeName = targetType->Cast()->GetName(); - - TExprBase valueNode = node; - if (isTargetOptional) { - if (auto maybeJust = node.Maybe()) { - valueNode = maybeJust.Cast().Input(); - } - - if (node.Maybe()) { - return Build(ctx, node.Pos()) - .OptionalType(ExpandType(node.Pos(), *type, ctx)) - .Done() - .Ptr(); - } - } - - TExprNode::TPtr literal; - if (auto maybeInt = valueNode.Maybe()) { - if (maybeInt.Cast().CallableName() == dataTypeName) { - return valueNode; - } - - if (AllowIntegralConversion(maybeInt.Cast(), false, NKikimr::NUdf::GetDataSlot(dataTypeName))) { - literal = maybeInt.Cast().Literal().Ptr(); - } - } - - if (auto maybeString = valueNode.Maybe()) { - if (dataTypeName == "String") { - return valueNode; - } - - if (dataTypeName == "Utf8") { - auto atom = maybeString.Cast().Literal(); - auto value = atom.Value(); - if (!IsUtf8(value)) { - return {}; - } - - literal = atom.Ptr(); - } - } - - if (auto maybeUtf8 = valueNode.Maybe()) { - if (dataTypeName == "String" || dataTypeName == "Utf8") { - literal = maybeUtf8.Cast().Literal().Ptr(); - } - } - - if (auto maybeBool = valueNode.Maybe()) { - if (dataTypeName == "Bool") { - literal = maybeBool.Cast().Literal().Ptr(); - } - } - - if (literal) { - auto ret = ctx.Builder(valueNode.Pos()) - .Callable(dataTypeName) - .Add(0, literal) - .Seal() - .Build(); - - return ret; - } - - auto valueType = valueNode.Ref().GetTypeAnn(); - if (isTargetOptional && valueType->GetKind() == ETypeAnnotationKind::Optional) { - valueType = valueType->Cast()->GetItemType(); - } - - if (valueType->GetKind() == ETypeAnnotationKind::Data && - valueType->Cast()->GetName() == dataTypeName) - { - return node; - } - - return Build(ctx, node.Pos()) - .Input(node) - .Type().Build(dataTypeName) - .Done(); -} - -TTableLookup::TCompareResult KqpTableLookupCompare(TExprBase left, TExprBase right) { - if (left.Maybe() && right.Maybe()) { - return CompareNodes(left.Cast().Literal(), - right.Cast().Literal(), NKikimr::NUdf::EDataSlot::Bool); - } - - if (left.Maybe() && right.Maybe()) { - return CompareNodes(left.Cast().Literal(), - right.Cast().Literal(), NKikimr::NUdf::EDataSlot::Uint64); - } - - if (left.Maybe() && right.Maybe()) { - return CompareNodes(left.Cast().Literal(), - right.Cast().Literal(), NKikimr::NUdf::EDataSlot::Int64); - } - - if (left.Maybe() && right.Maybe() || - left.Maybe() && right.Maybe()) - { - return CompareNodes(left.Cast().Literal(), - right.Cast().Literal(), NKikimr::NUdf::EDataSlot::String); - } - - YQL_ENSURE(false, "Unexpected nodes in Kikimr TableLookupCompare: (" << left.Ref().Content() - << ", " << right.Ref().Content() << ")"); -} - -} // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp index fa5636bf6f78..4734288dcdac 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log.cpp @@ -30,7 +30,6 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { , Config(config) { #define HNDL(name) "KqpLogical-"#name, Hndl(&TKqpLogicalOptTransformer::name) - AddHandler(0, &TCoFlatMapBase::Match, HNDL(PushPredicateToReadTable)); AddHandler(0, &TCoFlatMapBase::Match, HNDL(PushExtractedPredicateToReadTable)); AddHandler(0, &TCoAggregate::Match, HNDL(RewriteAggregate)); AddHandler(0, &TCoAggregateCombine::Match, HNDL(PushdownOlapGroupByKeys)); @@ -62,7 +61,6 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { AddHandler(0, &TCoNarrowMultiMap::Match, HNDL(DqReadWideWrapFieldSubset)); AddHandler(0, &TCoWideMap::Match, HNDL(DqReadWideWrapFieldSubset)); - AddHandler(1, &TCoFlatMap::Match, HNDL(LatePushExtractedPredicateToReadTable)); AddHandler(1, &TCoTop::Match, HNDL(RewriteTopSortOverIndexRead)); AddHandler(1, &TCoTopSort::Match, HNDL(RewriteTopSortOverIndexRead)); AddHandler(1, &TCoTake::Match, HNDL(RewriteTakeOverIndexRead)); @@ -88,28 +86,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } protected: - TMaybeNode PushPredicateToReadTable(TExprBase node, TExprContext& ctx) { - if (KqpCtx.Config->PredicateExtract20) { - return node; - } - TExprBase output = KqpPushPredicateToReadTable(node, ctx, KqpCtx); - DumpAppliedRule("PushPredicateToReadTable", node.Ptr(), output.Ptr(), ctx); - return output; - } - TMaybeNode PushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx, const TGetParents& getParents) { - if (!KqpCtx.Config->PredicateExtract20) { - return node; - } - TExprBase output = KqpPushExtractedPredicateToReadTable(node, ctx, KqpCtx, TypesCtx, *getParents()); - DumpAppliedRule("PushExtractedPredicateToReadTable", node.Ptr(), output.Ptr(), ctx); - return output; - } - - TMaybeNode LatePushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx, const TGetParents& getParents) { - if (KqpCtx.Config->PredicateExtract20) { - return node; - } TExprBase output = KqpPushExtractedPredicateToReadTable(node, ctx, KqpCtx, TypesCtx, *getParents()); DumpAppliedRule("PushExtractedPredicateToReadTable", node.Ptr(), output.Ptr(), ctx); return output; diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp index 22fd070e2194..078fca2f5318 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_extract.cpp @@ -6,7 +6,6 @@ #include #include -#include namespace NKikimr::NKqp::NOpt { diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp index 2309e97b67ef..a723be2aaa6a 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_join.cpp @@ -569,10 +569,6 @@ TMaybeNode KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext return {}; } - if ((!kqpCtx.Config->PredicateExtract20 || kqpCtx.Config->OldLookupJoinBehaviour) && prefixLookup->Filter.IsValid()) { - return {}; - } - TMap rightJoinKeyToLeft; TVector rightKeyColumns; rightKeyColumns.reserve(join.JoinKeys().Size()); @@ -632,9 +628,6 @@ TMaybeNode KqpJoinToIndexLookupImpl(const TDqJoin& join, TExprContext .Build() .Done()); deduplicateLeftColumns.insert(*leftColumn); - if ((!kqpCtx.Config->PredicateExtract20 || kqpCtx.Config->OldLookupJoinBehaviour)) { - return {}; - } } member = Build(ctx, prefixRowArg.Pos()) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp index a5eba5e404a5..8e163757c217 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges.cpp @@ -7,334 +7,14 @@ #include #include -#include namespace NKikimr::NKqp::NOpt { -namespace { - using namespace NYql; using namespace NYql::NCommon; using namespace NYql::NDq; using namespace NYql::NNodes; -TExprBase BuildEquiRangeLookup(const TKeyRange& keyRange, const TKikimrTableDescription& tableDesc, - TPositionHandle pos, TExprContext& ctx) -{ - YQL_ENSURE(keyRange.IsEquiRange()); - - TVector structMembers; - TVector skipNullColumns; - for (ui32 i = 0; i < keyRange.GetNumDefined(); ++i) { - const auto& columnName = tableDesc.Metadata->KeyColumnNames[i]; - TCoAtom columnNameAtom(ctx.NewAtom(pos, columnName)); - auto value = keyRange.GetFromTuple().GetValue(i).Cast(); - - if (TCoNull::Match(value.Raw())) { - value = Build(ctx, pos) - .OptionalType(NCommon::BuildTypeExpr(pos, *tableDesc.GetColumnType(columnName), ctx)) - .Done(); - } else { - skipNullColumns.push_back(columnNameAtom); - } - - auto member = Build(ctx, pos) - .Add(columnNameAtom) - .Add(value) - .Done(); - - structMembers.push_back(member); - } - - auto keysToLookup = Build(ctx, pos) - .Add() - .Add(structMembers) - .Build() - .Done(); - - // Actually residual predicate for the key range already has a check for NULL keys, - // but it's better to skip redundant lookup. Consider removing check from residual - // predicate in this case. - return Build(ctx, pos) - .Input(keysToLookup) - .Members() - .Add(skipNullColumns) - .Build() - .Done(); -} - -} // namespace - -TKqlKeyRange BuildKeyRangeExpr(const TKeyRange& keyRange, const TKikimrTableDescription& tableDesc, - TPositionHandle pos, TExprContext& ctx) -{ - bool fromInclusive = true; - bool toInclusive = true; - TVector fromValues; - TVector toValues; - - for (size_t i = 0; i < keyRange.GetColumnRangesCount(); ++i) { - const auto& columnName = tableDesc.Metadata->KeyColumnNames[i]; - const auto& range = keyRange.GetColumnRange(i); - - if (range.GetFrom().IsDefined()) { - fromInclusive = range.GetFrom().IsInclusive(); - if (TCoNull::Match(range.GetFrom().GetValue().Raw())) { - fromValues.emplace_back( - Build(ctx, pos) - .OptionalType(NCommon::BuildTypeExpr(pos, *tableDesc.GetColumnType(columnName), ctx)) - .Done()); - } else { - fromValues.emplace_back(range.GetFrom().GetValue()); - } - } - - if (range.GetTo().IsDefined()) { - toInclusive = range.GetTo().IsInclusive(); - if (TCoNull::Match(range.GetTo().GetValue().Raw())) { - toValues.emplace_back( - Build(ctx, pos) - .OptionalType(NCommon::BuildTypeExpr(pos, *tableDesc.GetColumnType(columnName), ctx)) - .Done()); - } else { - toValues.emplace_back(range.GetTo().GetValue()); - } - } - } - - auto fromExpr = fromInclusive - ? Build(ctx, pos).Add(fromValues).Done().Cast() - : Build(ctx, pos).Add(fromValues).Done().Cast(); - - auto toExpr = toInclusive - ? Build(ctx, pos).Add(toValues).Done().Cast() - : Build(ctx, pos).Add(toValues).Done().Cast(); - - return Build(ctx, pos) - .From(fromExpr) - .To(toExpr) - .Done(); -} - -bool IsPointPrefix(const TKeyRange& range) { - size_t prefixLen = 0; - for (size_t i = 0; i < range.GetColumnRangesCount(); ++i) { - if (range.GetColumnRange(i).IsPoint() && i == prefixLen) { - prefixLen += 1; - } - if (range.GetColumnRange(i).IsDefined() && i >= prefixLen) { - return false; - } - } - return prefixLen > 0; -} - -TExprBase KqpPushPredicateToReadTable(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - if (!node.Maybe()) { - return node; - } - auto flatmap = node.Cast(); - - if (!IsPredicateFlatMap(flatmap.Lambda().Body().Ref())) { - return node; - } - - bool onlyPointRanges = false; - auto readMatch = MatchRead(flatmap.Input()); - TMaybeNode indexName; - - //TODO: remove this branch KIKIMR-15255, KIKIMR-15321 - if (!readMatch && (kqpCtx.IsDataQuery() || kqpCtx.IsGenericQuery())) { - if (auto readRangesMatch = MatchRead(flatmap.Input())) { - auto read = readRangesMatch->Read.Cast(); - if (TCoVoid::Match(read.Ranges().Raw())) { - auto key = Build(ctx, read.Pos()).Done(); - readMatch = readRangesMatch; - readMatch->Read = - Build(ctx, read.Pos()) - .Settings(read.Settings()) - .Table(read.Table()) - .Columns(read.Columns()) - .Range() - .From(key) - .To(key) - .Build() - .Done(); - onlyPointRanges = true; - if (auto indexRead = read.Maybe()) { - indexName = indexRead.Index(); - } - } else { - return node; - } - } else { - return node; - } - } - - if (!readMatch) { - return node; - } - - if (readMatch->FlatMap) { - return node; - } - - auto read = readMatch->Read.Cast(); - - static const std::set supportedReads { - TKqlReadTable::CallableName(), - TKqlReadTableIndex::CallableName(), - }; - - if (!supportedReads.contains(read.CallableName())) { - return node; - } - - if (auto maybeIndexRead = read.Maybe()) { - indexName = maybeIndexRead.Cast().Index(); - } - - if (read.Range().From().ArgCount() > 0 || read.Range().To().ArgCount() > 0) { - return node; - } - - auto& mainTableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, read.Table().Path()); - - auto& tableDesc = indexName ? kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) : mainTableDesc; - - if (tableDesc.Metadata->Kind == EKikimrTableKind::Olap) { - return node; - } - - auto row = flatmap.Lambda().Args().Arg(0); - auto predicate = TExprBase(flatmap.Lambda().Body().Ref().ChildPtr(0)); - TTableLookup lookup = ExtractTableLookup(row, predicate, tableDesc.Metadata->KeyColumnNames, - &KqpTableLookupGetValue, &KqpTableLookupCanCompare, &KqpTableLookupCompare, ctx, false); - - if (lookup.IsFullScan()) { - return node; - } - - auto readSettings = TKqpReadTableSettings::Parse(read); - - TVector fetches; - fetches.reserve(lookup.GetKeyRanges().size()); - - for (auto& keyRange : lookup.GetKeyRanges()) { - bool useDataOrGenericQueryLookup = false; - bool useScanQueryLookup = false; - if (onlyPointRanges && !IsPointPrefix(keyRange)) { - return node; - } - if (keyRange.IsEquiRange()) { - bool isFullKey = keyRange.GetNumDefined() == tableDesc.Metadata->KeyColumnNames.size(); - - // NOTE: Use more efficient full key lookup implementation in datashard. - // Consider using lookup for partial keys as well once better constant folding - // is available, currently it can introduce redundant compute stage. - useDataOrGenericQueryLookup = (kqpCtx.IsDataQuery() || kqpCtx.IsGenericQuery()) && isFullKey; - useScanQueryLookup = kqpCtx.IsScanQuery() && isFullKey - && kqpCtx.Config->EnableKqpScanQueryStreamLookup; - } - - TMaybeNode readInput; - // TODO: Use single implementation for all kinds of queries. - if (useDataOrGenericQueryLookup) { - auto lookupKeys = BuildEquiRangeLookup(keyRange, tableDesc, read.Pos(), ctx); - - if (indexName) { - readInput = Build(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(lookupKeys) - .Columns(read.Columns()) - .Index(indexName.Cast()) - .Done(); - } else { - readInput = Build(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(lookupKeys) - .Columns(read.Columns()) - .Done(); - } - } else if (useScanQueryLookup) { - YQL_ENSURE(kqpCtx.Config->EnableKqpScanQueryStreamLookup); - auto lookupKeys = BuildEquiRangeLookup(keyRange, tableDesc, read.Pos(), ctx); - - if (indexName) { - readInput = Build(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(lookupKeys) - .Columns(read.Columns()) - .Index(indexName.Cast()) - .Done(); - } else { - readInput = Build(ctx, read.Pos()) - .Table(read.Table()) - .LookupKeys(lookupKeys) - .Columns(read.Columns()) - .LookupStrategy().Build(TKqpStreamLookupStrategyName) - .Done(); - } - } else { - auto keyRangeExpr = BuildKeyRangeExpr(keyRange, tableDesc, node.Pos(), ctx); - - TKqpReadTableSettings settings = readSettings; - for (size_t i = 0; i < keyRange.GetColumnRangesCount(); ++i) { - const auto& column = tableDesc.Metadata->KeyColumnNames[i]; - auto& range = keyRange.GetColumnRange(i); - if (range.IsDefined() && !range.IsNull()) { - settings.AddSkipNullKey(column); - } - } - - if (indexName) { - readInput = Build(ctx, read.Pos()) - .Table(read.Table()) - .Range(keyRangeExpr) - .Columns(read.Columns()) - .Index(indexName.Cast()) - .Settings(settings.BuildNode(ctx, read.Pos())) - .Done(); - } else { - readInput = Build(ctx, read.Pos()) - .Table(read.Table()) - .Range(keyRangeExpr) - .Columns(read.Columns()) - .Settings(settings.BuildNode(ctx, read.Pos())) - .Done(); - } - } - - auto input = readInput.Cast(); - - auto residualPredicate = keyRange.GetResidualPredicate() - ? keyRange.GetResidualPredicate().Cast().Ptr() - : MakeBool(node.Pos(), ctx); - - auto newBody = ctx.ChangeChild(flatmap.Lambda().Body().Ref(), 0, std::move(residualPredicate)); - - input = readMatch->BuildProcessNodes(input, ctx); - - auto fetch = Build(ctx, node.Pos()) - .Input(input) - .Lambda() - .Args({"item"}) - .Body() - .Apply(TExprBase(newBody)) - .With(flatmap.Lambda().Args().Arg(0), "item") - .Build() - .Build() - .Done(); - - fetches.push_back(fetch); - } - - return Build(ctx, node.Pos()) - .Add(fetches) - .Done(); -} - TMaybeNode KqpRewriteLiteralLookup(const TExprBase& node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { if (!node.Maybe()) { return {}; diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp index bcc8a8cf7150..d5cda7cb02c4 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_ranges_predext.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -44,139 +43,6 @@ bool IsValidForRange(const NYql::TExprNode::TPtr& node) { return true; } -TMaybeNode TryBuildTrivialReadTable(TCoFlatMap& flatmap, TKqlReadTableRangesBase readTable, - const TKqpMatchReadResult& readMatch, const TKikimrTableDescription& tableDesc, TExprContext& ctx, - const TKqpOptimizeContext& kqpCtx, TMaybeNode indexName) -{ - Y_UNUSED(kqpCtx); - - switch (tableDesc.Metadata->Kind) { - case EKikimrTableKind::Datashard: - case EKikimrTableKind::SysView: - break; - case EKikimrTableKind::Olap: - case EKikimrTableKind::External: - case EKikimrTableKind::Unspecified: - return {}; - case EKikimrTableKind::View: - YQL_ENSURE(false, "All views should have been rewritten at this stage."); - } - - auto row = flatmap.Lambda().Args().Arg(0); - auto predicate = TExprBase(flatmap.Lambda().Body().Ref().ChildPtr(0)); - TTableLookup lookup = ExtractTableLookup(row, predicate, tableDesc.Metadata->KeyColumnNames, - &KqpTableLookupGetValue, &KqpTableLookupCanCompare, &KqpTableLookupCompare, ctx, false); - - if (lookup.IsFullScan()) { - return {}; - } - - if (lookup.GetKeyRanges().size() > 1) { - return {}; // optimize trivial cases only - } - - auto isTrivialExpr = [](const TExprBase& expr) { - if (!expr.Maybe()) { - return false; - } - auto opt = expr.Cast().Optional(); - if (opt.Maybe()) { - return true; - } - if (opt.Maybe()) { - return opt.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Data; - } - return false; - }; - - auto isTrivialPredicate = [&isTrivialExpr](const TExprBase& expr) { - if (isTrivialExpr(expr)) { - return true; - } - if (expr.Maybe()) { - for (auto& predicate : expr.Cast().Args()) { - if (!isTrivialExpr(TExprBase(predicate))) { - return false; - } - } - return true; - } - return false; - }; - - TVector fetches; - fetches.reserve(lookup.GetKeyRanges().size()); - auto readSettings = TKqpReadTableSettings::Parse(readTable); - - for (const auto& keyRange : lookup.GetKeyRanges()) { - if (keyRange.HasResidualPredicate()) { - // In trivial cases the residual predicate look like: - // * (Exists ) - // * (And (Exists ) (Exists Key2Value) ...) - // where `KeyValue` is either explicit `Data` (so `Exists` is always true) - // or Parameter value (in that case we ensure that type is not optional) - if (!isTrivialPredicate(keyRange.GetResidualPredicate().Cast())) { - return {}; - } - } - - auto keyRangeExpr = BuildKeyRangeExpr(keyRange, tableDesc, readTable.Pos(), ctx); - - TKqpReadTableSettings settings = readSettings; - for (size_t i = 0; i < keyRange.GetColumnRangesCount(); ++i) { - const auto& column = tableDesc.Metadata->KeyColumnNames[i]; - auto& range = keyRange.GetColumnRange(i); - if (range.IsDefined() && !range.IsNull()) { - settings.AddSkipNullKey(column); - } - } - - auto buildReadTable = [&] () -> TExprBase { - return Build(ctx, readTable.Pos()) - .Table(readTable.Table()) - .Range(keyRangeExpr) - .Columns(readTable.Columns()) - .Settings(settings.BuildNode(ctx, readTable.Pos())) - .Done(); - }; - - auto buildReadIndex = [&] () -> TExprBase { - return Build(ctx, readTable.Pos()) - .Table(readTable.Table()) - .Range(keyRangeExpr) - .Columns(readTable.Columns()) - .Settings(settings.BuildNode(ctx, readTable.Pos())) - .Index(indexName.Cast()) - .Done(); - }; - - TExprBase input = indexName.IsValid() ? buildReadIndex() : buildReadTable(); - - input = readMatch.BuildProcessNodes(input, ctx); - - input = Build(ctx, readTable.Pos()) - .Input(input) - .Lambda() - .Args({"item"}) - .Body() - .Apply(TExprBase(ctx.ChangeChild(flatmap.Lambda().Body().Ref(), 0, MakeBool(readTable.Pos(), ctx)))) - .With(flatmap.Lambda().Args().Arg(0), "item") - .Build() - .Build() - .Done(); - - if (lookup.GetKeyRanges().size() == 1) { - return input; - } - - fetches.emplace_back(input); - } - - return Build(ctx, readTable.Pos()) - .Add(fetches) - .Done(); -} - } // namespace TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, @@ -239,6 +105,7 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx settings.MergeAdjacentPointRanges = true; settings.HaveNextValueCallable = true; settings.BuildLiteralRange = true; + settings.IsValidForRange = IsValidForRange; if (kqpCtx.Config->ExtractPredicateRangesLimit != 0) { settings.MaxRanges = kqpCtx.Config->ExtractPredicateRangesLimit; @@ -246,22 +113,6 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx settings.MaxRanges = Nothing(); } - if (!kqpCtx.Config->PredicateExtract20) { - // test for trivial cases (explicit literals or parameters) - auto& tableDesc = indexName - ? kqpCtx.Tables->ExistingTable( - kqpCtx.Cluster, - mainTableDesc.Metadata->GetIndexMetadata(TString(indexName.Cast())).first->Name) - : mainTableDesc; - YQL_ENSURE(node.Maybe(), "got OrderedFlatMap with disabled PredicateExtract20"); - auto flatmap = node.Cast(); - if (auto expr = TryBuildTrivialReadTable(flatmap, read, *readMatch, tableDesc, ctx, kqpCtx, indexName)) { - return expr.Cast(); - } - } else { - settings.IsValidForRange = IsValidForRange; - } - auto extractor = MakePredicateRangeExtractor(settings); YQL_ENSURE(mainTableDesc.SchemeNode); @@ -373,8 +224,7 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx YQL_CLOG(DEBUG, ProviderKqp) << "Residual lambda: " << KqpExprToPrettyString(*residualLambda, ctx); TMaybe input; - if (kqpCtx.Config->PredicateExtract20 && - (tableDesc.Metadata->Kind == EKikimrTableKind::Datashard || + if ((tableDesc.Metadata->Kind == EKikimrTableKind::Datashard || tableDesc.Metadata->Kind == EKikimrTableKind::SysView)) { auto buildLookup = [&] (TExprNode::TPtr keys, TMaybe& result) { @@ -474,27 +324,25 @@ TExprBase KqpPushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx TMaybeNode prefix; TMaybeNode predicateExpr; TMaybeNode usedColumnsList; - if (kqpCtx.Config->PredicateExtract20) { - prefix = prefixPointsExpr; - if (prefix) { - predicateExpr = ctx.DeepCopyLambda(flatmap.Lambda().Ref()); - TSet usedColumns; - if (!ExtractUsedFields( - flatmap.Lambda().Body().Ptr(), - flatmap.Lambda().Args().Arg(0).Ref(), - usedColumns, - parentsMap, - true)) - { - prefix = {}; - predicateExpr = {}; - } else { - TVector columnAtoms; - for (auto&& column : usedColumns) { - columnAtoms.push_back(Build(ctx, read.Pos()).Value(column).Done()); - } - usedColumnsList = Build(ctx, read.Pos()).Add(columnAtoms).Done(); + prefix = prefixPointsExpr; + if (prefix) { + predicateExpr = ctx.DeepCopyLambda(flatmap.Lambda().Ref()); + TSet usedColumns; + if (!ExtractUsedFields( + flatmap.Lambda().Body().Ptr(), + flatmap.Lambda().Args().Arg(0).Ref(), + usedColumns, + parentsMap, + true)) + { + prefix = {}; + predicateExpr = {}; + } else { + TVector columnAtoms; + for (auto&& column : usedColumns) { + columnAtoms.push_back(Build(ctx, read.Pos()).Value(column).Done()); } + usedColumnsList = Build(ctx, read.Pos()).Add(columnAtoms).Done(); } } diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h index 40a6ed4f0b01..703f2230e157 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_rules.h @@ -18,9 +18,6 @@ namespace NKikimr::NKqp::NOpt { NYql::NNodes::TKqlKeyRange BuildKeyRangeExpr(const NYql::NCommon::TKeyRange& keyRange, const NYql::TKikimrTableDescription& tableDesc, NYql::TPositionHandle pos, NYql::TExprContext& ctx); -NYql::NNodes::TExprBase KqpPushPredicateToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext &ctx, - const TKqpOptimizeContext &kqpCtx); - NYql::NNodes::TExprBase KqpPushExtractedPredicateToReadTable(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, NYql::TTypeAnnotationContext& typesCtx, const NYql::TParentsMap& parentsMap); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp index ecd2e664d49a..b6a1a3f158b2 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_sqlin.cpp @@ -90,10 +90,6 @@ TExprBase KqpRewriteSqlInToEquiJoin(const TExprBase& node, TExprContext& ctx, co return node; } - if ((!kqpCtx.Config->PredicateExtract20 || kqpCtx.Config->OldLookupJoinBehaviour) && pointSelection->Filter.IsValid()) { - return node; - } - fixedPrefixLen = pointSelection->PrefixSize; tableDesc = &kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, pointSelection->LookupTableName); diff --git a/ydb/core/kqp/opt/ya.make b/ydb/core/kqp/opt/ya.make index 6f07040d1180..312ee15f0425 100644 --- a/ydb/core/kqp/opt/ya.make +++ b/ydb/core/kqp/opt/ya.make @@ -9,7 +9,6 @@ SRCS( kqp_opt_phase.cpp kqp_opt_phy_check.cpp kqp_opt_phy_finalize.cpp - kqp_opt_range_legacy.cpp kqp_query_blocks_transformer.cpp kqp_query_plan.cpp kqp_statistics_transformer.cpp diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index fc2577749ebd..3ce736201db7 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -7,7 +7,6 @@ #include #include -#include namespace NYql { diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index e1a774630c0f..fa544f7c7f02 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -151,7 +151,6 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnableKqpDataQueryStreamLookup = false; bool EnableKqpScanQueryStreamIdxLookupJoin = false; bool EnableKqpDataQueryStreamIdxLookupJoin = false; - bool PredicateExtract20 = false; bool EnablePreparedDdl = false; bool EnableSequences = false; bool EnableColumnsWithDefault = false; @@ -163,7 +162,6 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnablePerStatementQueryExecution = false; bool EnableCreateTableAs = false; ui64 IdxLookupJoinsPrefixPointLimit = 1; - bool OldLookupJoinBehaviour = true; bool EnableOlapSink = false; bool EnableOltpSink = false; NKikimrConfig::TTableServiceConfig_EBlockChannelsMode BlockChannelsMode; diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index 9d65f8fd11d8..210701f1043f 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -159,13 +159,12 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { } } - void TestNoReadFromMainTableBeforeJoin(bool UseExtractPredicates) { + void TestNoReadFromMainTableBeforeJoin() { using namespace NYql; using namespace NYql::NNodes; TKikimrSettings settings; NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetPredicateExtract20(UseExtractPredicates); settings.SetAppConfig(appConfig); TKikimrRunner kikimr(settings); @@ -259,8 +258,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) { } } - Y_UNIT_TEST_TWIN(TestNoReadFromMainTableBeforeJoin, ExtractPredicate) { - TestNoReadFromMainTableBeforeJoin(ExtractPredicate); + Y_UNIT_TEST(TestNoReadFromMainTableBeforeJoin) { + TestNoReadFromMainTableBeforeJoin(); } Y_UNIT_TEST(HandleWriteOnlyIndex) { diff --git a/ydb/core/kqp/ut/join/kqp_join_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_ut.cpp index 207d570e5779..1208d6ff9dcf 100644 --- a/ydb/core/kqp/ut/join/kqp_join_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_ut.cpp @@ -1570,7 +1570,6 @@ Y_UNIT_TEST_SUITE(KqpJoin) { Y_UNIT_TEST_TWIN(AllowJoinsForComplexPredicates, StreamLookup) { NKikimrConfig::TAppConfig appConfig; appConfig.MutableTableServiceConfig()->SetEnableKqpDataQueryStreamIdxLookupJoin(StreamLookup); - appConfig.MutableTableServiceConfig()->SetOldLookupJoinBehaviour(false); appConfig.MutableTableServiceConfig()->SetIdxLookupJoinPointsLimit(10); //appConfig.MutableTableServiceConfig()->SetEnableKqpDataQueryStreamLookup(false); diff --git a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp index a369f4caba4e..5f633c1d2d33 100644 --- a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp @@ -3825,73 +3825,6 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { FormatResultSetYson(result.GetResultSet(0))); } - Y_UNIT_TEST(OddSkipNullKeys) { - TKikimrSettings settings; - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetPredicateExtract20(false); - settings.SetAppConfig(appConfig); - - TKikimrRunner kikimr(settings); - auto db = kikimr.GetTableClient(); - - { - auto session = db.CreateSession().GetValueSync().GetSession(); - AssertSuccessResult(session.ExecuteSchemeQuery(R"( - --!syntax_v1 - - CREATE TABLE `/Root/tickets` ( - entity_id Utf8, - updated_at_desc Uint64, - state Utf8, - access_type Utf8, - entity_type Utf8, - id Utf8, - iam_user_id Utf8, - PRIMARY KEY (id, entity_type, access_type, state, updated_at_desc, entity_id, iam_user_id) - ); - - )").GetValueSync()); - - AssertSuccessResult(session.ExecuteDataQuery(R"( - REPLACE INTO `/Root/tickets` (entity_id, updated_at_desc, state, access_type, entity_type, id, iam_user_id) VALUES - (null, 0, "state", "access", "type", "id", "iam_id"); - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync()); - - } - - auto params = - TParamsBuilder() - .AddParam("$id_6").OptionalUtf8("id").Build() - .AddParam("$iam_user_id_3").OptionalUtf8("iam_id").Build() - .AddParam("$state_3").OptionalUtf8("state").Build() - .AddParam("$updated_at_desc_2").OptionalUint64(0).Build() - .AddParam("$access_type_2").OptionalUtf8("access").Build() - .AddParam("$entity_type_0").OptionalUtf8("type").Build() - .Build(); - - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteDataQuery(R"( - --!syntax_v1 - DECLARE $access_type_2 AS Optional; - DECLARE $entity_type_0 AS Optional; - DECLARE $iam_user_id_3 AS Optional; - DECLARE $id_6 AS Optional; - DECLARE $state_3 AS Optional; - DECLARE $updated_at_desc_2 AS Optional; - - SELECT id FROM `/Root/tickets` - WHERE access_type = $access_type_2 - AND entity_id IS NULL - AND entity_type = $entity_type_0 - AND iam_user_id = $iam_user_id_3 - AND id = $id_6 - AND state = $state_3 - AND updated_at_desc = $updated_at_desc_2; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params).GetValueSync(); - - CompareYson(R"([[["id"]]])", FormatResultSetYson(result.GetResultSet(0))); - } - Y_UNIT_TEST(PrimaryView) { TKikimrSettings settings; NKikimrConfig::TAppConfig appConfig; @@ -3938,10 +3871,9 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { } - Y_UNIT_TEST_TWIN(ComplexLookupLimit, NewPredicateExtract) { + Y_UNIT_TEST(ComplexLookupLimit) { TKikimrSettings settings; NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetPredicateExtract20(NewPredicateExtract); settings.SetAppConfig(appConfig); TKikimrRunner kikimr(settings); @@ -4006,7 +3938,7 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params, querySettings).GetValueSync(); AssertSuccessResult(result); - AssertTableReads(result, "/Root/Sample", NewPredicateExtract ? 2 : 4); + AssertTableReads(result, "/Root/Sample", 2); CompareYson(R"([[[1u];[2u]];[[2u];[2u]]])", FormatResultSetYson(result.GetResultSet(0))); } diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index f9f6a626a45e..91203af2e43a 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -3044,7 +3044,6 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(true); AppConfig.MutableTableServiceConfig()->SetEnableKqpDataQueryStreamLookup(true); - AppConfig.MutableTableServiceConfig()->SetOldLookupJoinBehaviour(false); auto settings = TKikimrSettings().SetAppConfig(AppConfig).SetWithSampleTables(false); Kikimr = std::make_unique(settings); diff --git a/ydb/core/protos/table_service_config.proto b/ydb/core/protos/table_service_config.proto index 6494153530cb..e667b2edc945 100644 --- a/ydb/core/protos/table_service_config.proto +++ b/ydb/core/protos/table_service_config.proto @@ -249,7 +249,7 @@ message TTableServiceConfig { optional EBindingsMode BindingsMode = 40 [default = BM_ENABLED]; optional TIteratorReadsRetrySettings IteratorReadsRetrySettings = 41; - optional bool PredicateExtract20 = 44 [default = true]; + reserved 44; // optional bool PredicateExtract20 = 44 [default = true]; optional TIteratorReadQuotaSettings IteratorReadQuotaSettings = 45; enum EChannelTransportVersion { @@ -280,7 +280,7 @@ message TTableServiceConfig { optional bool EnableCreateTableAs = 57 [default = true]; optional uint64 IdxLookupJoinPointsLimit = 58 [default = 3]; - optional bool OldLookupJoinBehaviour = 59 [default = false]; + reserved 59; //optional bool OldLookupJoinBehaviour = 59 [default = false]; optional bool EnableOltpSink = 60 [default = false]; diff --git a/ydb/core/tx/datashard/datashard_ut_trace.cpp b/ydb/core/tx/datashard/datashard_ut_trace.cpp index dae9ae2b0349..ba6a6cdd83cf 100644 --- a/ydb/core/tx/datashard/datashard_ut_trace.cpp +++ b/ydb/core/tx/datashard/datashard_ut_trace.cpp @@ -340,99 +340,44 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; std::string canon; - if (server->GetSettings().AppConfig->GetTableServiceConfig().GetEnableKqpDataQueryStreamLookup() || server->GetSettings().AppConfig->GetTableServiceConfig().GetPredicateExtract20()) { - auto readActorSpan = trace.Root.BFSFindOne("ReadActor"); - UNIT_ASSERT(readActorSpan); - - auto dsReads = readActorSpan->get().FindAll("Datashard.Read"); // Read actor sends EvRead to each shard. - UNIT_ASSERT_VALUES_EQUAL(dsReads.size(), 2); - - canon = ExpectedSpan("Session.query.QUERY_ACTION_EXECUTE", - ExpectedSpan("CompileService", "CompileActor"), - "LiteralExecuter", - ExpectedSpan("DataExecuter", - "WaitForTableResolve", - "WaitForShardsResolve", - "WaitForSnapshot", - ExpectedSpan("ComputeActor", - ExpectedSpan("ReadActor", - "WaitForShardsResolve", - Repeat( - ExpectedSpan("Datashard.Read", - ExpectedSpan("Tablet.Transaction", - ExpectedSpan("Tablet.Transaction.Execute", - Repeat("Datashard.Unit", 3)), - // No extra page fault with btree index (root is in meta) - ConditionalSpanVec(!bTreeIndex, - "Tablet.Transaction.Wait", - "Tablet.Transaction.Enqueued", - ExpectedSpan("Tablet.Transaction.Execute", - "Datashard.Unit")), + auto readActorSpan = trace.Root.BFSFindOne("ReadActor"); + UNIT_ASSERT(readActorSpan); + + auto dsReads = readActorSpan->get().FindAll("Datashard.Read"); // Read actor sends EvRead to each shard. + UNIT_ASSERT_VALUES_EQUAL(dsReads.size(), 2); + + canon = ExpectedSpan("Session.query.QUERY_ACTION_EXECUTE", + ExpectedSpan("CompileService", "CompileActor"), + "LiteralExecuter", + ExpectedSpan("DataExecuter", + "WaitForTableResolve", + "WaitForShardsResolve", + "WaitForSnapshot", + ExpectedSpan("ComputeActor", + ExpectedSpan("ReadActor", + "WaitForShardsResolve", + Repeat( + ExpectedSpan("Datashard.Read", + ExpectedSpan("Tablet.Transaction", + ExpectedSpan("Tablet.Transaction.Execute", + Repeat("Datashard.Unit", 3)), + // No extra page fault with btree index (root is in meta) + ConditionalSpanVec(!bTreeIndex, "Tablet.Transaction.Wait", "Tablet.Transaction.Enqueued", ExpectedSpan("Tablet.Transaction.Execute", - Repeat("Datashard.Unit", 2)), - ExpectedSpan("Tablet.WriteLog", "Tablet.WriteLog.LogEntry"), - "Tablet.Transaction.Complete"), - "Datashard.SendWithConfirmedReadOnlyLease"), - 2))), - "ComputeActor", - "RunTasks")) - .ToString(); - } else { - auto deSpan = trace.Root.BFSFindOne("DataExecuter"); - UNIT_ASSERT(deSpan); - - auto dsTxSpans = deSpan->get().FindAll("Datashard.Transaction"); - UNIT_ASSERT_VALUES_EQUAL(2, dsTxSpans.size()); // Two shards, each executes a user transaction. - - for (auto dsTxSpan : dsTxSpans) { - auto tabletTxs = dsTxSpan.get().FindAll("Tablet.Transaction"); - UNIT_ASSERT_VALUES_EQUAL(1, tabletTxs.size()); - - auto propose = tabletTxs[0]; - CheckTxHasWriteLog(propose); - - // Blobs are loaded from BS. - UNIT_ASSERT_VALUES_EQUAL(2, propose.get().FindAll("Tablet.Transaction.Wait").size()); - UNIT_ASSERT_VALUES_EQUAL(2, propose.get().FindAll("Tablet.Transaction.Enqueued").size()); - - // We execute tx multiple times, because we have to load data for it to execute. - auto executeSpans = propose.get().FindAll("Tablet.Transaction.Execute"); - UNIT_ASSERT_VALUES_EQUAL(3, executeSpans.size()); - - CheckExecuteHasDatashardUnits(executeSpans[0], 3); - CheckExecuteHasDatashardUnits(executeSpans[1], 1); - CheckExecuteHasDatashardUnits(executeSpans[2], 3); - } - - canon = ExpectedSpan("Session.query.QUERY_ACTION_EXECUTE", - ExpectedSpan("CompileService", "CompileActor"), - "LiteralExecuter", - ExpectedSpan("DataExecuter", - "WaitForTableResolve", - "WaitForSnapshot", - "RunTasks", - Repeat( - ExpectedSpan("Datashard.Transaction", - ExpectedSpan("Tablet.Transaction", - ExpectedSpan("Tablet.Transaction.Execute", - Repeat("Datashard.Unit", 3)), - "Tablet.Transaction.Wait", - "Tablet.Transaction.Enqueued", - ExpectedSpan("Tablet.Transaction.Execute", - "Datashard.Unit"), - "Tablet.Transaction.Wait", - "Tablet.Transaction.Enqueued", - ExpectedSpan("Tablet.Transaction.Execute", - Repeat("Datashard.Unit", 3)), - ExpectedSpan("Tablet.WriteLog", "Tablet.WriteLog.LogEntry"), - "Tablet.Transaction.Complete"), - "Datashard.SendResult"), - 2), - "ComputeActor")) - .ToString(); - } + "Datashard.Unit")), + "Tablet.Transaction.Wait", + "Tablet.Transaction.Enqueued", + ExpectedSpan("Tablet.Transaction.Execute", + Repeat("Datashard.Unit", 2)), + ExpectedSpan("Tablet.WriteLog", "Tablet.WriteLog.LogEntry"), + "Tablet.Transaction.Complete"), + "Datashard.SendWithConfirmedReadOnlyLease"), + 2))), + "ComputeActor", + "RunTasks")) + .ToString(); UNIT_ASSERT_VALUES_EQUAL(trace.ToString(), canon); } diff --git a/ydb/library/yql/providers/common/provider/ya.make b/ydb/library/yql/providers/common/provider/ya.make index dc999b8fd199..26cd61bf5c04 100644 --- a/ydb/library/yql/providers/common/provider/ya.make +++ b/ydb/library/yql/providers/common/provider/ya.make @@ -6,8 +6,6 @@ SRCS( yql_provider.cpp yql_provider.h yql_provider_names.h - yql_table_lookup.cpp - yql_table_lookup.h ) PEERDIR( diff --git a/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp b/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp deleted file mode 100644 index caa3ea11cdef..000000000000 --- a/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp +++ /dev/null @@ -1,1061 +0,0 @@ -#include "yql_table_lookup.h" - -#include -#include - -namespace NYql { -namespace NCommon { - -using namespace NNodes; - -namespace { - -template -TExprBase BuildColumnCompare(TExprBase row, const TString& columnName, TExprBase value, TExprContext& ctx) { - auto compare = Build(ctx, row.Pos()) - .template Left() - .Struct(row) - .Name().Build(columnName) - .Build() - .Right(value) - .Done(); - - TMaybeNode ret = compare; - - auto rowType = row.Ref().GetTypeAnn()->Cast(); - auto columnType = rowType->GetItems()[*rowType->FindItem(columnName)]->GetItemType(); - - if (columnType->GetKind() == ETypeAnnotationKind::Optional) { - ret = Build(ctx, row.Pos()) - .Predicate(compare) - .template Value() - .Literal().Build("false") - .Build() - .Done(); - } - - return ret.Cast(); -} - -TMaybeNode CombinePredicatesAnd(TMaybeNode left, TMaybeNode right, TExprContext& ctx) { - if (left && right) { - return Build(ctx, left.Cast().Pos()) - .Add({left.Cast(), right.Cast()}) - .Done(); - } else if (left) { - return left; - } else if (right) { - return right; - } - - return TMaybeNode(); -} - -TMaybeNode CombinePredicatesOr(TMaybeNode left, TMaybeNode right, TExprContext& ctx) { - if (left && right) { - return Build(ctx, left.Cast().Pos()) - .Add({left.Cast(), right.Cast()}) - .Done(); - } - - return TMaybeNode(); -} - -template -TMaybeNode CombinePredicateListOr(const TVector& list, TExprContext& ctx, - TFunc getPredicateFunc) -{ - if (list.empty()) { - return {}; - } - - TMaybeNode residualPredicate = getPredicateFunc(list[0]); - for (size_t i = 1; i < list.size(); ++i) { - auto predicate = getPredicateFunc(list[i]); - residualPredicate = CombinePredicatesOr(residualPredicate, predicate, ctx); - } - - return residualPredicate; -} - -TMaybeNode BuildColumnRangePredicate(const TString& column, const TColumnRange& range, - TExprBase row, TExprContext& ctx) -{ - if (range.IsNull()) { - return Build(ctx, row.Pos()) - .Value() - .Optional() - .Struct(row) - .Name().Build(column) - .Build() - .Build() - .Done(); - } - - if (range.IsPoint()) { - return BuildColumnCompare(row, column, range.GetFrom().GetValue(), ctx); - } - - TMaybeNode fromPredicate; - if (range.GetFrom().IsDefined()) { - fromPredicate = range.GetFrom().IsInclusive() - ? BuildColumnCompare(row, column, range.GetFrom().GetValue(), ctx) - : BuildColumnCompare(row, column, range.GetFrom().GetValue(), ctx); - } - - TMaybeNode toPredicate; - if (range.GetTo().IsDefined()) { - toPredicate = range.GetTo().IsInclusive() - ? BuildColumnCompare(row, column, range.GetTo().GetValue(), ctx) - : BuildColumnCompare(row, column, range.GetTo().GetValue(), ctx); - } - - if (fromPredicate && toPredicate) { - return Build(ctx, row.Pos()) - .Add({fromPredicate.Cast(), toPredicate.Cast()}) - .Done(); - } else if (fromPredicate) { - return fromPredicate.Cast(); - } else if (toPredicate){ - return toPredicate.Cast(); - } - - return TMaybeNode(); -} - -TMaybeNode BuildKeyRangePredicate(const TVector& columns, const TKeyRange& keyRange, - TExprBase row, TExprContext& ctx) -{ - YQL_ENSURE(columns.size() == keyRange.GetColumnRangesCount()); - - TMaybeNode predicate = keyRange.GetResidualPredicate(); - for (size_t i = 0; i < columns.size(); ++i) { - auto& column = columns[i]; - auto& range = keyRange.GetColumnRange(i); - - auto rangePredicate = BuildColumnRangePredicate(column, range, row, ctx); - predicate = CombinePredicatesAnd(predicate, rangePredicate, ctx); - } - - return predicate; -} - -struct TLookupContext { - TLookupContext(TExprContext& exprCtx, TTableLookup::TGetValueFunc getValueFunc, - TTableLookup::TCanCompareFunc canCompareFunc, TTableLookup::TCompareFunc compareFunc) - : ExprCtx(exprCtx) - , GetValueFunc(getValueFunc) - , CanCompareFunc(canCompareFunc) - , CompareFunc(compareFunc) {} - - TRangeBound MinBound(const TRangeBound& a, const TRangeBound& b) const { - YQL_ENSURE(a.IsDefined() && b.IsDefined()); - - if (!CanCompareFunc(a.GetValue()) || !CanCompareFunc(b.GetValue())) { - return TRangeBound::MakeUndefined(); - } - - switch (CompareFunc(a.GetValue(), b.GetValue()).Result) { - case TTableLookup::TCompareResult::Less: - return a; - case TTableLookup::TCompareResult::Greater: - return b; - case TTableLookup::TCompareResult::Equal: - return a.IsInclusive() && b.IsInclusive() ? a : TRangeBound(a.GetValue(), false); - } - - YQL_ENSURE(false); - } - - TRangeBound MaxBound(const TRangeBound& a, const TRangeBound& b) const { - YQL_ENSURE(a.IsDefined() && b.IsDefined()); - - if (!CanCompareFunc(a.GetValue()) || !CanCompareFunc(b.GetValue())) { - return TRangeBound::MakeUndefined(); - } - - switch (CompareFunc(a.GetValue(), b.GetValue()).Result) { - case TTableLookup::TCompareResult::Less: - return b; - case TTableLookup::TCompareResult::Greater: - return a; - case TTableLookup::TCompareResult::Equal: - return a.IsInclusive() && b.IsInclusive() ? a : TRangeBound(a.GetValue(), false); - } - - YQL_ENSURE(false); - } - - TExprContext& ExprCtx; - TTableLookup::TGetValueFunc GetValueFunc; - TTableLookup::TCanCompareFunc CanCompareFunc; - TTableLookup::TCompareFunc CompareFunc; - bool AllowNullCompare = false; -}; - -TColumnRange MakeColumnRange(TRangeBound left, TRangeBound right, const TLookupContext& ctx) { - if (!left.IsDefined() || !right.IsDefined()) { - return TColumnRange(left, right); - } - - if (!ctx.CanCompareFunc(left.GetValue()) || !ctx.CanCompareFunc(right.GetValue())) { - return TColumnRange(left, right); - } - - auto cmp = ctx.CompareFunc(left.GetValue(), right.GetValue()); - switch (cmp.Result) { - case TTableLookup::TCompareResult::Less: - if (cmp.AreAdjacent()) { - if (left.IsInclusive() && !right.IsInclusive()) { - return TColumnRange::MakePoint(left.GetValue()); - } else if (!left.IsInclusive() && right.IsInclusive()) { - return TColumnRange::MakePoint(right.GetValue()); - } - } - break; - - case TTableLookup::TCompareResult::Greater: - break; - - case TTableLookup::TCompareResult::Equal: - if (left.IsInclusive() && right.IsInclusive()) { - return TColumnRange::MakePoint(left.GetValue()); - } - break; - } - - return TColumnRange(left, right); -} - -class TKeyRangeBuilder { -public: - TKeyRangeBuilder(const TVector& keyColumns) - : KeyColumns(keyColumns) - { - for (auto& keyColumn : keyColumns) { - ColumnRanges[keyColumn] = TColumnRange::MakeUnbounded(); - } - } - - TKeyRangeBuilder(const TKeyRangeBuilder& builder, TMaybeNode residualPredicate) - : KeyColumns(builder.KeyColumns) - , ColumnRanges(builder.ColumnRanges) - , ResidualPredicate(residualPredicate) {} - - bool IsFullScan() const { - return std::find_if(ColumnRanges.begin(), ColumnRanges.end(), - [] (const std::pair& pair) { - return pair.second.IsDefined(); - }) == ColumnRanges.end(); - } - - const TVector& GetKeyColumns() const { - return KeyColumns; - } - - TColumnRange GetColumnRange(const TStringBuf& column) const { - auto* keyRange = ColumnRanges.FindPtr(column); - YQL_ENSURE(keyRange); - - return *keyRange; - } - - bool HasColumnRange(const TStringBuf& column) const { - auto* keyRange = ColumnRanges.FindPtr(column); - return keyRange != nullptr; - } - - void SetColumnRange(const TStringBuf& column, const TColumnRange& range) { - auto it = ColumnRanges.find(column); - YQL_ENSURE(it != ColumnRanges.end()); - - it->second = range; - } - - TMaybeNode GetResidualPredicate() const { - return ResidualPredicate; - } - - void SetResidualPredicate(TMaybeNode predicate) { - ResidualPredicate = predicate; - } - - TMaybeNode BuildPredicate(const TVector& columns, - TExprBase row, TExprContext& ctx) const - { - auto keyRange = BuildKeyRange(row, ctx); - return BuildKeyRangePredicate(columns, keyRange, row, ctx); - } - - TKeyRange BuildKeyRange(TExprBase row, TExprContext& ctx) const { - if (IsFullScan()) { - return TKeyRange(ctx, KeyColumns.size(), ResidualPredicate); - } - - auto newResidualPredicate = ResidualPredicate; - TVector newColumnRanges; - - bool keyRangeFinished = false; - for (const TString& column : KeyColumns) { - auto columnRange = GetColumnRange(column); - - if (keyRangeFinished) { - if (columnRange.IsDefined()) { - auto predicate = BuildColumnRangePredicate(column, columnRange, row, ctx); - newResidualPredicate = CombinePredicatesAnd(newResidualPredicate, predicate, ctx); - } - newColumnRanges.push_back(TColumnRange::MakeUnbounded()); - } else { - keyRangeFinished = !columnRange.IsPoint(); - newColumnRanges.push_back(columnRange); - } - } - - return TKeyRange(ctx, newColumnRanges, newResidualPredicate); - } - -private: - TVector KeyColumns; - THashMap ColumnRanges; - TMaybeNode ResidualPredicate; -}; - -class TTableLookupBuilder { -public: - TTableLookupBuilder(const TVector& keyColumns, const TVector& keyRangeBuilders) - : KeyColumns(keyColumns) - , KeyRangeBuilders(keyRangeBuilders) {} - - TTableLookupBuilder(const TVector& keyColumns) - : TTableLookupBuilder(keyColumns, {}) {} - - TTableLookupBuilder(const TVector& keyColumns, const TKeyRangeBuilder& keyRangeBuilder) - : TTableLookupBuilder(keyColumns, TVector{keyRangeBuilder}) {} - - bool IsSingleRange() const { - return KeyRangeBuilders.size() == 1; - } - - bool HasNonFullscanRanges() const { - for (auto& range : KeyRangeBuilders) { - if (!range.IsFullScan()) { - return true; - } - } - - return false; - } - - const TVector& GetKeyRangeBuilders() const { - return KeyRangeBuilders; - } - - const TKeyRangeBuilder& GetKeyRangeBuilder() const { - YQL_ENSURE(IsSingleRange()); - return KeyRangeBuilders.front(); - } - - TTableLookup BuildTableLookup(TExprBase row, const TLookupContext& ctx) const { - TVector keyRanges; - for (auto& keyRangeBuilder : KeyRangeBuilders) { - keyRanges.push_back(keyRangeBuilder.BuildKeyRange(row, ctx.ExprCtx)); - } - - if (!IsSingleRange()) { - if (CheckIndependentRanges(keyRanges, ctx)) { - return TTableLookup(KeyColumns, keyRanges); - } - - if (TryBuildPointRanges(keyRanges, ctx)) { - return TTableLookup(KeyColumns, keyRanges); - } - - auto keyRange = BuildBoundingKeyRange(keyRanges, row, ctx.ExprCtx); - return TTableLookup(KeyColumns, {keyRange}); - } - - return TTableLookup(KeyColumns, keyRanges); - } - -private: - static bool TryBuildPointRanges(TVector& keyRanges, const TLookupContext& ctx) { - YQL_ENSURE(keyRanges.size() > 1); - - const auto& firstRange = keyRanges[0]; - ui32 numDefined = firstRange.GetNumDefined(); - - for (const auto& range : keyRanges) { - if (range.GetNumDefined() != numDefined) { - return false; - } - - for (size_t i = 0; i < numDefined; ++i) { - auto& columnRange = range.GetColumnRange(i); - if (!columnRange.IsPoint()) { - return false; - } - - if (columnRange.IsNull()) { - return false; - } - - if (!ctx.CanCompareFunc(columnRange.GetFrom().GetValue())) { - return false; - } - } - } - - auto less = [&ctx, numDefined] (const TKeyRange &a, const TKeyRange& b) { - for (size_t i = 0; i < numDefined; ++i) { - auto cmpResult = ctx.CompareFunc( - a.GetColumnRange(i).GetFrom().GetValue(), - b.GetColumnRange(i).GetFrom().GetValue()); - - switch (cmpResult.Result) { - case TTableLookup::TCompareResult::Less: - return true; - case TTableLookup::TCompareResult::Greater: - return false; - case TTableLookup::TCompareResult::Equal: - break; - } - } - - return false; - }; - - std::stable_sort(keyRanges.begin(), keyRanges.end(), less); - - TMaybeNode residualPredicate; - size_t curIndex = 0; - for (size_t i = 0; i < keyRanges.size(); ++i) { - auto predicate = keyRanges[i].GetResidualPredicate(); - - residualPredicate = residualPredicate - ? CombinePredicatesOr(residualPredicate, predicate, ctx.ExprCtx) - : predicate; - - if (i == keyRanges.size() - 1 || less(keyRanges[i], keyRanges[i + 1])) { - TKeyRange keyRange(ctx.ExprCtx, keyRanges[i].GetColumnRanges(), residualPredicate); - keyRanges[curIndex] = keyRange; - ++curIndex; - - residualPredicate = {}; - } - } - - keyRanges.erase(keyRanges.begin() + curIndex, keyRanges.end()); - return true; - } - - static bool CheckIndependentRanges(TVector& keyRanges, const TLookupContext& ctx) { - for (auto& keyRange : keyRanges) { - for (auto& columnRange : keyRange.GetColumnRanges()) { - if (columnRange.GetFrom().IsDefined() && !ctx.CanCompareFunc(columnRange.GetFrom().GetValue())) { - return false; - } - - if (columnRange.GetTo().IsDefined() && !ctx.CanCompareFunc(columnRange.GetTo().GetValue())) { - return false; - } - } - } - - auto less = [&ctx] (const TKeyRange &a, const TKeyRange& b) { - return KeyTupleLess(a.GetFromTuple(), b.GetFromTuple(), ctx.CompareFunc); - }; - - std::stable_sort(keyRanges.begin(), keyRanges.end(), less); - - for (size_t i = 0; i < keyRanges.size() - 1; ++i) { - if (!KeyTupleLess(keyRanges[i].GetToTuple(), keyRanges[i + 1].GetFromTuple(), ctx.CompareFunc)) { - return false; - } - } - - return true; - } - - TKeyRange BuildBoundingKeyRange(const TVector &keyRanges, - TExprBase row, TExprContext& ctx) const - { - YQL_ENSURE(keyRanges.size() > 0); - - auto residualPredicate = CombinePredicateListOr(keyRanges, ctx, - [this, row, &ctx] (const TKeyRange& keyRange) { - return BuildKeyRangePredicate(KeyColumns, keyRange, row, ctx); - }); - - // TODO: Compute actual bounding range. - return TKeyRange(ctx, KeyColumns.size(), residualPredicate); - } - -private: - TVector KeyColumns; - TVector KeyRangeBuilders; -}; - -TKeyRangeBuilder CombineKeyRangesAnd(TExprBase row, const TVector& keyColumns, const TKeyRangeBuilder& left, - const TKeyRangeBuilder& right, const TLookupContext& ctx) -{ - TKeyRangeBuilder combinedLookup(keyColumns); - - auto rp = CombinePredicatesAnd(left.GetResidualPredicate(), right.GetResidualPredicate(), ctx.ExprCtx); - - for (const TString& column : left.GetKeyColumns()) { - auto columnLeft = left.GetColumnRange(column); - auto columnRight = right.GetColumnRange(column); - - TColumnRange columnRange; - - if (columnLeft.IsNull()) { - columnRange = columnLeft; - if (columnRight.IsDefined()) { - auto predicate = BuildColumnRangePredicate(column, columnRight, row, ctx.ExprCtx); - rp = CombinePredicatesAnd(rp, predicate, ctx.ExprCtx); - } - } else if (columnRight.IsNull()) { - columnRange = columnRight; - if (columnLeft.IsDefined()) { - auto predicate = BuildColumnRangePredicate(column, columnLeft, row, ctx.ExprCtx); - rp = CombinePredicatesAnd(rp, predicate, ctx.ExprCtx); - } - } else { - TRangeBound combinedFrom; - TRangeBound combinedTo; - - if (columnLeft.GetFrom().IsDefined() && columnRight.GetFrom().IsDefined()) { - combinedFrom = ctx.MaxBound(columnLeft.GetFrom(), columnRight.GetFrom()); - if (!combinedFrom.IsDefined()) { - combinedFrom = columnLeft.GetFrom(); - rp = CombinePredicatesAnd(rp, - BuildColumnRangePredicate(column, columnRight, row, ctx.ExprCtx), - ctx.ExprCtx); - } - } else if (columnLeft.GetFrom().IsDefined()) { - combinedFrom = columnLeft.GetFrom(); - } else if (columnRight.GetFrom().IsDefined()) { - combinedFrom = columnRight.GetFrom(); - } - - if (columnLeft.GetTo().IsDefined() && columnRight.GetTo().IsDefined()) { - combinedTo = ctx.MinBound(columnLeft.GetTo(), columnRight.GetTo()); - if (!combinedTo.IsDefined()) { - combinedTo = columnLeft.GetTo(); - rp = CombinePredicatesAnd(rp, - BuildColumnRangePredicate(column, columnRight, row, ctx.ExprCtx), - ctx.ExprCtx); - } - } else if (columnLeft.GetTo().IsDefined()) { - combinedTo = columnLeft.GetTo(); - } else if (columnRight.GetTo().IsDefined()) { - combinedTo = columnRight.GetTo(); - } - - columnRange = MakeColumnRange(combinedFrom, combinedTo, ctx); - } - - combinedLookup.SetColumnRange(column, columnRange); - } - - combinedLookup.SetResidualPredicate(rp); - - return combinedLookup; -} - -TTableLookupBuilder CombineLookupsAnd(TExprBase row, const TVector& keyColumns, - const TTableLookupBuilder* builders, size_t size, const TLookupContext& ctx) -{ - switch (size) { - case 0U: Y_ABORT("Wrong case"); - case 1U: return *builders; - case 2U: { - const auto& left = builders[0U]; - const auto& right = builders[1U]; - if (left.IsSingleRange() && right.IsSingleRange()) { - auto combinedKeyRange = CombineKeyRangesAnd(row, keyColumns, left.GetKeyRangeBuilder(), - right.GetKeyRangeBuilder(), ctx); - return TTableLookupBuilder(keyColumns, combinedKeyRange); - } else { - auto combineRanges = - [&keyColumns, row, &ctx] (const TTableLookupBuilder& lookup, const TTableLookupBuilder& residual) { - auto& residualKeyRanges = residual.GetKeyRangeBuilders(); - auto residualPredicate = CombinePredicateListOr(residualKeyRanges, ctx.ExprCtx, - [&keyColumns, row, &ctx](const TKeyRangeBuilder& keyRange) { - return keyRange.BuildPredicate(keyColumns, row, ctx.ExprCtx); - }); - - TVector newKeyRanges; - for (auto& keyRange : lookup.GetKeyRangeBuilders()) { - auto newResidualPredicate = CombinePredicatesAnd(keyRange.GetResidualPredicate(), - residualPredicate, ctx.ExprCtx); - auto newRange = TKeyRangeBuilder(keyRange, newResidualPredicate); - newKeyRanges.push_back(newRange); - } - - return TTableLookupBuilder(keyColumns, newKeyRanges); - }; - - const bool keepRight = right.HasNonFullscanRanges() && !left.HasNonFullscanRanges(); - return keepRight ? combineRanges(right, left) : combineRanges(left, right); - } - } - default: break; - } - - const auto half = (size + 1U) >> 1U; - const std::array pair = {{ - CombineLookupsAnd(row, keyColumns, builders, half, ctx), - CombineLookupsAnd(row, keyColumns, builders + half, size - half, ctx) - }}; - return CombineLookupsAnd(row, keyColumns, pair.data(), pair.size(), ctx); -} - -TTableLookupBuilder CombineLookupsOr(TExprBase, const TVector& keyColumns, - const TTableLookupBuilder* builders, size_t size, const TLookupContext&) -{ - TVector newKeyRanges; - newKeyRanges.reserve(size); - for (size_t i = 0U; i < size; ++i) { - newKeyRanges.insert(newKeyRanges.end(), builders[i].GetKeyRangeBuilders().cbegin(), builders[i].GetKeyRangeBuilders().cend()); - } - return TTableLookupBuilder(keyColumns, newKeyRanges); -} - -TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate, - const TVector& keyColumns, const TLookupContext& ctx) -{ - if (const auto maybeAnd = predicate.Maybe()) { - const auto size = maybeAnd.Cast().Args().size(); - std::vector builders; - builders.reserve(size); - for (auto i = 0U; i < size; ++i) { - builders.emplace_back(CollectLookups(row, maybeAnd.Cast().Arg(i), keyColumns, ctx)); - } - return CombineLookupsAnd(row, keyColumns, builders.data(), builders.size(), ctx); - } - - if (const auto maybeOr = predicate.Maybe()) { - const auto size = maybeOr.Cast().Args().size(); - std::vector builders; - builders.reserve(size); - for (auto i = 0U; i < size; ++i) { - builders.emplace_back(CollectLookups(row, maybeOr.Cast().Arg(i), keyColumns, ctx)); - } - return CombineLookupsOr(row, keyColumns, builders.data(), builders.size(), ctx); - } - - TMaybeNode maybeCompare = predicate.Maybe(); - if (auto maybeLiteral = predicate.Maybe().Value().Maybe().Literal()) { - if (maybeLiteral.Cast().Value() == "false") { - maybeCompare = predicate.Cast().Predicate().Maybe(); - } - } - - TTableLookupBuilder fullScan(keyColumns, TKeyRangeBuilder(keyColumns, predicate)); - - auto getRowMember = [row] (TExprBase expr) { - if (auto maybeMember = expr.Maybe()) { - if (maybeMember.Cast().Struct().Raw() == row.Raw()) { - return maybeMember; - } - } - - return TMaybeNode(); - }; - - auto getTableLookup = [&keyColumns, fullScan] (const TStringBuf& column, const TColumnRange& range, - TMaybeNode residualPredicate) - { - TKeyRangeBuilder keyRange(keyColumns, residualPredicate); - if (keyRange.HasColumnRange(column)) { - keyRange.SetColumnRange(column, range); - return TTableLookupBuilder(keyColumns, keyRange); - } - - return fullScan; - }; - - auto getNullComparePredicate = [&ctx](TExprBase value, TExprBase rowValue) -> TMaybeNode { - if (ctx.AllowNullCompare || rowValue.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional) { - return TMaybeNode(); - } - - return Build(ctx.ExprCtx, rowValue.Pos()) - .Optional(value) - .Done(); - }; - - if (maybeCompare) { - auto left = maybeCompare.Cast().Left(); - auto right = maybeCompare.Cast().Right(); - - TMaybeNode maybeLeftMember = getRowMember(left); - TMaybeNode maybeRightMember = getRowMember(right); - - TMaybeNode maybeMember; - TMaybeNode maybeValue; - bool reverseCompare = false; - if (maybeLeftMember) { - maybeMember = maybeLeftMember; - maybeValue = ctx.GetValueFunc(right, maybeMember.Cast().Ref().GetTypeAnn(), ctx.ExprCtx); - } else if (maybeRightMember) { - maybeMember = maybeRightMember; - maybeValue = ctx.GetValueFunc(left, maybeMember.Cast().Ref().GetTypeAnn(), ctx.ExprCtx); - reverseCompare = true; - } - - if (!maybeValue) { - return fullScan; - } - - auto value = maybeValue.Cast(); - auto column = maybeMember.Cast().Name().Value(); - - TMaybe columnRange; - - if (maybeCompare.Maybe()) { - columnRange = TColumnRange::MakePoint(value); - } - - if (maybeCompare.Maybe()) { - columnRange = reverseCompare - ? MakeColumnRange(TRangeBound::MakeExclusive(value), TRangeBound::MakeUndefined(), ctx) - : MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeExclusive(value), ctx); - } - - if (maybeCompare.Maybe()) { - columnRange = reverseCompare - ? MakeColumnRange(TRangeBound::MakeInclusive(value), TRangeBound::MakeUndefined(), ctx) - : MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeInclusive(value), ctx); - } - - if (maybeCompare.Maybe()) { - columnRange = reverseCompare - ? MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeExclusive(value), ctx) - : MakeColumnRange(TRangeBound::MakeExclusive(value), TRangeBound::MakeUndefined(), ctx); - } - - if (maybeCompare.Maybe()) { - columnRange = reverseCompare - ? MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeInclusive(value), ctx) - : MakeColumnRange(TRangeBound::MakeInclusive(value), TRangeBound::MakeUndefined(), ctx); - } - - if (columnRange) { - return getTableLookup(column, *columnRange, getNullComparePredicate(value, maybeMember.Cast())); - } - } - - auto maybeLookup = TMaybeNode(); - if (auto maybeLiteral = predicate.Maybe().Value().Maybe().Literal()) { - if (maybeLiteral.Cast().Value() == "false") { - maybeLookup = predicate.Maybe().Predicate().Maybe(); - } - } else { - maybeLookup = predicate.Maybe(); - } - - if (maybeLookup) { - auto lookup = maybeLookup.Cast(); - - if (!lookup.Lookup().Maybe()) { - return fullScan; - } - - auto member = lookup.Lookup().Cast(); - auto column = member.Name().Value(); - if (member.Struct().Raw() != row.Raw()) { - return fullScan; - } - - TExprNode::TPtr collection; - if (lookup.Collection().Ref().IsList()) { - collection = lookup.Collection().Ptr(); - } else if (auto maybeDictFromKeys = lookup.Collection().Maybe()) { - collection = maybeDictFromKeys.Cast().Keys().Ptr(); - } else { - return fullScan; - } - - auto size = collection->ChildrenSize(); - if (!size) { - return fullScan; - } - - TVector keyRanges; - keyRanges.reserve(size); - for (const auto& key : collection->Children()) { - auto maybeValue = ctx.GetValueFunc(TExprBase(key), member.Ref().GetTypeAnn(), ctx.ExprCtx); - if (!maybeValue) { - return fullScan; - } - - TKeyRangeBuilder keyRange(keyColumns); - if (!keyRange.HasColumnRange(column)) { - return fullScan; - } - keyRange.SetColumnRange(column, TColumnRange::MakePoint(maybeValue.Cast())); - - keyRanges.push_back(keyRange); - } - - return TTableLookupBuilder(keyColumns, keyRanges); - } - - if (auto maybeBool = predicate.Maybe().Value().Maybe().Literal()) { - if (maybeBool.Cast().Value() == "false") { - TExprBase value = predicate.Cast().Predicate(); - TMaybeNode maybeMember = getRowMember(value); - if (maybeMember) { - auto column = maybeMember.Cast().Name().Value(); - auto columnRange = TColumnRange::MakePoint( - Build(ctx.ExprCtx, row.Pos()).Literal().Build("true").Done()); - return getTableLookup(column, columnRange, {}); - } - } - } - - if (auto maybeBool = predicate.Maybe().Value().Maybe().Value().Maybe().Literal()) { - if (maybeBool.Cast().Value() == "true") { - TExprBase value = predicate.Cast().Value().Cast().Predicate(); - TMaybeNode maybeMember = getRowMember(value); - if (maybeMember) { - auto column = maybeMember.Cast().Name().Value(); - auto columnRange = TColumnRange::MakePoint( - Build(ctx.ExprCtx, row.Pos()).Literal().Build("false").Done()); - return getTableLookup(column, columnRange, {}); - } - } - } - - if (auto maybeExists = predicate.Maybe().Value().Maybe()) { - TMaybeNode maybeMember = getRowMember(maybeExists.Cast().Optional()); - if (maybeMember) { - auto column = maybeMember.Cast().Name().Value(); - auto columnRange = TColumnRange::MakeNull(ctx.ExprCtx); - return getTableLookup(column, columnRange, {}); - } - } - - return fullScan; -} - -} // namespace - -TColumnRange TColumnRange::MakeNull(TExprContext& ctx) { - auto nullValue = Build(ctx, TPositionHandle()).Done(); - TColumnRange range = MakePoint(nullValue); - range.Null = true; - return range; -} - -TKeyRange::TKeyRange(TExprContext& ctx, const TVector& columnRanges, TMaybeNode residualPredicate) - : Ctx(&ctx) - , ColumnRanges(columnRanges) - , ResidualPredicate(residualPredicate) - , NumDefined(0) -{ - EquiRange = true; - - TVector> fromValues(columnRanges.size()); - bool fromInclusive = true; - - TVector> toValues(columnRanges.size()); - bool toInclusive = true; - - for (size_t i = 0; i < columnRanges.size(); ++i) { - const auto& range = columnRanges[i]; - - if (!range.IsPoint()) { - if (range.IsDefined()) { - YQL_ENSURE(EquiRange); - } - - EquiRange = false; - } - - if (range.IsDefined()) { - ++NumDefined; - } - - if (range.GetFrom().IsDefined()) { - fromValues[i] = range.GetFrom().GetValue(); - fromInclusive = range.GetFrom().IsInclusive(); - } - - if (range.GetTo().IsDefined()) { - toValues[i] = range.GetTo().GetValue(); - toInclusive = range.GetTo().IsInclusive(); - } - } - - FromTuple = TKeyTuple(Ctx, fromValues, true, fromInclusive); - ToTuple = TKeyTuple(Ctx, toValues, false, toInclusive); -} - -void TTableLookup::Print(IOutputStream& output) const { - output << Endl << "[" << Endl; - for (auto& keyRange : KeyRanges) { - keyRange.Print(output); - } - output << "]" << Endl; -} - -void TKeyRange::Print(IOutputStream& output) const { - auto printExpr = [ctx = Ctx] (TExprBase node, IOutputStream& output) { - auto ast = ConvertToAst(node.Ref(), *ctx, TExprAnnotationFlags::None, true); - ast.Root->PrintTo(output); - }; - - output << "{" << Endl; - for (size_t i = 0; i < GetColumnRangesCount(); ++i) { - auto& range = GetColumnRange(i); - - output << " " << i << " "; - - if (range.IsNull()) { - output << "NULL" << Endl; - continue; - } - - if (range.GetFrom().IsInclusive()) { - output << "["; - } else { - output << "("; - } - - if (range.GetFrom().IsDefined()) { - printExpr(range.GetFrom().GetValue(), output); - } else { - output << "*"; - } - - output << "; "; - - if (range.GetTo().IsDefined()) { - printExpr(range.GetTo().GetValue(), output); - } else { - output << "*"; - } - - if (range.GetTo().IsInclusive()) { - output << "]"; - } else { - output << ")"; - } - - output << Endl; - } - - auto residualPredicate = GetResidualPredicate(); - output << " Residual: "; - if (residualPredicate) { - printExpr(residualPredicate.Cast(), output); - } else { - output << "None"; - } - - output << Endl << "}" << Endl; -} - -void TKeyTuple::Print(IOutputStream& output) const { - auto printExpr = [] (TExprContext& ctx, TExprBase node, IOutputStream& output) { - auto ast = ConvertToAst(node.Ref(), ctx, TExprAnnotationFlags::None, true); - ast.Root->PrintTo(output); - }; - - output << "\""; - output << (IsFrom() - ? IsInclusive() ? "[" : "(" - : IsInclusive() ? "]" : ")"); - output << "\" ("; - - for (size_t i = 0; i < Size(); ++i) { - auto value = GetValue(i); - if (value) { - YQL_ENSURE(Ctx); - printExpr(*Ctx, value.Cast(), output); - } else { - output << (IsFrom() - ? IsInclusive() ? "-Inf" : "+Inf" - : IsInclusive() ? "+Inf" : "-Inf"); - } - - if (i < Size() - 1) { - output << ", "; - } - } - - output << ")"; -} - -bool KeyTupleLess(const TKeyTuple& left, const TKeyTuple& right, const TTableLookup::TCompareFunc& cmpFunc) { - YQL_ENSURE(left.Size() == right.Size()); - - TTableLookup::TCompareResult::TResult cmpResult = TTableLookup::TCompareResult::Equal; - for (size_t i = 0; i < left.Size(); ++i) { - auto leftValue = left.GetValue(i); - auto rightValue = right.GetValue(i); - - if (leftValue && rightValue) { - cmpResult = cmpFunc(leftValue.Cast(),rightValue.Cast()).Result; - } else if (!leftValue) { - if (left.IsFrom() == left.IsInclusive()) { - if (rightValue || right.IsFrom() != right.IsInclusive()) { - cmpResult = TTableLookup::TCompareResult::Less; - } - } else { - if (rightValue || right.IsFrom() == right.IsInclusive()) { - cmpResult = TTableLookup::TCompareResult::Greater; - } - } - } else { - cmpResult = right.IsFrom() == right.IsInclusive() - ? TTableLookup::TCompareResult::Greater - : TTableLookup::TCompareResult::Less; - } - - if (cmpResult != TTableLookup::TCompareResult::Equal) { - break; - } - } - - if (cmpResult == TTableLookup::TCompareResult::Equal) { - if (left.IsFrom() && right.IsFrom() && left.IsInclusive() && !right.IsInclusive()) { - return true; - } - - if (!left.IsFrom() && right.IsFrom() && !(left.IsInclusive() && right.IsInclusive())) { - return true; - } - - if (!left.IsFrom() && !right.IsFrom() && !left.IsInclusive() && right.IsInclusive()) { - return true; - } - } - - return cmpResult == TTableLookup::TCompareResult::Less; -} - -TTableLookup ExtractTableLookup( - TExprBase row, - TExprBase predicate, - const TVector& keyColumns, - TTableLookup::TGetValueFunc getValueFunc, - TTableLookup::TCanCompareFunc canCompareFunc, - TTableLookup::TCompareFunc compareFunc, - TExprContext& exprCtx, - bool allowNullCompare) -{ - TLookupContext ctx(exprCtx, getValueFunc, canCompareFunc, compareFunc); - ctx.AllowNullCompare = allowNullCompare; - - auto tableLookupBuilder = CollectLookups(row, predicate, keyColumns, ctx); - auto tableLookup = tableLookupBuilder.BuildTableLookup(row, ctx); - - return tableLookup; -} - -} // namespace NCommon -} // namespace NYql diff --git a/ydb/library/yql/providers/common/provider/yql_table_lookup.h b/ydb/library/yql/providers/common/provider/yql_table_lookup.h deleted file mode 100644 index 988fb5b4baa5..000000000000 --- a/ydb/library/yql/providers/common/provider/yql_table_lookup.h +++ /dev/null @@ -1,249 +0,0 @@ -#pragma once - -#include - -#include - -namespace NYql { -namespace NCommon { - -class TRangeBound -{ -public: - TRangeBound(NNodes::TMaybeNode value, bool isInclusive) - : Value(value) - , Inclusive(isInclusive) {} - - TRangeBound() - : TRangeBound(NNodes::TMaybeNode(), false) {} - - bool IsDefined() const { - return Value.IsValid(); - } - - NNodes::TExprBase GetValue() const { - YQL_ENSURE(IsDefined()); - return Value.Cast(); - } - - bool IsInclusive() const { - return Inclusive; - } - - static TRangeBound MakeUndefined() { - return TRangeBound(); - } - - static TRangeBound MakeInclusive(NNodes::TExprBase value) { - return TRangeBound(value, true); - } - - static TRangeBound MakeExclusive(NNodes::TExprBase value) { - return TRangeBound(value, false); - } - -private: - NNodes::TMaybeNode Value; - bool Inclusive; -}; - -class TColumnRange { -public: - TColumnRange(const TRangeBound& from, const TRangeBound& to) - : From(from) - , To(to) - , Defined(From.IsDefined() || To.IsDefined()) - { - Point = From.IsDefined() && To.IsDefined() && - From.IsInclusive() && To.IsInclusive() && - From.GetValue().Raw() == To.GetValue().Raw(); - } - - TColumnRange() - : TColumnRange(TRangeBound(), TRangeBound()) {} - - TRangeBound GetFrom() const { return From; } - TRangeBound GetTo() const { return To; } - - bool IsDefined() const { - return Defined; - } - - bool IsPoint() const { - return Point; - } - - bool IsNull() const { - return Null; - } - - static TColumnRange MakeUnbounded() { - return TColumnRange(TRangeBound(), TRangeBound()); - } - - static TColumnRange MakePoint(NNodes::TExprBase value) { - return TColumnRange(TRangeBound(value, true), TRangeBound(value, true)); - } - - static TColumnRange MakeNull(TExprContext& ctx); - -private: - TRangeBound From; - TRangeBound To; - bool Defined; - bool Point; - bool Null = false; -}; - -class TKeyTuple { -public: - TKeyTuple(TExprContext* ctx, const TVector>& values, bool isFrom, bool isInclusive) - : Ctx(ctx) - , Values(values) - , From(isFrom) - , Inclusive(isInclusive) {} - - TKeyTuple() - : TKeyTuple(nullptr, {}, false, false) {} - - size_t Size() const { return Values.size(); } - bool IsFrom() const { return From; } - bool IsInclusive() const { return Inclusive; } - NNodes::TMaybeNode GetValue(size_t index) const { return Values[index]; } - - void Print(IOutputStream& output) const; - -private: - TExprContext* Ctx; - TVector> Values; - bool From; - bool Inclusive; -}; - -class TKeyRange { -public: - TKeyRange(TExprContext& ctx, const TVector& columnRanges, NNodes::TMaybeNode residualPredicate); - - TKeyRange(TExprContext& ctx, size_t keySize, NNodes::TMaybeNode residualPredicate) - : TKeyRange(ctx, TVector(keySize, TColumnRange()), residualPredicate) {} - - size_t GetColumnRangesCount() const { - return ColumnRanges.size(); - } - - const TColumnRange& GetColumnRange(size_t index) const { - return ColumnRanges[index]; - } - - const TVector& GetColumnRanges() const { - return ColumnRanges; - } - - bool IsFullScan() const { - return NumDefined == 0; - } - - bool IsEquiRange() const { - return EquiRange; - } - - size_t GetNumDefined() const { - return NumDefined; - } - - bool HasResidualPredicate() const { - return ResidualPredicate.IsValid(); - } - - NNodes::TMaybeNode GetResidualPredicate() const { - return ResidualPredicate; - } - - const TKeyTuple& GetFromTuple() const { - return FromTuple; - } - - const TKeyTuple& GetToTuple() const { - return ToTuple; - } - - void Print(IOutputStream& output) const; - -private: - TExprContext* Ctx; - TVector ColumnRanges; - NNodes::TMaybeNode ResidualPredicate; - bool EquiRange; - size_t NumDefined; - TKeyTuple FromTuple; - TKeyTuple ToTuple; -}; - -class TTableLookup { -public: - struct TCompareResult - { - enum TResult { - Equal, - Less, - Greater - }; - - TCompareResult(TResult result, TMaybe adjacent = {}) - : Result(result) - , Adjacent(adjacent) {} - - bool AreAdjacent() const { - return Adjacent.Defined() && *Adjacent; - } - - TResult Result; - TMaybe Adjacent; - }; - - typedef std::function(NNodes::TExprBase, - const TTypeAnnotationNode*, TExprContext& ctx)> TGetValueFunc; - typedef std::function TCanCompareFunc; - typedef std::function TCompareFunc; - - TTableLookup(const TVector& keyColumns, const TVector& keyRanges) - : KeyColumns(keyColumns) - , KeyRanges(keyRanges) {} - - const TVector& GetKeyColumns() const { - return KeyColumns; - } - - const TVector& GetKeyRanges() const { - return KeyRanges; - } - - bool IsSingleRange() const { - return KeyRanges.size() == 1; - } - - bool IsFullScan() const { - return IsSingleRange() && KeyRanges.front().IsFullScan(); - } - - void Print(IOutputStream& output) const; - -private: - TVector KeyColumns; - TVector KeyRanges; -}; - -TTableLookup ExtractTableLookup( - NNodes::TExprBase row, - NNodes::TExprBase predicate, - const TVector& keyColumns, - TTableLookup::TGetValueFunc getValueFunc, - TTableLookup::TCanCompareFunc canCompareFunc, - TTableLookup::TCompareFunc compareFunc, - TExprContext& exprCtx, - bool allowNullCompare); - -bool KeyTupleLess(const TKeyTuple& left, const TKeyTuple& right, const TTableLookup::TCompareFunc& cmpFunc); - -} // namespace NCommon -} // namespace NYql diff --git a/ydb/tools/query_replay_yt/query_compiler.cpp b/ydb/tools/query_replay_yt/query_compiler.cpp index d3e410973d52..ac72f0fe169e 100644 --- a/ydb/tools/query_replay_yt/query_compiler.cpp +++ b/ydb/tools/query_replay_yt/query_compiler.cpp @@ -233,7 +233,6 @@ class TReplayCompileActor: public TActorBootstrapped { , HttpGateway(std::move(httpGateway)) { Config->EnableKqpScanQueryStreamLookup = true; - Config->PredicateExtract20 = true; Config->EnablePreparedDdl = true; }