Skip to content

Commit

Permalink
init
Browse files Browse the repository at this point in the history
  • Loading branch information
@vitstn
vitstn committed Oct 28, 2024
1 parent 634dc20 commit a3f92d2
Show file tree
Hide file tree
Showing 33 changed files with 640 additions and 26 deletions.
1 change: 1 addition & 0 deletions ydb/library/yql/sql/v1/context.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = {
{"ValidateUnusedExprs", &TContext::ValidateUnusedExprs},
{"AnsiImplicitCrossJoin", &TContext::AnsiImplicitCrossJoin},
{"DistinctOverWindow", &TContext::DistinctOverWindow},
{"SeqMode", &TContext::SeqMode},
};

typedef TMaybe<bool> TContext::*TPragmaMaybeField;
Expand Down
1 change: 1 addition & 0 deletions ydb/library/yql/sql/v1/context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ namespace NSQLTranslationV1 {
bool ValidateUnusedExprs = false;
bool AnsiImplicitCrossJoin = false; // select * from A,B
bool DistinctOverWindow = false;
bool SeqMode = false;
};

class TColumnRefScope {
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/sql/v1/node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1521,7 +1521,7 @@ namespace NSQLTranslationV1 {
TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings);
TNodePtr BuildCommitClusters(TPosition pos);
TNodePtr BuildRollbackClusters(TPosition pos);
TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped);
TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq);
TNodePtr BuildPragma(TPosition pos, const TString& prefix, const TString& name, const TVector<TDeferredAtom>& values, bool valueDefault);
TNodePtr BuildSqlLambda(TPosition pos, TVector<TString>&& args, TVector<TNodePtr>&& exprSeq);
TNodePtr BuildWorldIfNode(TPosition pos, TNodePtr predicate, TNodePtr thenNode, TNodePtr elseNode, bool isEvaluate);
Expand Down
91 changes: 70 additions & 21 deletions ydb/library/yql/sql/v1/query.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2694,15 +2694,44 @@ TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings

class TYqlProgramNode: public TAstListNode {
public:
TYqlProgramNode(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped)
TYqlProgramNode(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq)
: TAstListNode(pos)
, Blocks(blocks)
, TopLevel(topLevel)
, Scoped(scoped)
, UseSeq(useSeq)
{}

bool DoInit(TContext& ctx, ISource* src) override {
bool hasError = false;
INode::TPtr currentWorldsHolder;
INode::TPtr seqNode;
if (UseSeq) {
currentWorldsHolder = new TAstListNodeImpl(GetPos());
seqNode = new TAstListNodeImpl(GetPos());
seqNode->Add("Seq!","world");
}

INode* currentWorlds = UseSeq ? currentWorldsHolder.Get() : this;
auto flushCurrentWorlds = [&](bool changeSeq, bool finish) {
currentWorldsHolder->Add(Y("return","world"));
auto lambda = BuildLambda(GetPos(), Y("world"), Y("block", Q(currentWorldsHolder)));
seqNode->Add(lambda);

if (finish) {
Add(Y("let", "world", seqNode));
} else {
currentWorldsHolder = new TAstListNodeImpl(GetPos());
currentWorlds = currentWorldsHolder.Get();
}

if (changeSeq) {
Add(Y("let", "world", seqNode));
seqNode = new TAstListNodeImpl(GetPos());
seqNode->Add("Seq!","world");
}
};

if (TopLevel) {
for (auto& var: ctx.Variables) {
if (!var.second.second->Init(ctx, src)) {
Expand Down Expand Up @@ -2799,33 +2828,33 @@ class TYqlProgramNode: public TAstListNode {
auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName)));

for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()),
BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction()))))));
}

if (ctx.ResultSizeLimit > 0) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink,
BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit)))));
}

if (!ctx.PragmaPullUpFlatMapOverJoin) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin"))));
}

if (ctx.FilterPushdownOverJoinOptionalSide) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "FilterPushdownOverJoinOptionalSide"))));
}

if (!ctx.RotateJoinTree) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "RotateJoinTree"), BuildQuotedAtom(Pos, "false"))));
}

if (ctx.DiscoveryMode) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "DiscoveryMode"))));
}

Expand All @@ -2836,12 +2865,12 @@ class TYqlProgramNode: public TAstListNode {
} else if (ctx.DqEngineForce) {
mode = "force";
}
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "DqEngine"), BuildQuotedAtom(Pos, mode))));
}

if (ctx.CostBasedOptimizer) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "CostBasedOptimizer"), BuildQuotedAtom(Pos, ctx.CostBasedOptimizer))));
}

Expand All @@ -2851,43 +2880,43 @@ class TYqlProgramNode: public TAstListNode {
pragmaName = "JsonQueryReturnsJsonDocument";
}

Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
}

if (ctx.OrderedColumns) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "OrderedColumns"))));
}

if (ctx.PqReadByRtmrCluster) {
auto pqSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(PqProviderName)), BuildQuotedAtom(Pos, "$all"));
Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll,
BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster))));

auto rtmrSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, "$all"));
Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll,
BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster))));

if (ctx.PqReadByRtmrCluster != "dq") {
// set any dynamic settings for particular RTMR cluster for CommitAll!
auto rtmrSource = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster));
Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource,
BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "Dummy_"), BuildQuotedAtom(Pos, "1"))));
}
}

if (ctx.YsonCastToString.Defined()) {
const TString pragmaName = *ctx.YsonCastToString ? "YsonCastToString" : "DisableYsonCastToString";
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
}

if (ctx.UseBlocks) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "UseBlocks"))));
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "UseBlocks"))));
}

if (ctx.BlockEngineEnable) {
TString mode = ctx.BlockEngineForce ? "force" : "auto";
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "BlockEngine"), BuildQuotedAtom(Pos, mode))));
}
}
Expand Down Expand Up @@ -2915,22 +2944,41 @@ class TYqlProgramNode: public TAstListNode {
Add(Y("let", data.first, node));
}

if (UseSeq) {
flushCurrentWorlds(false, false);
}

for (auto& block: Blocks) {
const auto subqueryAliasPtr = block->SubqueryAlias();
if (subqueryAliasPtr) {
if (block->UsedSubquery()) {
if (UseSeq) {
flushCurrentWorlds(true, false);
}

const auto& ref = block->GetLabel();
YQL_ENSURE(!ref.empty());
Add(block);
Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0"))));
currentWorlds->Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0"))));
Add(Y("let", ref, Y("Nth", *subqueryAliasPtr, Q("1"))));
}
} else {
const auto& ref = block->GetLabel();
Add(Y("let", ref ? ref : "world", block));
if (ref) {
Add(Y("let", ref, block));
} else {
currentWorlds->Add(Y("let", "world", block));
if (UseSeq) {
flushCurrentWorlds(false, false);
}
}
}
}

if (UseSeq) {
flushCurrentWorlds(false, true);
}

if (TopLevel) {
if (ctx.UniversalAliases) {
decltype(Nodes) preparedNodes;
Expand Down Expand Up @@ -2975,10 +3023,11 @@ class TYqlProgramNode: public TAstListNode {
TVector<TNodePtr> Blocks;
const bool TopLevel;
TScopedStatePtr Scoped;
const bool UseSeq;
};

TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped) {
return new TYqlProgramNode(pos, blocks, topLevel, scoped);
TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq) {
return new TYqlProgramNode(pos, blocks, topLevel, scoped, useSeq);
}

class TPragmaNode final: public INode {
Expand Down
10 changes: 8 additions & 2 deletions ydb/library/yql/sql/v1/sql_query.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2960,6 +2960,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else if (normalizedPragma == "disabledistinctoverwindow") {
Ctx.DistinctOverWindow = false;
Ctx.IncrementMonCounter("sql_pragma", "DisableDistinctOverWindow");
} else if (normalizedPragma == "seqmode") {
Ctx.SeqMode = true;
Ctx.IncrementMonCounter("sql_pragma", "SeqMode");
} else if (normalizedPragma == "disableseqmode") {
Ctx.SeqMode = false;
Ctx.IncrementMonCounter("sql_pragma", "DisableSeqMode");
} else {
Error() << "Unknown pragma: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "UnknownPragma");
Expand Down Expand Up @@ -3308,7 +3314,7 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) {
AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos()));
}

auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped);
auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped, Ctx.SeqMode);
WarnUnusedNodes();
return result;
}
Expand Down Expand Up @@ -3378,7 +3384,7 @@ TNodePtr TSqlQuery::Build(const std::vector<::NSQLv1Generated::TRule_sql_stmt_co
AddStatementToBlocks(blocks, BuildCommitClusters(Ctx.Pos()));
}

auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped);
auto result = BuildQuery(Ctx.Pos(), blocks, true, Ctx.Scoped, Ctx.SeqMode);
return result;
}
namespace {
Expand Down
4 changes: 2 additions & 2 deletions ydb/library/yql/sql/v1/sql_translation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4508,7 +4508,7 @@ TNodePtr TSqlTranslation::DoStatement(const TRule_do_stmt& stmt, bool makeLambda
TBlocks innerBlocks;

const bool hasValidBody = DefineActionOrSubqueryBody(query, innerBlocks, body);
auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr;
auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped, Ctx.SeqMode) : nullptr;
WarnUnusedNodes();
Ctx.ScopeLevel--;
Ctx.Scoped = saveScoped;
Expand Down Expand Up @@ -4621,7 +4621,7 @@ bool TSqlTranslation::DefineActionOrSubqueryStatement(const TRule_define_action_
return false;
}

auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped) : nullptr;
auto ret = hasValidBody ? BuildQuery(Ctx.Pos(), innerBlocks, false, Ctx.Scoped, Ctx.SeqMode) : nullptr;
WarnUnusedNodes();
Ctx.Scoped = saveScoped;
Ctx.ScopeLevel--;
Expand Down
25 changes: 25 additions & 0 deletions ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,29 @@
{
"test.test[SeqMode-shared_subquery_expr_after_commit-default.txt-Analyze]": [
{
"checksum": "73cd2136358e0f6a50f06140703d344d",
"size": 5769,
"uri": "https://{canondata_backend}/1871182/b1a07621126ede0b23bbe3e8dfe267adf8f37a78/resource.tar.gz#test.test_SeqMode-shared_subquery_expr_after_commit-default.txt-Analyze_/plan.txt"
},
{
"uri": "file://test.test_SeqMode-shared_subquery_expr_after_commit-default.txt-Analyze_/extracted"
}
],
"test.test[SeqMode-shared_subquery_expr_after_commit-default.txt-Debug]": [
{
"checksum": "26a3e3ddd721add06784b4423a455e2a",
"size": 2158,
"uri": "https://{canondata_backend}/1871182/b1a07621126ede0b23bbe3e8dfe267adf8f37a78/resource.tar.gz#test.test_SeqMode-shared_subquery_expr_after_commit-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[SeqMode-shared_subquery_expr_after_commit-default.txt-Plan]": [
{
"checksum": "a1a4f9852ea9c21bf83b47c360a6c604",
"size": 7588,
"uri": "https://{canondata_backend}/1871182/b1a07621126ede0b23bbe3e8dfe267adf8f37a78/resource.tar.gz#test.test_SeqMode-shared_subquery_expr_after_commit-default.txt-Plan_/plan.txt"
}
],
"test.test[SeqMode-shared_subquery_expr_after_commit-default.txt-Results]": [],
"test.test[action-action_opt_args-default.txt-Analyze]": [
{
"checksum": "a3b64a2cf9903b3868a2dd88a18fc46e",
Expand Down
3 changes: 3 additions & 0 deletions ...ondata/test.test_SeqMode-shared_subquery_expr_after_commit-default.txt-Analyze_/extracted
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<tmp_path>/program.sql:<main>: Info: Optimization

<tmp_path>/program.sql:<main>: Info: DQ cannot execute the query. Cause: table without statistics
22 changes: 22 additions & 0 deletions ydb/library/yql/tests/sql/dq_file/part16/canondata/result.json
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,26 @@
{
"test.test[SeqMode-simple1-default.txt-Analyze]": [
{
"checksum": "b4dd508a329723c74293d80f0278c705",
"size": 505,
"uri": "https://{canondata_backend}/1925821/53324a6da5db86940359e666456af74325f417cd/resource.tar.gz#test.test_SeqMode-simple1-default.txt-Analyze_/plan.txt"
}
],
"test.test[SeqMode-simple1-default.txt-Debug]": [
{
"checksum": "b40f67aff1a1f2d7dcf60ba1281e9d65",
"size": 289,
"uri": "https://{canondata_backend}/1925821/53324a6da5db86940359e666456af74325f417cd/resource.tar.gz#test.test_SeqMode-simple1-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[SeqMode-simple1-default.txt-Plan]": [
{
"checksum": "b4dd508a329723c74293d80f0278c705",
"size": 505,
"uri": "https://{canondata_backend}/1925821/53324a6da5db86940359e666456af74325f417cd/resource.tar.gz#test.test_SeqMode-simple1-default.txt-Plan_/plan.txt"
}
],
"test.test[SeqMode-simple1-default.txt-Results]": [],
"test.test[action-action_eval_cluster_use--Analyze]": [
{
"checksum": "4f1be07e94fcf0a8c67e043e37e0b610",
Expand Down
22 changes: 22 additions & 0 deletions ydb/library/yql/tests/sql/dq_file/part18/canondata/result.json
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,26 @@
{
"test.test[SeqMode-shared_subquery_expr-default.txt-Analyze]": [
{
"checksum": "a3b64a2cf9903b3868a2dd88a18fc46e",
"size": 922,
"uri": "https://{canondata_backend}/1871182/d5d4fe05765faf626fe33998e3ca4326f3cbee6a/resource.tar.gz#test.test_SeqMode-shared_subquery_expr-default.txt-Analyze_/plan.txt"
}
],
"test.test[SeqMode-shared_subquery_expr-default.txt-Debug]": [
{
"checksum": "2960fb9caf762052aa597a19df446228",
"size": 332,
"uri": "https://{canondata_backend}/1871182/d5d4fe05765faf626fe33998e3ca4326f3cbee6a/resource.tar.gz#test.test_SeqMode-shared_subquery_expr-default.txt-Debug_/opt.yql_patched"
}
],
"test.test[SeqMode-shared_subquery_expr-default.txt-Plan]": [
{
"checksum": "a3b64a2cf9903b3868a2dd88a18fc46e",
"size": 922,
"uri": "https://{canondata_backend}/1871182/d5d4fe05765faf626fe33998e3ca4326f3cbee6a/resource.tar.gz#test.test_SeqMode-shared_subquery_expr-default.txt-Plan_/plan.txt"
}
],
"test.test[SeqMode-shared_subquery_expr-default.txt-Results]": [],
"test.test[action-eval_input_output_table_subquery--Analyze]": [
{
"checksum": "302b334624dce8fdd832200067248ff7",
Expand Down
Loading

0 comments on commit a3f92d2

Please sign in to comment.