Skip to content

Commit

Permalink
Extend TTL syntax to support tiers
Browse files Browse the repository at this point in the history 
RFC: **[nda.ya.ru/t/JsIT3hp679nYxn](https://nda.ya.ru/t/JsIT3hp679nYxn)**
commit_hash:a0a4f65b24ee591cb76fd3cf253ffe24a01bfaf5

Conflicts:
	ydb/library/yql/sql/v1/node.cpp
	yql/essentials/sql/v1/SQLv1Antlr4.g.in
	yql/essentials/sql/v1/format/sql_format_ut.h
	yql/essentials/sql/v1/sql_ut_antlr4.cpp
  • Loading branch information
@zverevgeny
yentsovsemyon authored and zverevgeny committed Jan 5, 2025
1 parent 32caab1 commit 5698710
Show file tree
Hide file tree
Showing 7 changed files with 234 additions and 19 deletions.
8 changes: 7 additions & 1 deletion ydb/library/yql/sql/v1/SQLv1.g.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -748,10 +748,16 @@ table_setting_value:
| STRING_VALUE
| integer
| split_boundaries
| expr ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
| ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
| bool_value
;

ttl_tier_list: expr (ttl_tier_action (COMMA expr ttl_tier_action)*)?;
ttl_tier_action:
TO EXTERNAL DATA SOURCE an_id
| DELETE
;

family_entry: FAMILY an_id family_settings;
family_settings: LPAREN (family_settings_entry (COMMA family_settings_entry)*)? RPAREN;
family_settings_entry: an_id EQUALS family_setting_value;
Expand Down
62 changes: 62 additions & 0 deletions ydb/library/yql/sql/v1/format/sql_format.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2436,6 +2436,66 @@ friend struct TStaticData;
Visit(msg.GetToken5());
}

void VisitTableSettingValue(const TRule_table_setting_value& msg) {
switch (msg.GetAltCase()) {
case TRule_table_setting_value::kAltTableSettingValue5: {
// | ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
const auto& ttlSettings = msg.GetAlt_table_setting_value5();
const auto& tierList = ttlSettings.GetRule_ttl_tier_list1();
const bool needIndent = tierList.HasBlock2() && tierList.GetBlock2().Block2Size() > 0; // more then one tier
if (needIndent) {
NewLine();
PushCurrentIndent();
Visit(tierList.GetRule_expr1());
VisitTtlTierAction(tierList.GetBlock2().GetRule_ttl_tier_action1());

for (const auto& tierEntry : tierList.GetBlock2().GetBlock2()) {
Visit(tierEntry.GetToken1()); // comma
NewLine();
Visit(tierEntry.GetRule_expr2());
VisitTtlTierAction(tierEntry.GetRule_ttl_tier_action3());
}

PopCurrentIndent();
NewLine();
} else {
Visit(tierList.GetRule_expr1());
if (tierList.HasBlock2()) {
VisitTtlTierAction(tierList.GetBlock2().GetRule_ttl_tier_action1());
}
}

VisitKeyword(ttlSettings.GetToken2());
Visit(ttlSettings.GetRule_an_id3());
if (ttlSettings.HasBlock4()) {
VisitKeyword(ttlSettings.GetBlock4().GetToken1());
VisitKeyword(ttlSettings.GetBlock4().GetToken2());
}
} break;
default:
VisitAllFields(TRule_table_setting_value::GetDescriptor(), msg);
}
}

void VisitTtlTierAction(const TRule_ttl_tier_action& msg) {
switch (msg.GetAltCase()) {
case TRule_ttl_tier_action::kAltTtlTierAction1:
// | TO EXTERNAL DATA SOURCE an_id
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken1());
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken2());
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken3());
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken4());
Visit(msg.GetAlt_ttl_tier_action1().GetRule_an_id5());
break;
case TRule_ttl_tier_action::kAltTtlTierAction2:
// | DELETE
VisitKeyword(msg.GetAlt_ttl_tier_action2().GetToken1());
break;
case TRule_ttl_tier_action::ALT_NOT_SET:
break;
}
}

void VisitExpr(const TRule_expr& msg) {
if (msg.HasAlt_expr2()) {
Visit(msg.GetAlt_expr2());
Expand Down Expand Up @@ -2724,6 +2784,8 @@ TStaticData::TStaticData()
{TRule_case_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCaseExpr)},
{TRule_when_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitWhenExpr)},
{TRule_with_table_settings::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitWithTableSettingsExpr)},
{TRule_table_setting_value::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitTableSettingValue)},
{TRule_ttl_tier_action::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitTtlTierAction)},

{TRule_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitExpr)},
{TRule_or_subexpr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitOrSubexpr)},
Expand Down
15 changes: 10 additions & 5 deletions ydb/library/yql/sql/v1/node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1853,9 +1853,14 @@ TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, co
(ctx.AnsiQuotedIdentifiers && input.StartsWith('"'))? EStringContentMode::AnsiIdent : EStringContentMode::Default);
}

TTtlSettings::TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr, const TMaybe<EUnit>& columnUnit)
TTtlSettings::TTierSettings::TTierSettings(const TNodePtr& evictionDelay, const std::optional<TIdentifier>& storageName)
: EvictionDelay(evictionDelay)
, StorageName(storageName) {
}

TTtlSettings::TTtlSettings(const TIdentifier& columnName, const std::vector<TTierSettings>& tiers, const TMaybe<EUnit>& columnUnit)
: ColumnName(columnName)
, Expr(expr)
, Tiers(tiers)
, ColumnUnit(columnUnit)
{
}
Expand Down Expand Up @@ -3225,7 +3230,7 @@ TSourcePtr TryMakeSourceFromExpression(TPosition pos, TContext& ctx, const TStri
return nullptr;
}

auto wrappedNode = new TAstListNodeImpl(pos, {
auto wrappedNode = new TAstListNodeImpl(pos, {
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
node
});
Expand Down Expand Up @@ -3254,7 +3259,7 @@ void MakeTableFromExpression(TPosition pos, TContext& ctx, TNodePtr node, TDefer
node = node->Y("Concat", node->Y("String", node->Q(prefix)), node);
}

auto wrappedNode = new TAstListNodeImpl(pos, {
auto wrappedNode = new TAstListNodeImpl(pos, {
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
node
});
Expand All @@ -3271,7 +3276,7 @@ TDeferredAtom MakeAtomFromExpression(TPosition pos, TContext& ctx, TNodePtr node
node = node->Y("Concat", node->Y("String", node->Q(prefix)), node);
}

auto wrappedNode = new TAstListNodeImpl(pos, {
auto wrappedNode = new TAstListNodeImpl(pos, {
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
node
});
Expand Down
11 changes: 9 additions & 2 deletions ydb/library/yql/sql/v1/node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1098,11 +1098,18 @@ namespace NSQLTranslationV1 {
Nanoseconds /* "nanoseconds" */,
};

struct TTierSettings {
TNodePtr EvictionDelay;
std::optional<TIdentifier> StorageName;

TTierSettings(const TNodePtr& evictionDelay, const std::optional<TIdentifier>& storageName = std::nullopt);
};

TIdentifier ColumnName;
TNodePtr Expr;
std::vector<TTierSettings> Tiers;
TMaybe<EUnit> ColumnUnit;

TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr, const TMaybe<EUnit>& columnUnit = {});
TTtlSettings(const TIdentifier& columnName, const std::vector<TTierSettings>& tiers, const TMaybe<EUnit>& columnUnit = {});
};

struct TTableSettings {
Expand Down
12 changes: 11 additions & 1 deletion ydb/library/yql/sql/v1/query.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,17 @@ static INode::TPtr CreateTableSettings(const TTableSettings& tableSettings, ETab
auto opts = Y();

opts = L(opts, Q(Y(Q("columnName"), BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name))));
opts = L(opts, Q(Y(Q("expireAfter"), ttlSettings.Expr)));

auto tiersDesc = Y();
for (const auto& tier : ttlSettings.Tiers) {
auto tierDesc = Y();
tierDesc = L(tierDesc, Q(Y(Q("evictionDelay"), tier.EvictionDelay)));
if (tier.StorageName) {
tierDesc = L(tierDesc, Q(Y(Q("storageName"), BuildQuotedAtom(tier.StorageName->Pos, tier.StorageName->Name))));
}
tiersDesc = L(tiersDesc, Q(tierDesc));
}
opts = L(opts, Q(Y(Q("tiers"), Q(tiersDesc))));

if (ttlSettings.ColumnUnit) {
opts = L(opts, Q(Y(Q("columnUnit"), Q(ToString(*ttlSettings.ColumnUnit)))));
Expand Down
65 changes: 57 additions & 8 deletions ydb/library/yql/sql/v1/sql_translation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1802,19 +1802,68 @@ namespace {
return true;
}

bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to,
TSqlExpression& expr, TContext& ctx, TTranslation& txc) {
bool FillTieringInterval(const TRule_expr& from, TNodePtr& tieringInterval, TSqlExpression& expr, TContext& ctx) {
auto exprNode = expr.Build(from);
if (!exprNode) {
return false;
}

if (exprNode->GetOpName() != "Interval") {
ctx.Error() << "Literal of Interval type is expected for TTL";
return false;
}

tieringInterval = exprNode;
return true;
}

bool FillTierAction(const TRule_ttl_tier_action& from, std::optional<TIdentifier>& storageName, TTranslation& txc) {
switch (from.GetAltCase()) {
case TRule_ttl_tier_action::kAltTtlTierAction1:
storageName = IdEx(from.GetAlt_ttl_tier_action1().GetRule_an_id5(), txc);
break;
case TRule_ttl_tier_action::kAltTtlTierAction2:
storageName.reset();
break;
case TRule_ttl_tier_action::ALT_NOT_SET:
Y_ABORT("You should change implementation according to grammar changes");
}
return true;
}

bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to, TSqlExpression& expr, TContext& ctx,
TTranslation& txc) {
switch (from.Alt_case()) {
case TRule_table_setting_value::kAltTableSettingValue5: {
auto columnName = IdEx(from.GetAlt_table_setting_value5().GetRule_an_id3(), txc);
auto exprNode = expr.Build(from.GetAlt_table_setting_value5().GetRule_expr1());
if (!exprNode) {
auto tiersLiteral = from.GetAlt_table_setting_value5().GetRule_ttl_tier_list1();

TNodePtr firstInterval;
if (!FillTieringInterval(tiersLiteral.GetRule_expr1(), firstInterval, expr, ctx)) {
return false;
}

if (exprNode->GetOpName() != "Interval") {
ctx.Error() << "Literal of Interval type is expected for TTL";
return false;
std::vector<TTtlSettings::TTierSettings> tiers;
if (!tiersLiteral.HasBlock2()) {
tiers.emplace_back(firstInterval);
} else {
std::optional<TIdentifier> firstStorageName;
if (!FillTierAction(tiersLiteral.GetBlock2().GetRule_ttl_tier_action1(), firstStorageName, txc)) {
return false;
}
tiers.emplace_back(firstInterval, firstStorageName);

for (const auto& tierLiteral : tiersLiteral.GetBlock2().GetBlock2()) {
TNodePtr intervalExpr;
if (!FillTieringInterval(tierLiteral.GetRule_expr2(), intervalExpr, expr, ctx)) {
return false;
}
std::optional<TIdentifier> storageName;
if (!FillTierAction(tierLiteral.GetRule_ttl_tier_action3(), storageName, txc)) {
return false;
}
tiers.emplace_back(intervalExpr, storageName);
}
}

TMaybe<TTtlSettings::EUnit> columnUnit;
Expand All @@ -1827,7 +1876,7 @@ namespace {
}
}

to.Set(TTtlSettings(columnName, exprNode, columnUnit));
to.Set(TTtlSettings(columnName, tiers, columnUnit));
break;
}
default:
Expand Down
80 changes: 78 additions & 2 deletions ydb/library/yql/sql/v1/sql_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2026,7 +2026,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
if (word == "Write") {
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("expireAfter"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000"));
}
};
Expand All @@ -2048,7 +2049,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
if (word == "Write") {
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("expireAfter"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds"));
Expand All @@ -2061,6 +2063,80 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
}

Y_UNIT_TEST(TtlTieringParseCorrect) {
NYql::TAstParseResult res = SqlToYql(
R"( USE plato;
CREATE TABLE tableName (Key Uint32, CreatedAt Uint32, PRIMARY KEY (Key))
WITH (TTL =
Interval("P1D") TO EXTERNAL DATA SOURCE Tier1,
Interval("P2D") TO EXTERNAL DATA SOURCE Tier2,
Interval("P30D") DELETE
ON CreatedAt AS SECONDS);)"
);
UNIT_ASSERT(res.Root);

TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
if (word == "Write") {
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("columnUnit"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("seconds"));
}
};

TWordCountHive elementStat = { {TString("Write"), 0} };
VerifyProgram(res, elementStat, verifyLine);

UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
}

Y_UNIT_TEST(TtlTieringWithOtherActionsParseCorrect) {
NYql::TAstParseResult res = SqlToYql(
R"( USE plato;
ALTER TABLE tableName
ADD FAMILY cold (DATA = "rot"),
SET TTL
Interval("P1D") TO EXTERNAL DATA SOURCE Tier1,
Interval("P2D") TO EXTERNAL DATA SOURCE Tier2,
Interval("P30D") DELETE
ON CreatedAt,
ALTER COLUMN payload_v2 SET FAMILY cold,
ALTER FAMILY default SET DATA "ssd"
;)"
);
UNIT_ASSERT(res.Root);

TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
if (word == "Write") {
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("addColumnFamilies"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("cold"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("alterColumnFamilies"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("default"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("setTtlSettings"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("tiers"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("evictionDelay"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("storageName"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier1"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("Tier2"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("86400000"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("172800000"));
UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("2592000000"));
}
};

TWordCountHive elementStat = { {TString("Write"), 0} };
VerifyProgram(res, elementStat, verifyLine);

UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
}

Y_UNIT_TEST(TieringParseCorrect) {
NYql::TAstParseResult res = SqlToYql(
R"( USE plato;
Expand Down

0 comments on commit 5698710

Please sign in to comment.