Skip to content

Commit

Permalink
Extend TTL syntax to support tiers
Browse files Browse the repository at this point in the history 
RFC: **[nda.ya.ru/t/JsIT3hp679nYxn](https://nda.ya.ru/t/JsIT3hp679nYxn)**
commit_hash:a0a4f65b24ee591cb76fd3cf253ffe24a01bfaf5
  • Loading branch information
yentsovsemyon committed Nov 21, 2024
1 parent 90a8f8c commit 4743a95
Show file tree
Hide file tree
Showing 10 changed files with 332 additions and 25 deletions.
8 changes: 7 additions & 1 deletion yql/essentials/sql/v1/SQLv1.g.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -791,10 +791,16 @@ table_setting_value:
| STRING_VALUE
| integer
| split_boundaries
| expr ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
| ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
| bool_value
;

ttl_tier_list: expr (ttl_tier_action (COMMA expr ttl_tier_action)*)?;
ttl_tier_action:
TO EXTERNAL DATA SOURCE an_id
| DELETE
;

family_entry: FAMILY an_id family_settings;
family_settings: LPAREN (family_settings_entry (COMMA family_settings_entry)*)? RPAREN;
family_settings_entry: an_id EQUALS family_setting_value;
Expand Down
8 changes: 7 additions & 1 deletion yql/essentials/sql/v1/SQLv1Antlr4.g.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,10 +790,16 @@ table_setting_value:
| STRING_VALUE
| integer
| split_boundaries
| expr ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
| ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
| bool_value
;

ttl_tier_list: expr (ttl_tier_action (COMMA expr ttl_tier_action)*)?;
ttl_tier_action:
TO EXTERNAL DATA SOURCE an_id
| DELETE
;

family_entry: FAMILY an_id family_settings;
family_settings: LPAREN (family_settings_entry (COMMA family_settings_entry)*)? RPAREN;
family_settings_entry: an_id EQUALS family_setting_value;
Expand Down
62 changes: 62 additions & 0 deletions yql/essentials/sql/v1/format/sql_format.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2495,6 +2495,66 @@ friend struct TStaticData;
Visit(msg.GetToken5());
}

void VisitTableSettingValue(const TRule_table_setting_value& msg) {
switch (msg.GetAltCase()) {
case TRule_table_setting_value::kAltTableSettingValue5: {
// | ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
const auto& ttlSettings = msg.GetAlt_table_setting_value5();
const auto& tierList = ttlSettings.GetRule_ttl_tier_list1();
const bool needIndent = tierList.HasBlock2() && tierList.GetBlock2().Block2Size() > 0; // more then one tier
if (needIndent) {
NewLine();
PushCurrentIndent();
Visit(tierList.GetRule_expr1());
VisitTtlTierAction(tierList.GetBlock2().GetRule_ttl_tier_action1());

for (const auto& tierEntry : tierList.GetBlock2().GetBlock2()) {
Visit(tierEntry.GetToken1()); // comma
NewLine();
Visit(tierEntry.GetRule_expr2());
VisitTtlTierAction(tierEntry.GetRule_ttl_tier_action3());
}

PopCurrentIndent();
NewLine();
} else {
Visit(tierList.GetRule_expr1());
if (tierList.HasBlock2()) {
VisitTtlTierAction(tierList.GetBlock2().GetRule_ttl_tier_action1());
}
}

VisitKeyword(ttlSettings.GetToken2());
Visit(ttlSettings.GetRule_an_id3());
if (ttlSettings.HasBlock4()) {
VisitKeyword(ttlSettings.GetBlock4().GetToken1());
VisitKeyword(ttlSettings.GetBlock4().GetToken2());
}
} break;
default:
VisitAllFields(TRule_table_setting_value::GetDescriptor(), msg);
}
}

void VisitTtlTierAction(const TRule_ttl_tier_action& msg) {
switch (msg.GetAltCase()) {
case TRule_ttl_tier_action::kAltTtlTierAction1:
// | TO EXTERNAL DATA SOURCE an_id
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken1());
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken2());
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken3());
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken4());
Visit(msg.GetAlt_ttl_tier_action1().GetRule_an_id5());
break;
case TRule_ttl_tier_action::kAltTtlTierAction2:
// | DELETE
VisitKeyword(msg.GetAlt_ttl_tier_action2().GetToken1());
break;
case TRule_ttl_tier_action::ALT_NOT_SET:
break;
}
}

void VisitExpr(const TRule_expr& msg) {
if (msg.HasAlt_expr2()) {
Visit(msg.GetAlt_expr2());
Expand Down Expand Up @@ -2783,6 +2843,8 @@ TStaticData::TStaticData()
{TRule_case_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCaseExpr)},
{TRule_when_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitWhenExpr)},
{TRule_with_table_settings::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitWithTableSettingsExpr)},
{TRule_table_setting_value::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitTableSettingValue)},
{TRule_ttl_tier_action::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitTtlTierAction)},

{TRule_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitExpr)},
{TRule_or_subexpr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitOrSubexpr)},
Expand Down
10 changes: 10 additions & 0 deletions yql/essentials/sql/v1/format/sql_format_ut.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,16 @@ Y_UNIT_TEST(CreateTable) {
"CREATE TABLE user (\n\tuser int32\n)\nWITH (ttl = interval('P1D') ON user AS MICROSECONDS);\n"},
{"create table user(user int32) with (ttl=interval('P1D') on user as nAnOsEcOnDs)",
"CREATE TABLE user (\n\tuser int32\n)\nWITH (ttl = interval('P1D') ON user AS NANOSECONDS);\n"},
{"create table user(user int32) with (ttl=interval('P1D') delete on user as nAnOsEcOnDs)",
"CREATE TABLE user (\n\tuser int32\n)\nWITH (ttl = interval('P1D') DELETE ON user AS NANOSECONDS);\n"},
{"create table user(user int32) with (ttl=interval('P1D')to external data source tier1 ,interval('P10D')delete on user as seconds)",
"CREATE TABLE user (\n"
"\tuser int32\n"
")\n"
"WITH (ttl =\n"
"\tinterval('P1D') TO EXTERNAL DATA SOURCE tier1,\n"
"\tinterval('P10D') DELETE\n"
"ON user AS SECONDS);\n"},
{"create table user(index user global unique sync on (user,user) with (user=user,user=user))",
"CREATE TABLE user (\n\tINDEX user GLOBAL UNIQUE SYNC ON (user, user) WITH (user = user, user = user)\n);\n"},
{"create table user(index user global async on (user) with (user=user,))",
Expand Down
21 changes: 13 additions & 8 deletions yql/essentials/sql/v1/node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1894,9 +1894,14 @@ TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, co
(ctx.AnsiQuotedIdentifiers && input.StartsWith('"'))? EStringContentMode::AnsiIdent : EStringContentMode::Default);
}

TTtlSettings::TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr, const TMaybe<EUnit>& columnUnit)
TTtlSettings::TTierSettings::TTierSettings(const TNodePtr& evictionDelay, const std::optional<TIdentifier>& storageName)
: EvictionDelay(evictionDelay)
, StorageName(storageName) {
}

TTtlSettings::TTtlSettings(const TIdentifier& columnName, const std::vector<TTierSettings>& tiers, const TMaybe<EUnit>& columnUnit)
: ColumnName(columnName)
, Expr(expr)
, Tiers(tiers)
, ColumnUnit(columnUnit)
{
}
Expand Down Expand Up @@ -3131,10 +3136,10 @@ class TCalcOverWindow final: public INode {
Y_DEBUG_ABORT_UNLESS(FuncNode);
FuncNode->VisitTree(func, visited);
}

void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override {
if (ctx.DistinctOverWindow) {
FuncNode->CollectPreaggregateExprs(ctx, src, exprs);
FuncNode->CollectPreaggregateExprs(ctx, src, exprs);
} else {
INode::CollectPreaggregateExprs(ctx, src, exprs);
}
Expand Down Expand Up @@ -3274,7 +3279,7 @@ TSourcePtr TryMakeSourceFromExpression(TPosition pos, TContext& ctx, const TStri
return nullptr;
}

auto wrappedNode = new TAstListNodeImpl(pos, {
auto wrappedNode = new TAstListNodeImpl(pos, {
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
node
});
Expand Down Expand Up @@ -3303,7 +3308,7 @@ void MakeTableFromExpression(TPosition pos, TContext& ctx, TNodePtr node, TDefer
node = node->Y("Concat", node->Y("String", node->Q(prefix)), node);
}

auto wrappedNode = new TAstListNodeImpl(pos, {
auto wrappedNode = new TAstListNodeImpl(pos, {
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
node
});
Expand All @@ -3320,7 +3325,7 @@ TDeferredAtom MakeAtomFromExpression(TPosition pos, TContext& ctx, TNodePtr node
node = node->Y("Concat", node->Y("String", node->Q(prefix)), node);
}

auto wrappedNode = new TAstListNodeImpl(pos, {
auto wrappedNode = new TAstListNodeImpl(pos, {
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
node
});
Expand Down Expand Up @@ -3462,7 +3467,7 @@ bool TVectorIndexSettings::Validate(TContext& ctx) const {
if (!Distance && !Similarity) {
ctx.Error() << "either distance or similarity should be set";
return false;
}
}
if (!VectorType) {
ctx.Error() << "vector_type should be set";
return false;
Expand Down
11 changes: 9 additions & 2 deletions yql/essentials/sql/v1/node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1112,11 +1112,18 @@ namespace NSQLTranslationV1 {
Nanoseconds /* "nanoseconds" */,
};

struct TTierSettings {
TNodePtr EvictionDelay;
std::optional<TIdentifier> StorageName;

TTierSettings(const TNodePtr& evictionDelay, const std::optional<TIdentifier>& storageName = std::nullopt);
};

TIdentifier ColumnName;
TNodePtr Expr;
std::vector<TTierSettings> Tiers;
TMaybe<EUnit> ColumnUnit;

TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr, const TMaybe<EUnit>& columnUnit = {});
TTtlSettings(const TIdentifier& columnName, const std::vector<TTierSettings>& tiers, const TMaybe<EUnit>& columnUnit = {});
};

struct TTableSettings {
Expand Down
12 changes: 11 additions & 1 deletion yql/essentials/sql/v1/query.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,17 @@ static INode::TPtr CreateTableSettings(const TTableSettings& tableSettings, ETab
auto opts = Y();

opts = L(opts, Q(Y(Q("columnName"), BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name))));
opts = L(opts, Q(Y(Q("expireAfter"), ttlSettings.Expr)));

auto tiersDesc = Y();
for (const auto& tier : ttlSettings.Tiers) {
auto tierDesc = Y();
tierDesc = L(tierDesc, Q(Y(Q("evictionDelay"), tier.EvictionDelay)));
if (tier.StorageName) {
tierDesc = L(tierDesc, Q(Y(Q("storageName"), BuildQuotedAtom(tier.StorageName->Pos, tier.StorageName->Name))));
}
tiersDesc = L(tiersDesc, Q(tierDesc));
}
opts = L(opts, Q(Y(Q("tiers"), Q(tiersDesc))));

if (ttlSettings.ColumnUnit) {
opts = L(opts, Q(Y(Q("columnUnit"), Q(ToString(*ttlSettings.ColumnUnit)))));
Expand Down
65 changes: 57 additions & 8 deletions yql/essentials/sql/v1/sql_translation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1963,19 +1963,68 @@ namespace {
return true;
}

bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to,
TSqlExpression& expr, TContext& ctx, TTranslation& txc) {
bool FillTieringInterval(const TRule_expr& from, TNodePtr& tieringInterval, TSqlExpression& expr, TContext& ctx) {
auto exprNode = expr.Build(from);
if (!exprNode) {
return false;
}

if (exprNode->GetOpName() != "Interval") {
ctx.Error() << "Literal of Interval type is expected for TTL";
return false;
}

tieringInterval = exprNode;
return true;
}

bool FillTierAction(const TRule_ttl_tier_action& from, std::optional<TIdentifier>& storageName, TTranslation& txc) {
switch (from.GetAltCase()) {
case TRule_ttl_tier_action::kAltTtlTierAction1:
storageName = IdEx(from.GetAlt_ttl_tier_action1().GetRule_an_id5(), txc);
break;
case TRule_ttl_tier_action::kAltTtlTierAction2:
storageName.reset();
break;
case TRule_ttl_tier_action::ALT_NOT_SET:
Y_ABORT("You should change implementation according to grammar changes");
}
return true;
}

bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to, TSqlExpression& expr, TContext& ctx,
TTranslation& txc) {
switch (from.Alt_case()) {
case TRule_table_setting_value::kAltTableSettingValue5: {
auto columnName = IdEx(from.GetAlt_table_setting_value5().GetRule_an_id3(), txc);
auto exprNode = expr.Build(from.GetAlt_table_setting_value5().GetRule_expr1());
if (!exprNode) {
auto tiersLiteral = from.GetAlt_table_setting_value5().GetRule_ttl_tier_list1();

TNodePtr firstInterval;
if (!FillTieringInterval(tiersLiteral.GetRule_expr1(), firstInterval, expr, ctx)) {
return false;
}

if (exprNode->GetOpName() != "Interval") {
ctx.Error() << "Literal of Interval type is expected for TTL";
return false;
std::vector<TTtlSettings::TTierSettings> tiers;
if (!tiersLiteral.HasBlock2()) {
tiers.emplace_back(firstInterval);
} else {
std::optional<TIdentifier> firstStorageName;
if (!FillTierAction(tiersLiteral.GetBlock2().GetRule_ttl_tier_action1(), firstStorageName, txc)) {
return false;
}
tiers.emplace_back(firstInterval, firstStorageName);

for (const auto& tierLiteral : tiersLiteral.GetBlock2().GetBlock2()) {
TNodePtr intervalExpr;
if (!FillTieringInterval(tierLiteral.GetRule_expr2(), intervalExpr, expr, ctx)) {
return false;
}
std::optional<TIdentifier> storageName;
if (!FillTierAction(tierLiteral.GetRule_ttl_tier_action3(), storageName, txc)) {
return false;
}
tiers.emplace_back(intervalExpr, storageName);
}
}

TMaybe<TTtlSettings::EUnit> columnUnit;
Expand All @@ -1988,7 +2037,7 @@ namespace {
}
}

to.Set(TTtlSettings(columnName, exprNode, columnUnit));
to.Set(TTtlSettings(columnName, tiers, columnUnit));
break;
}
default:
Expand Down
Loading

0 comments on commit 4743a95

Please sign in to comment.