diff --git a/ydb/core/grpc_services/rpc_log_store.cpp b/ydb/core/grpc_services/rpc_log_store.cpp index bb84bc25d450..9f95a83d6048 100644 --- a/ydb/core/grpc_services/rpc_log_store.cpp +++ b/ydb/core/grpc_services/rpc_log_store.cpp @@ -508,29 +508,8 @@ class TDescribeLogTableRPC : public TRpcSchemeRequestActormutable_date_type_column(); - outTTL.set_column_name(inTTL.GetColumnName()); - outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); - break; - } - - case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: - case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: - case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: - case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: { - auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_value_since_unix_epoch(); - outTTL.set_column_name(inTTL.GetColumnName()); - outTTL.set_column_unit(static_cast(inTTL.GetColumnUnit())); - outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); - break; - } - - default: - break; + if (!FillTtlSettings(*describeLogTableResult.mutable_ttl_settings(), tableDescription.GetTtlSettings().GetEnabled(), status, error)) { + return Reply(status, error, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); } } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index 2244519c28ad..233058ee4a01 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -311,15 +311,16 @@ bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::Read void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto) { if (!settings.ColumnUnit) { - auto& opts = *proto.mutable_date_type_column(); + auto& opts = *proto.mutable_date_type_column_v1(); opts.set_column_name(settings.ColumnName); - opts.set_expire_after_seconds(settings.ExpireAfter.Seconds()); } else { - auto& opts = *proto.mutable_value_since_unix_epoch(); + auto& opts = *proto.mutable_value_since_unix_epoch_v1(); opts.set_column_name(settings.ColumnName); opts.set_column_unit(static_cast(*settings.ColumnUnit)); - opts.set_expire_after_seconds(settings.ExpireAfter.Seconds()); } + auto* deleteTier = proto.add_tiers(); + deleteTier->set_apply_after_seconds(settings.ExpireAfter.Seconds()); + deleteTier->mutable_delete_(); } Ydb::FeatureFlag::Status GetFlagValue(const TMaybe& value) { diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp b/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp index 280ef82f1fb3..113b4cfee997 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp +++ b/ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp @@ -35,6 +35,9 @@ NKikimrSchemeOp::TColumnDescription Col(const TString& columnName, NScheme::TTyp [[maybe_unused]] NKikimrSchemeOp::TTTLSettings TtlCol(const TString& columnName) { NKikimrSchemeOp::TTTLSettings settings; + auto* deleteTier = settings.MutableEnabled()->AddTiers(); + deleteTier->MutableDelete(); + deleteTier->SetApplyAfterSeconds(TDuration::Minutes(20).Seconds()); settings.MutableEnabled()->SetExpireAfterSeconds(TDuration::Minutes(20).Seconds()); settings.MutableEnabled()->SetColumnName(columnName); settings.MutableEnabled()->MutableSysSettings()->SetRunInterval(TDuration::Minutes(60).MicroSeconds()); diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 1ddfa0ccdb33..895cdd4b04d6 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -221,11 +221,24 @@ message TTTLSettings { optional uint32 MaxShardsInFlight = 6 [default = 0]; // zero means no limit } + message TEvictionToExternalStorageSettings { + optional string StorageName = 1; + } + + message TTier { + optional uint32 ApplyAfterSeconds = 1; + oneof Action { + google.protobuf.Empty Delete = 2; + TEvictionToExternalStorageSettings EvictToExternalStorage = 3; + } + } + message TEnabled { optional string ColumnName = 1; - optional uint32 ExpireAfterSeconds = 2; + optional uint32 ExpireAfterSeconds = 2 [deprecated = true]; optional EUnit ColumnUnit = 3; optional TSysSettings SysSettings = 4; + repeated TTier Tiers = 5; } message TDisabled { @@ -571,10 +584,11 @@ message TColumnDataLifeCycle { message TTtl { optional string ColumnName = 1; oneof Expire { - uint32 ExpireAfterSeconds = 2; + uint32 ExpireAfterSeconds = 2 [deprecated = true]; // ignored if Tiers are not empty uint64 ExpireAfterBytes = 4; } optional TTTLSettings.EUnit ColumnUnit = 3; + repeated TTTLSettings.TTier Tiers = 5; } message TDisabled { diff --git a/ydb/core/tx/schemeshard/common/validation.cpp b/ydb/core/tx/schemeshard/common/validation.cpp index d14e4aba416a..2ef0e4f89a37 100644 --- a/ydb/core/tx/schemeshard/common/validation.cpp +++ b/ydb/core/tx/schemeshard/common/validation.cpp @@ -57,4 +57,14 @@ bool TTTLValidator::ValidateUnit(const NScheme::TTypeInfo columnType, NKikimrSch return true; } -} \ No newline at end of file +bool TTTLValidator::ValidateTiers(const NKikimrSchemeOp::TTTLSettings::TEnabled ttlSettings, TString& errStr) { + for (ui64 i = 0; i < ttlSettings.TiersSize(); ++i) { + if (ttlSettings.GetTiers(i).HasDelete() && i + 1 != ttlSettings.TiersSize()) { + errStr = "Only the last tier in TTL settings can have Delete action"; + return false; + } + } + return true; +} + +} diff --git a/ydb/core/tx/schemeshard/common/validation.h b/ydb/core/tx/schemeshard/common/validation.h index 8a98dcf71a36..0ee739982ea7 100644 --- a/ydb/core/tx/schemeshard/common/validation.h +++ b/ydb/core/tx/schemeshard/common/validation.h @@ -9,5 +9,6 @@ namespace NKikimr::NSchemeShard::NValidation { class TTTLValidator { public: static bool ValidateUnit(const NScheme::TTypeInfo columnType, NKikimrSchemeOp::TTTLSettings::EUnit unit, TString& errStr); + static bool ValidateTiers(const NKikimrSchemeOp::TTTLSettings::TEnabled ttlSettings, TString& errStr); }; -} \ No newline at end of file +} diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h index 7a45468961b1..bd28cff6768a 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h @@ -25,6 +25,9 @@ class TConverterModifyToAlter { if (enabled.HasColumnUnit()) { alterEnabled->SetColumnUnit(enabled.GetColumnUnit()); } + for (const auto& tier : enabled.GetTiers()) { + alterEnabled->AddTiers()->CopyFrom(tier); + } } else if (tableTtl.HasDisabled()) { alterTtl->MutableDisabled(); } diff --git a/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp b/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp index 08b812a07dc8..62c4ed28ef76 100644 --- a/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp @@ -141,8 +141,16 @@ struct TSchemeShard::TTxRunConditionalErase: public TSchemeShard::TRwTxBase { } const auto& settings = tableInfo->TTLSettings().GetEnabled(); - const TDuration expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds()); - const TInstant wallClock = ctx.Now() - expireAfter; + + auto expireAfter = GetExpireAfter(settings, true); + if (expireAfter.IsFail()) { + LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Invalid TTL settings: " << expireAfter.GetErrorMessage() + << ": shardIdx: " << tableShardInfo.ShardIdx << ": pathId: " << shardInfo.PathId + << ", at schemeshard: " << Self->TabletID()); + return false; + } + const TInstant wallClock = ctx.Now() - *expireAfter; NKikimrTxDataShard::TEvConditionalEraseRowsRequest request; request.SetTableId(shardInfo.PathId.LocalPathId); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 96adfcbc4205..b41587c2ab14 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3606,6 +3606,8 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, const THashMap& colName2Id, const TSubDomainInfo& subDomain, TString& errStr); +TConclusion GetExpireAfter(const NKikimrSchemeOp::TTTLSettings::TEnabled& settings, const bool allowNonDeleteTiers); + std::optional> ValidateSequenceType(const TString& sequenceName, const TString& dataType, const NKikimr::NScheme::TTypeRegistry& typeRegistry, bool pgTypesEnabled, TString& errStr); diff --git a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp index b408d8cde301..ac27997fe55f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp @@ -1,7 +1,6 @@ #include "schemeshard_info_types.h" #include "common/validation.h" -#include "olap/columns/schema.h" #include @@ -58,8 +57,18 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, return false; } + if (!NValidation::TTTLValidator::ValidateTiers(enabled, errStr)) { + return false; + } + + const auto expireAfter = GetExpireAfter(enabled, false); + if (expireAfter.IsFail()) { + errStr = expireAfter.GetErrorMessage(); + return false; + } + const TInstant now = TInstant::Now(); - if (enabled.GetExpireAfterSeconds() > now.Seconds()) { + if (expireAfter->Seconds() > now.Seconds()) { errStr = Sprintf("TTL should be less than %" PRIu64 " seconds (%" PRIu64 " days, %" PRIu64 " years). The ttl behaviour is undefined before 1970.", now.Seconds(), now.Days(), now.Days() / 365); return false; } @@ -85,4 +94,20 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, return true; } +TConclusion GetExpireAfter(const NKikimrSchemeOp::TTTLSettings::TEnabled& settings, const bool allowNonDeleteTiers) { + if (settings.TiersSize()) { + for (const auto& tier : settings.GetTiers()) { + if (tier.HasDelete()) { + return TDuration::Seconds(tier.GetApplyAfterSeconds()); + } else if (!allowNonDeleteTiers) { + return TConclusionStatus::Fail("Only DELETE via TTL is allowed for row-oriented tables"); + } + } + return TConclusionStatus::Fail("TTL settings does not contain DELETE action"); + } else { + // legacy format + return TDuration::Seconds(settings.GetExpireAfterSeconds()); + } +} + }} diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 2f440b74c04a..760776022fb2 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -1139,6 +1139,9 @@ TCheckFunc HasTtlEnabled(const TString& columnName, const TDuration& expireAfter UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), columnName); UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnUnit(), columnUnit); UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), expireAfter.Seconds()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 1); + UNIT_ASSERT(ttl.GetEnabled().GetTiers(0).HasDelete()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetTiers(0).GetApplyAfterSeconds(), expireAfter.Seconds()); }; } diff --git a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp index 206c378f4f55..7c221db5c879 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp +++ b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp @@ -10,10 +10,21 @@ using namespace NSchemeShardUT_Private; namespace { template -void CheckTtlSettings(const TTtlSettings& ttl, const char* ttlColumnName) { +void CheckTtlSettings(const TTtlSettings& ttl, const char* ttlColumnName, bool legacyTiering = false) { UNIT_ASSERT(ttl.HasEnabled()); UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), ttlColumnName); UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 3600); + if (legacyTiering) { + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 0); + } else { + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 1); + UNIT_ASSERT(ttl.GetEnabled().GetTiers(0).HasDelete()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetTiers(0).GetApplyAfterSeconds(), 3600); + } +} + +void LegacyOltpTtlChecker(const NKikimrScheme::TEvDescribeSchemeResult& record) { + CheckTtlSettings(record.GetPathDescription().GetTable().GetTTLSettings(), "modified_at", true); } void OltpTtlChecker(const NKikimrScheme::TEvDescribeSchemeResult& record) { @@ -54,6 +65,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { ColumnName: "modified_at" ExpireAfterSeconds: 3600 ColumnUnit: %s + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )", name, ttlColumnType, unit)); @@ -94,7 +109,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { ColumnName: "created_at" } } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "Cannot enable TTL on unknown column: 'created_at'"}}); } Y_UNIT_TEST(CreateTableShouldFailOnWrongColumnType) { @@ -112,7 +127,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { ColumnName: "modified_at" } } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "Unsupported column type"}}); } void CreateTableShouldFailOnWrongUnit(const char* ttlColumnType, bool enableTablePgTypes, const char* unit) { @@ -120,7 +135,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { TTestEnv env(runtime, TTestEnvOptions().EnableTablePgTypes(enableTablePgTypes)); ui64 txId = 100; - TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + TestCreateTable(runtime, ++txId, "/MyRoot", + Sprintf(R"( Name: "TTLEnabledTable" Columns { Name: "key" Type: "Uint64" } Columns { Name: "modified_at" Type: "%s" } @@ -131,7 +147,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { ColumnUnit: %s } } - )", ttlColumnType, unit), {NKikimrScheme::StatusSchemeError}); + )", + ttlColumnType, unit), { + { NKikimrScheme::StatusSchemeError, "To enable TTL on date PG type column 'DateTypeColumnModeSettings' should be specified" }, + { NKikimrScheme::StatusSchemeError, "To enable TTL on date type column 'DateTypeColumnModeSettings' should be specified" }, + { NKikimrScheme::StatusSchemeError, "To enable TTL on integral type column 'ValueSinceUnixEpochModeSettings' should be specified" }, + { NKikimrScheme::StatusSchemeError, "To enable TTL on integral PG type column 'ValueSinceUnixEpochModeSettings' should be specified" } + }); } Y_UNIT_TEST_FLAG(CreateTableShouldFailOnWrongUnit, EnableTablePgTypes) { @@ -164,7 +186,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { KeyColumnNames: ["key"] TTLSettings { } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "TTL status must be specified"}}); } Y_UNIT_TEST(CreateTableShouldFailOnBeforeEpochTTL) { @@ -185,9 +207,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3153600000 + Tiers: { + ApplyAfterSeconds: 3153600000 + Delete: {} + } } } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "TTL should be less than"}}); } void CreateTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType) { @@ -205,6 +231,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } } @@ -246,6 +276,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -255,7 +289,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { TestAlterTable(runtime, ++txId, "/MyRoot", R"( Name: "TTLEnabledTable" DropColumns { Name: "modified_at" } - )", {NKikimrScheme::StatusInvalidParameter}); + )", {{NKikimrScheme::StatusInvalidParameter, "Can't drop TTL column: 'modified_at', disable TTL first"}}); TestAlterTable(runtime, ++txId, "/MyRoot", R"( Name: "TTLEnabledTable" @@ -296,6 +330,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -324,7 +362,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { ColumnName: "modified_at" } } - )", {NKikimrScheme::StatusInvalidParameter}); + )", {{NKikimrScheme::StatusInvalidParameter, "Cannot enable TTL on dropped column: 'modified_at'"}}); } void AlterTableOnIndexedTable(NKikimrSchemeOp::EIndexType indexType) { @@ -353,6 +391,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -384,6 +426,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -524,6 +570,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "ts" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -785,6 +835,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -803,6 +857,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { SysSettings { RunInterval: 1800000000 } + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -821,9 +879,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { SysSettings { RunInterval: 1799999999 } + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "TTL run interval cannot be less than limit"}}); } Y_UNIT_TEST(ShouldSkipDroppedColumn) { @@ -867,6 +929,53 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { WaitForCondErase(runtime); } + Y_UNIT_TEST(LegacyTtlSettingsNoTiers) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )")); + env.TestWaitNotification(runtime, txId); + CheckTtlSettings(runtime, LegacyOltpTtlChecker); + } + + Y_UNIT_TEST(LegacyTtlSettingsNoTiersAlterTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + )")); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Enabled { + ColumnName: "modified_at" + ExpireAfterSeconds: 3600 + } + } + )"); + env.TestWaitNotification(runtime, txId); + CheckTtlSettings(runtime, LegacyOltpTtlChecker); + } + NKikimrTabletBase::TEvGetCountersResponse GetCounters(TTestBasicRuntime& runtime) { const auto sender = runtime.AllocateEdgeActor(); runtime.SendToPipe(TTestTxConfig::SchemeShard, sender, new TEvTablet::TEvGetCounters); @@ -1069,6 +1178,56 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) { WaitForStats(runtime, 2); CheckPercentileCounter(runtime, "SchemeShard/NumShardsByTtlLag", {{"900", 2}, {"1800", 0}, {"inf", 0}}); } + + Y_UNIT_TEST(TtlTiersValidation) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + Name: "TTLEnabledTable" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "modified_at" Type: "Timestamp" } + KeyColumnNames: ["key"] + )")); + env.TestWaitNotification(runtime, txId); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Enabled { + ColumnName: "modified_at" + Tiers { + Delete {} + ApplyAfterSeconds: 3600 + } + Tiers { + Delete {} + ApplyAfterSeconds: 7200 + } + } + } + )", {{NKikimrScheme::StatusInvalidParameter, "Only the last tier in TTL settings can have Delete action"}}); + + TestAlterTable(runtime, ++txId, "/MyRoot", R"( + Name: "TTLEnabledTable" + TTLSettings { + Enabled { + ColumnName: "modified_at" + Tiers { + EvictToExternalStorage { + StorageName: "/Root/abc" + } + ApplyAfterSeconds: 3600 + } + Tiers { + Delete {} + ApplyAfterSeconds: 7200 + } + } + } + )", {{NKikimrScheme::StatusInvalidParameter, "Only DELETE via TTL is allowed for row-oriented tables"}}); + } } Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { @@ -1089,6 +1248,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { ColumnName: "modified_at" ExpireAfterSeconds: 3600 ColumnUnit: %s + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )", name, ttlColumnType, unit)); @@ -1114,7 +1277,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { ui64 txId = 100; for (auto ct : {"String", "DyNumber"}) { - TestCreateColumnTable(runtime, ++txId, "/MyRoot", Sprintf(R"( + TestCreateColumnTable(runtime, ++txId, "/MyRoot", + Sprintf(R"( Name: "TTLEnabledTable" Schema { Columns { Name: "key" Type: "Uint64" NotNull: true } @@ -1125,9 +1289,17 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } - )", ct), {NKikimrScheme::StatusSchemeError}); + )", + ct), { + { NKikimrScheme::StatusSchemeError, "Type 'DyNumber' specified for column 'modified_at' is not supported" }, + { NKikimrScheme::StatusSchemeError, "Unsupported column type" + } }); } } @@ -1148,7 +1320,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { ColumnName: "created_at" } } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "Incorrect ttl column - not found in scheme"}}); } Y_UNIT_TEST(AlterColumnTable) { @@ -1183,6 +1355,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -1212,7 +1388,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { AlterSchema { AlterColumns {Name: "data" DefaultValue: "10"} } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "sparsed columns are disabled"}}); env.TestWaitNotification(runtime, txId); } @@ -1247,9 +1423,13 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { Enabled { ColumnName: "str" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } - )", {NKikimrScheme::StatusSchemeError}); + )", {{NKikimrScheme::StatusSchemeError, "Unsupported column type"}}); } } @@ -1266,6 +1446,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -1298,6 +1482,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -1324,6 +1512,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); @@ -1354,6 +1546,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) { Enabled { ColumnName: "modified_at" ExpireAfterSeconds: 3600 + Tiers: { + ApplyAfterSeconds: 3600 + Delete: {} + } } } )"); diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 3370615dd1c3..bab1a87e06ef 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -507,38 +507,6 @@ Ydb::Type* AddColumn(Ydb::Table::ColumnMeta return columnType; } -template -static void AddTtl(TYdbProto& out, const TTtl& inTTL) { - switch (inTTL.GetColumnUnit()) { - case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: { - auto& outTTL = *out.mutable_ttl_settings()->mutable_date_type_column(); - outTTL.set_column_name(inTTL.GetColumnName()); - outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); - break; - } - - case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: - case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: - case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: - case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: { - auto& outTTL = *out.mutable_ttl_settings()->mutable_value_since_unix_epoch(); - outTTL.set_column_name(inTTL.GetColumnName()); - outTTL.set_column_unit(static_cast(inTTL.GetColumnUnit())); - outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds()); - break; - } - - default: - break; - } - - if constexpr (std::is_same_v) { - if (inTTL.HasSysSettings() && inTTL.GetSysSettings().HasRunInterval()) { - out.mutable_ttl_settings()->set_run_interval_seconds(TDuration::FromValue(inTTL.GetSysSettings().GetRunInterval()).Seconds()); - } - } -} - template void FillColumnDescriptionImpl(TYdbProto& out, NKikimrMiniKQL::TType& splitKeyType, const NKikimrSchemeOp::TTableDescription& in) { @@ -570,7 +538,11 @@ void FillColumnDescriptionImpl(TYdbProto& out, if (in.HasTTLSettings()) { if (in.GetTTLSettings().HasEnabled()) { - AddTtl(out, in.GetTTLSettings().GetEnabled()); + Ydb::StatusIds::StatusCode code; + TString error; + if (!FillTtlSettings(*out.mutable_ttl_settings(), in.GetTTLSettings().GetEnabled(), code, error)) { + ythrow yexception() << "invalid TTL settings: " << error; + } } if (in.GetTTLSettings().HasUseTiering()) { @@ -610,7 +582,11 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc if (in.HasTtlSettings()) { if (in.GetTtlSettings().HasEnabled()) { - AddTtl(out, in.GetTtlSettings().GetEnabled()); + Ydb::StatusIds::StatusCode status; + TString error; + if (!FillTtlSettings(*out.mutable_ttl_settings(), in.GetTtlSettings().GetEnabled(), status, error)) { + ythrow yexception() << "invalid TTL settings: " << error; + } } if (in.GetTtlSettings().HasUseTiering()) { diff --git a/ydb/core/ydb_convert/table_settings.cpp b/ydb/core/ydb_convert/table_settings.cpp index c719b5d7e950..367eb7a8db89 100644 --- a/ydb/core/ydb_convert/table_settings.cpp +++ b/ydb/core/ydb_convert/table_settings.cpp @@ -460,4 +460,86 @@ bool FillIndexTablePartitioning( return true; } +template +bool FillTtlSettingsImpl(Ydb::Table::TtlSettings& out, const TTtl& in, Ydb::StatusIds::StatusCode& code, TString& error) { + std::optional fillLegacyExpireAfterSeconds; + if (!in.TiersSize()) { + // handle legacy input format for backwards-compatibility + fillLegacyExpireAfterSeconds = in.GetExpireAfterSeconds(); + } else if (in.TiersSize() == 1 && in.GetTiers(0).HasDelete()) { + // convert delete-only TTL to legacy mode for backwards-compatibility + fillLegacyExpireAfterSeconds = in.GetTiers(0).GetApplyAfterSeconds(); + } else { + for (const auto& inTier : in.GetTiers()) { + auto* outTier = out.add_tiers(); + outTier->set_apply_after_seconds(inTier.GetApplyAfterSeconds()); + switch (inTier.GetActionCase()) { + case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kDelete: + outTier->mutable_delete_(); + break; + case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kEvictToExternalStorage: + outTier->mutable_evict_to_external_storage()->set_storage_name(inTier.GetEvictToExternalStorage().GetStorageName()); + break; + case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::ACTION_NOT_SET: + code = Ydb::StatusIds::BAD_REQUEST; + error = "Undefined tier action"; + return false; + } + } + } + + switch (in.GetColumnUnit()) { + case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: { + if (fillLegacyExpireAfterSeconds) { + auto& outTTL = *out.mutable_date_type_column(); + outTTL.set_column_name(in.GetColumnName()); + outTTL.set_expire_after_seconds(*fillLegacyExpireAfterSeconds); + } else { + auto& outTTL = *out.mutable_date_type_column_v1(); + outTTL.set_column_name(in.GetColumnName()); + } + break; + } + + case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: + case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: + case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: + case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: { + const auto unit = static_cast(in.GetColumnUnit()); + if (fillLegacyExpireAfterSeconds) { + auto& outTTL = *out.mutable_value_since_unix_epoch(); + outTTL.set_column_name(in.GetColumnName()); + outTTL.set_column_unit(unit); + outTTL.set_expire_after_seconds(*fillLegacyExpireAfterSeconds); + } else { + auto& outTTL = *out.mutable_value_since_unix_epoch_v1(); + outTTL.set_column_name(in.GetColumnName()); + outTTL.set_column_unit(unit); + } + break; + } + + default: + code = Ydb::StatusIds::BAD_REQUEST; + error = "Undefined column unit"; + return false; + } + + if constexpr (std::is_same_v) { + if (in.HasSysSettings() && in.GetSysSettings().HasRunInterval()) { + out.set_run_interval_seconds(TDuration::FromValue(in.GetSysSettings().GetRunInterval()).Seconds()); + } + } + + return true; +} + +bool FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TTTLSettings::TEnabled& in, Ydb::StatusIds::StatusCode& code, TString& error) { + return FillTtlSettingsImpl(out, in, code, error); +} + +bool FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& in, Ydb::StatusIds::StatusCode& code, TString& error) { + return FillTtlSettingsImpl(out, in, code, error); +} + } // namespace NKikimr diff --git a/ydb/core/ydb_convert/table_settings.h b/ydb/core/ydb_convert/table_settings.h index 49c36c0f6820..e9ac4a50b368 100644 --- a/ydb/core/ydb_convert/table_settings.h +++ b/ydb/core/ydb_convert/table_settings.h @@ -1,6 +1,8 @@ #pragma once #include + +#include #include #include @@ -18,6 +20,11 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& out, const Ydb::Table::AlterTableRequest& in, Ydb::StatusIds::StatusCode& code, TString& error, bool changed); + +// out +bool FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TTTLSettings::TEnabled& in, Ydb::StatusIds::StatusCode& code, TString& error); +bool FillTtlSettings(Ydb::Table::TtlSettings& out, const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& in, Ydb::StatusIds::StatusCode& code, TString& error); +// in template bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in, Ydb::StatusIds::StatusCode& code, TString& error) @@ -28,38 +35,88 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in return false; }; - switch (in.mode_case()) { - case Ydb::Table::TtlSettings::kDateTypeColumn: - out.SetColumnName(in.date_type_column().column_name()); - out.SetExpireAfterSeconds(in.date_type_column().expire_after_seconds()); - break; + static const auto& fillColumnName = [](TTtlSettingsEnabled& out, const TModeSettings& in) { + out.SetColumnName(in.column_name()); + }; - case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: - out.SetColumnName(in.value_since_unix_epoch().column_name()); - out.SetExpireAfterSeconds(in.value_since_unix_epoch().expire_after_seconds()); + static const auto& fillDeleteTier = [](TTtlSettingsEnabled& out, const TModeSettings& in) { + auto* deleteTier = out.AddTiers(); + deleteTier->SetApplyAfterSeconds(in.expire_after_seconds()); + deleteTier->MutableDelete(); + }; + static const auto& fillColumnUnit = [&unsupported] (TTtlSettingsEnabled& out, const TModeSettings& in) -> bool { #define CASE_UNIT(type) \ case Ydb::Table::ValueSinceUnixEpochModeSettings::type: \ out.SetColumnUnit(NKikimrSchemeOp::TTTLSettings::type); \ break - switch (in.value_since_unix_epoch().column_unit()) { + switch (in.column_unit()) { CASE_UNIT(UNIT_SECONDS); CASE_UNIT(UNIT_MILLISECONDS); CASE_UNIT(UNIT_MICROSECONDS); CASE_UNIT(UNIT_NANOSECONDS); default: return unsupported(TStringBuilder() << "Unsupported unit: " - << static_cast(in.value_since_unix_epoch().column_unit())); + << static_cast(in.column_unit())); } + return true; #undef CASE_UNIT + }; + + for (const auto& inTier : in.tiers()) { + auto* outTier = out.AddTiers(); + outTier->SetApplyAfterSeconds(inTier.apply_after_seconds()); + switch (inTier.action_case()) { + case Ydb::Table::TtlTier::kDelete: + outTier->MutableDelete(); + break; + case Ydb::Table::TtlTier::kEvictToExternalStorage: + outTier->MutableEvictToExternalStorage()->SetStorageName(inTier.evict_to_external_storage().storage_name()); + break; + case Ydb::Table::TtlTier::ACTION_NOT_SET: + break; + } + } + + switch (in.mode_case()) { + case Ydb::Table::TtlSettings::kDateTypeColumn: + fillColumnName(out, in.date_type_column()); + fillDeleteTier(out, in.date_type_column()); + break; + + case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: + fillColumnName(out, in.value_since_unix_epoch()); + fillDeleteTier(out, in.value_since_unix_epoch()); + if (!fillColumnUnit(out, in.value_since_unix_epoch())) { + return false; + } + break; + + case Ydb::Table::TtlSettings::kDateTypeColumnV1: + fillColumnName(out, in.date_type_column_v1()); break; - default: + case Ydb::Table::TtlSettings::kValueSinceUnixEpochV1: + fillColumnName(out, in.value_since_unix_epoch_v1()); + if (!fillColumnUnit(out, in.value_since_unix_epoch_v1())) { + return false; + } + break; + + case Ydb::Table::TtlSettings::MODE_NOT_SET: return unsupported("Unsupported ttl settings"); } + std::optional expireAfterSeconds; + for (const auto& tier : out.GetTiers()) { + if (tier.HasDelete()) { + expireAfterSeconds = tier.GetApplyAfterSeconds(); + } + } + out.SetExpireAfterSeconds(expireAfterSeconds.value_or(std::numeric_limits::max())); + if constexpr (std::is_same_v) { if (in.run_interval_seconds()) { out.MutableSysSettings()->SetRunInterval(TDuration::Seconds(in.run_interval_seconds()).GetValue()); diff --git a/ydb/core/ydb_convert/ya.make b/ydb/core/ydb_convert/ya.make index d81a6b472c0b..3a2843f45bc8 100644 --- a/ydb/core/ydb_convert/ya.make +++ b/ydb/core/ydb_convert/ya.make @@ -20,6 +20,7 @@ PEERDIR( ydb/core/util yql/essentials/types/binary_json yql/essentials/types/dynumber + ydb/library/conclusion ydb/library/mkql_proto/protos yql/essentials/minikql/dom yql/essentials/public/udf diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index 0cd658c45392..1c0406e09b07 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -433,8 +433,46 @@ message ColumnMeta { } } +message EvictionToExternalStorageSettings { + // Path to external data source + string storage_name = 1; +} + +message TtlTier { + uint32 apply_after_seconds = 1; + + oneof action { + google.protobuf.Empty delete = 2; + EvictionToExternalStorageSettings evict_to_external_storage = 3; + } +} + +message DateTypeColumnModeSettingsV1 { + // The row will be assigned a tier at the moment of time, when the value + // stored in is less than or equal to the current time (in epoch + // time format), and has passed since that moment; + // i.e. the eviction threshold is the value of plus . + + // The column type must be a date type + string column_name = 1; +} + +message ValueSinceUnixEpochModeSettingsV1 { + // Same as DateTypeColumnModeSettings (above), but useful when type of the + // value stored in is not a date type. + + // The column type must be one of: + // - Uint32 + // - Uint64 + // - DyNumber + string column_name = 1; + + // Interpretation of the value stored in + ValueSinceUnixEpochModeSettings.Unit column_unit = 2; +} + message DateTypeColumnModeSettings { - // The row will be considered as expired at the moment of time, when the value + // The row will be assigned a tier at the moment of time, when the value // stored in is less than or equal to the current time (in epoch // time format), and has passed since that moment; // i.e. the expiration threshold is the value of plus . @@ -473,8 +511,10 @@ message ValueSinceUnixEpochModeSettings { message TtlSettings { oneof mode { - DateTypeColumnModeSettings date_type_column = 1; - ValueSinceUnixEpochModeSettings value_since_unix_epoch = 2; + DateTypeColumnModeSettings date_type_column = 1 [deprecated = true]; + ValueSinceUnixEpochModeSettings value_since_unix_epoch = 2 [deprecated = true]; + DateTypeColumnModeSettingsV1 date_type_column_v1 = 4; + ValueSinceUnixEpochModeSettingsV1 value_since_unix_epoch_v1 = 5; } // There is no guarantee that expired row will be deleted immediately upon @@ -490,6 +530,8 @@ message TtlSettings { // How often to run BRO on the same partition. // BRO will not be started more often, but may be started less often. uint32 run_interval_seconds = 3; + + repeated TtlTier tiers = 6; } message StorageSettings { diff --git a/ydb/public/lib/experimental/ydb_logstore.cpp b/ydb/public/lib/experimental/ydb_logstore.cpp index 5878ee9d8cf3..758e14986afd 100644 --- a/ydb/public/lib/experimental/ydb_logstore.cpp +++ b/ydb/public/lib/experimental/ydb_logstore.cpp @@ -16,23 +16,10 @@ namespace NYdb { namespace NLogStore { TMaybe TtlSettingsFromProto(const Ydb::Table::TtlSettings& proto) { - switch (proto.mode_case()) { - case Ydb::Table::TtlSettings::kDateTypeColumn: - return TTtlSettings( - proto.date_type_column(), - proto.run_interval_seconds() - ); - - case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: - return TTtlSettings( - proto.value_since_unix_epoch(), - proto.run_interval_seconds() - ); - - default: - break; + if (auto settings = TTtlSettings::DeserializeFromProto(proto)) { + return *settings; } - return {}; + return Nothing(); } static TCompression CompressionFromProto(const Ydb::LogStore::Compression& compression) { diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 1130d5697275..4f461f920607 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -25,6 +25,7 @@ #include +#include #include #include #include @@ -326,23 +327,8 @@ class TTableDescription::TImpl { } // ttl settings - switch (proto.ttl_settings().mode_case()) { - case Ydb::Table::TtlSettings::kDateTypeColumn: - TtlSettings_ = TTtlSettings( - proto.ttl_settings().date_type_column(), - proto.ttl_settings().run_interval_seconds() - ); - break; - - case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: - TtlSettings_ = TTtlSettings( - proto.ttl_settings().value_since_unix_epoch(), - proto.ttl_settings().run_interval_seconds() - ); - break; - - default: - break; + if (auto ttlSettings = TTtlSettings::DeserializeFromProto(proto.ttl_settings())) { + TtlSettings_ = std::move(*ttlSettings); } // tiering @@ -2941,14 +2927,58 @@ bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription& //////////////////////////////////////////////////////////////////////////////// -TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter) +TTtlTierSettings::TTtlTierSettings(TDuration evictionDelay, const TAction& action) + : EvictAfter_(evictionDelay) + , Action_(action) { +} + +TTtlTierSettings::TTtlTierSettings(const Ydb::Table::TtlTier& tier) + : EvictAfter_(TDuration::Seconds(tier.apply_after_seconds())) { + switch (tier.action_case()) { + case Ydb::Table::TtlTier::kDelete: + Action_ = TTtlDeleteAction(); + break; + case Ydb::Table::TtlTier::kEvictToExternalStorage: + Action_ = TTtlEvictToExternalStorageAction(tier.evict_to_external_storage().storage_name()); + break; + case Ydb::Table::TtlTier::ACTION_NOT_SET: + break; + } +} + +void TTtlTierSettings::SerializeTo(Ydb::Table::TtlTier& proto) const { + proto.set_apply_after_seconds(EvictAfter_.Seconds()); + + std::visit(TOverloaded{ + [&proto](const TTtlDeleteAction&) { proto.mutable_delete_(); }, + [&proto](const TTtlEvictToExternalStorageAction& action) { + proto.mutable_evict_to_external_storage()->set_storage_name(action.StorageName); + }, + [](const std::monostate) {}, + }, + Action_); +} + +TDuration TTtlTierSettings::GetEvictAfter() const { + return EvictAfter_; +} + +const TTtlTierSettings::TAction& TTtlTierSettings::GetAction() const { + return Action_; +} + +TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const TString& columnName, const TDuration& deprecatedExpireAfter) : ColumnName_(columnName) - , ExpireAfter_(expireAfter) + , DeprecatedExpireAfter_(deprecatedExpireAfter) {} void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const { proto.set_column_name(ColumnName_); - proto.set_expire_after_seconds(ExpireAfter_.Seconds()); + proto.set_expire_after_seconds(DeprecatedExpireAfter_.Seconds()); +} + +void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettingsV1& proto) const { + proto.set_column_name(ColumnName_); } const TString& TDateTypeColumnModeSettings::GetColumnName() const { @@ -2956,19 +2986,24 @@ const TString& TDateTypeColumnModeSettings::GetColumnName() const { } const TDuration& TDateTypeColumnModeSettings::GetExpireAfter() const { - return ExpireAfter_; + return DeprecatedExpireAfter_; } -TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) +TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& deprecatedExpireAfter) : ColumnName_(columnName) , ColumnUnit_(columnUnit) - , ExpireAfter_(expireAfter) + , DeprecatedExpireAfter_(deprecatedExpireAfter) {} void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const { proto.set_column_name(ColumnName_); proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_)); - proto.set_expire_after_seconds(ExpireAfter_.Seconds()); + proto.set_expire_after_seconds(DeprecatedExpireAfter_.Seconds()); +} + +void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettingsV1& proto) const { + proto.set_column_name(ColumnName_); + proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_)); } const TString& TValueSinceUnixEpochModeSettings::GetColumnName() const { @@ -2980,7 +3015,7 @@ TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::GetCol } const TDuration& TValueSinceUnixEpochModeSettings::GetExpireAfter() const { - return ExpireAfter_; + return DeprecatedExpireAfter_; } void TValueSinceUnixEpochModeSettings::Out(IOutputStream& out, EUnit unit) { @@ -3023,13 +3058,17 @@ TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::UnitFr return EUnit::Unknown; } +TTtlSettings::TTtlSettings(const TString& columnName, const TVector& tiers) + : Mode_(TDateTypeColumnModeSettings(columnName, GetExpireAfterFrom(tiers).value_or(TDuration::Max()))) + , Tiers_(tiers) +{} + TTtlSettings::TTtlSettings(const TString& columnName, const TDuration& expireAfter) - : Mode_(TDateTypeColumnModeSettings(columnName, expireAfter)) + : TTtlSettings(columnName, {TTtlTierSettings(expireAfter, TTtlDeleteAction())}) {} TTtlSettings::TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds) - : TTtlSettings(mode.column_name(), TDuration::Seconds(mode.expire_after_seconds())) -{ + : TTtlSettings(mode.column_name(), TDuration::Seconds(mode.expire_after_seconds())) { RunInterval_ = TDuration::Seconds(runIntervalSeconds); } @@ -3037,13 +3076,17 @@ const TDateTypeColumnModeSettings& TTtlSettings::GetDateTypeColumn() const { return std::get(Mode_); } +TTtlSettings::TTtlSettings(const TString& columnName, EUnit columnUnit, const TVector& tiers) + : Mode_(TValueSinceUnixEpochModeSettings(columnName, columnUnit, GetExpireAfterFrom(tiers).value_or(TDuration::Max()))) + , Tiers_(tiers) +{} + TTtlSettings::TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter) - : Mode_(TValueSinceUnixEpochModeSettings(columnName, columnUnit, expireAfter)) + : TTtlSettings(columnName, columnUnit, {TTtlTierSettings(expireAfter, TTtlDeleteAction())}) {} TTtlSettings::TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds) - : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()), TDuration::Seconds(mode.expire_after_seconds())) -{ + : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()), TDuration::Seconds(mode.expire_after_seconds())) { RunInterval_ = TDuration::Seconds(runIntervalSeconds); } @@ -3051,16 +3094,44 @@ const TValueSinceUnixEpochModeSettings& TTtlSettings::GetValueSinceUnixEpoch() c return std::get(Mode_); } +std::optional TTtlSettings::DeserializeFromProto(const Ydb::Table::TtlSettings& proto) { + TDuration legacyExpireAfter = TDuration::Max(); + for (const auto& tier : proto.tiers()) { + if (tier.has_delete_()) { + legacyExpireAfter = TDuration::Seconds(tier.apply_after_seconds()); + break; + } + } + + switch(proto.mode_case()) { + case Ydb::Table::TtlSettings::kDateTypeColumn: + return TTtlSettings(proto.date_type_column(), proto.run_interval_seconds()); + case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: + return TTtlSettings(proto.value_since_unix_epoch(), proto.run_interval_seconds()); + case Ydb::Table::TtlSettings::kDateTypeColumnV1: + return TTtlSettings(TDateTypeColumnModeSettings(proto.date_type_column_v1().column_name(), legacyExpireAfter), proto.run_interval_seconds()); + case Ydb::Table::TtlSettings::kValueSinceUnixEpochV1: + return TTtlSettings(TValueSinceUnixEpochModeSettings(proto.value_since_unix_epoch_v1().column_name(), TProtoAccessor::FromProto(proto.value_since_unix_epoch_v1().column_unit()), legacyExpireAfter), proto.run_interval_seconds()); + case Ydb::Table::TtlSettings::MODE_NOT_SET: + return std::nullopt; + break; + } +} + void TTtlSettings::SerializeTo(Ydb::Table::TtlSettings& proto) const { switch (GetMode()) { case EMode::DateTypeColumn: - GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column()); + GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column_v1()); break; case EMode::ValueSinceUnixEpoch: - GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch()); + GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch_v1()); break; } + for (const auto& tier : Tiers_) { + tier.SerializeTo(*proto.add_tiers()); + } + if (RunInterval_) { proto.set_run_interval_seconds(RunInterval_.Seconds()); } @@ -3079,6 +3150,25 @@ const TDuration& TTtlSettings::GetRunInterval() const { return RunInterval_; } +const TVector& TTtlSettings::GetTiers() const { + return Tiers_; +} + +std::optional TTtlSettings::GetExpireAfter() const { + return GetExpireAfterFrom(Tiers_); +} + +std::optional TTtlSettings::GetExpireAfterFrom(const TVector& tiers) { + for (const auto& tier : tiers) { + if (std::holds_alternative(tier.GetAction())) { + return tier.GetEvictAfter(); + } + } + return std::nullopt; +} + +TTtlSettings::TTtlSettings(TMode mode, ui32 runIntervalSeconds) : Mode_(std::move(mode)), RunInterval_(TDuration::Seconds(runIntervalSeconds)) {} + TAlterTtlSettings::EAction TAlterTtlSettings::GetAction() const { return static_cast(Action_.index()); } diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index d69a9493d38c..b58b99a70f4c 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -30,9 +30,12 @@ class KMeansTreeSettings; class PartitioningSettings; class DateTypeColumnModeSettings; class TtlSettings; +class TtlTier; class TableIndex; class TableIndexDescription; class ValueSinceUnixEpochModeSettings; +class DateTypeColumnModeSettingsV1; +class ValueSinceUnixEpochModeSettingsV1; } // namespace Table } // namespace Ydb @@ -421,17 +424,46 @@ struct TPartitionStats { ui32 LeaderNodeId = 0; }; +struct TTtlDeleteAction {}; + +struct TTtlEvictToExternalStorageAction { + TString StorageName; +}; + +class TTtlTierSettings { +public: + using TAction = std::variant< + std::monostate, + TTtlDeleteAction, + TTtlEvictToExternalStorageAction + >; + +public: + explicit TTtlTierSettings(TDuration evictionDelay, const TAction& action); + explicit TTtlTierSettings(const Ydb::Table::TtlTier& tier); + void SerializeTo(Ydb::Table::TtlTier& proto) const; + + TDuration GetEvictAfter() const; + const TAction& GetAction() const; + +private: + TDuration EvictAfter_; + TAction Action_; +}; + class TDateTypeColumnModeSettings { public: - explicit TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter); + explicit TDateTypeColumnModeSettings(const TString& columnName, const TDuration& deprecatedExpireAfter = TDuration::Max()); void SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const; + void SerializeTo(Ydb::Table::DateTypeColumnModeSettingsV1& proto) const; const TString& GetColumnName() const; + // Deprecated. Use TTtlSettings::GetExpireAfter() const TDuration& GetExpireAfter() const; private: TString ColumnName_; - TDuration ExpireAfter_; + TDuration DeprecatedExpireAfter_; }; class TValueSinceUnixEpochModeSettings { @@ -446,11 +478,13 @@ class TValueSinceUnixEpochModeSettings { }; public: - explicit TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); + explicit TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& deprecatedExpireAfter = TDuration::Max()); void SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const; + void SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettingsV1& proto) const; const TString& GetColumnName() const; EUnit GetColumnUnit() const; + // Deprecated. Use TTtlSettings::GetExpireAfter() const TDuration& GetExpireAfter() const; static void Out(IOutputStream& o, EUnit unit); @@ -460,11 +494,17 @@ class TValueSinceUnixEpochModeSettings { private: TString ColumnName_; EUnit ColumnUnit_; - TDuration ExpireAfter_; + TDuration DeprecatedExpireAfter_; }; //! Represents ttl settings class TTtlSettings { +private: + using TMode = std::variant< + TDateTypeColumnModeSettings, + TValueSinceUnixEpochModeSettings + >; + public: using EUnit = TValueSinceUnixEpochModeSettings::EUnit; @@ -473,25 +513,35 @@ class TTtlSettings { ValueSinceUnixEpoch = 1, }; + explicit TTtlSettings(const TString& columnName, const TVector& tiers); explicit TTtlSettings(const TString& columnName, const TDuration& expireAfter); - explicit TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds); const TDateTypeColumnModeSettings& GetDateTypeColumn() const; + // Deprecated. Use DeserializeFromProto() + explicit TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds); + explicit TTtlSettings(const TString& columnName, EUnit columnUnit, const TVector& tiers); explicit TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); - explicit TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds); const TValueSinceUnixEpochModeSettings& GetValueSinceUnixEpoch() const; + // Deprecated. Use DeserializeFromProto() + explicit TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds); + static std::optional DeserializeFromProto(const Ydb::Table::TtlSettings& proto); void SerializeTo(Ydb::Table::TtlSettings& proto) const; EMode GetMode() const; TTtlSettings& SetRunInterval(const TDuration& value); const TDuration& GetRunInterval() const; + const TVector& GetTiers() const; + std::optional GetExpireAfter() const; + private: - std::variant< - TDateTypeColumnModeSettings, - TValueSinceUnixEpochModeSettings - > Mode_; + explicit TTtlSettings(TMode mode, ui32 runIntervalSeconds); + static std::optional GetExpireAfterFrom(const TVector& tiers); + +private: + TMode Mode_; + TVector Tiers_; TDuration RunInterval_ = TDuration::Zero(); }; diff --git a/ydb/services/ext_index/metadata/object.cpp b/ydb/services/ext_index/metadata/object.cpp index 609e0357cc19..5e7ac93ac6f2 100644 --- a/ydb/services/ext_index/metadata/object.cpp +++ b/ydb/services/ext_index/metadata/object.cpp @@ -1,5 +1,8 @@ -#include "object.h" #include "behaviour.h" +#include "object.h" + +#include + #include #include @@ -73,9 +76,14 @@ bool TObject::TryProvideTtl(const NKikimrSchemeOp::TColumnTableDescription& csDe return false; } if (cRequest) { - auto& newTtl = *cRequest->mutable_ttl_settings()->mutable_date_type_column(); - newTtl.set_column_name("pk_" + ttl.GetColumnName()); - newTtl.set_expire_after_seconds(ttl.GetExpireAfterSeconds()); + auto newTtl = ttl; + newTtl.SetColumnName("pk_" + ttl.GetColumnName()); + + Ydb::StatusIds::StatusCode status; + TString error; + if (!FillTtlSettings(*cRequest->mutable_ttl_settings(), newTtl, status, error)) { + return false; + } } } return true; diff --git a/ydb/services/ext_index/metadata/ya.make b/ydb/services/ext_index/metadata/ya.make index 3f3dbbc69192..9d2cf0e9b8fa 100644 --- a/ydb/services/ext_index/metadata/ya.make +++ b/ydb/services/ext_index/metadata/ya.make @@ -15,6 +15,7 @@ PEERDIR( ydb/core/grpc_services/local_rpc ydb/core/grpc_services/base ydb/core/grpc_services + ydb/core/ydb_convert ydb/services/metadata/request ydb/services/ext_index/metadata/extractor )