add tiering info to TTL in public api

ydb/core/grpc_services/rpc_log_store.cpp

@@ -508,29 +508,8 @@ class TDescribeLogTableRPC : public TRpcSchemeRequestActor<TDescribeLogTableRPC,
                 }
 
                 if (tableDescription.HasTtlSettings() && tableDescription.GetTtlSettings().HasEnabled()) {
-                    const auto& inTTL = tableDescription.GetTtlSettings().GetEnabled();
-
-                    switch (inTTL.GetColumnUnit()) {
-                    case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
-                        auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_date_type_column();
-                        outTTL.set_column_name(inTTL.GetColumnName());
-                        outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
-                        break;
-                    }
-
-                    case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS:
-                    case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS:
-                    case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS:
-                    case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: {
-                        auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_value_since_unix_epoch();
-                        outTTL.set_column_name(inTTL.GetColumnName());
-                        outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit()));
-                        outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
-                        break;
-                    }
-
-                    default:
-                        break;
+                    if (!FillTtlSettings(*describeLogTableResult.mutable_ttl_settings(), tableDescription.GetTtlSettings().GetEnabled(), status, error)) {
+                        return Reply(status, error, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx);
                     }
                 }
 

ydb/core/kqp/provider/yql_kikimr_gateway.cpp

@@ -311,15 +311,16 @@ bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::Read
 
 void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto) {
     if (!settings.ColumnUnit) {
-        auto& opts = *proto.mutable_date_type_column();
+        auto& opts = *proto.mutable_date_type_column_v1();
         opts.set_column_name(settings.ColumnName);
-        opts.set_expire_after_seconds(settings.ExpireAfter.Seconds());
     } else {
-        auto& opts = *proto.mutable_value_since_unix_epoch();
+        auto& opts = *proto.mutable_value_since_unix_epoch_v1();
         opts.set_column_name(settings.ColumnName);
         opts.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(*settings.ColumnUnit));
-        opts.set_expire_after_seconds(settings.ExpireAfter.Seconds());
     }
+    auto* deleteTier = proto.add_tiers();
+    deleteTier->set_evict_after_seconds(settings.ExpireAfter.Seconds());
+    deleteTier->mutable_delete_();
 }
 
 Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) {

ydb/core/kqp/proxy_service/kqp_script_executions_ut.cpp

@@ -35,6 +35,9 @@ NKikimrSchemeOp::TColumnDescription Col(const TString& columnName, NScheme::TTyp
 
 [[maybe_unused]] NKikimrSchemeOp::TTTLSettings TtlCol(const TString& columnName) {
     NKikimrSchemeOp::TTTLSettings settings;
+    auto* deleteTier = settings.MutableEnabled()->AddTiers();
+    deleteTier->MutableDelete();
+    deleteTier->SetEvictAfterSeconds(TDuration::Minutes(20).Seconds());
     settings.MutableEnabled()->SetExpireAfterSeconds(TDuration::Minutes(20).Seconds());
     settings.MutableEnabled()->SetColumnName(columnName);
     settings.MutableEnabled()->MutableSysSettings()->SetRunInterval(TDuration::Minutes(60).MicroSeconds());

ydb/core/persqueue/writer/metadata_initializers.cpp

@@ -53,10 +53,12 @@ void TSrcIdMetaInitializer::DoPrepare(NInitializer::IInitializerInput::TPtr cont
         }
         {
             auto* ttlSettings = request.mutable_ttl_settings();
-            auto* columnTtl = ttlSettings->mutable_value_since_unix_epoch();
+            auto* columnTtl = ttlSettings->mutable_value_since_unix_epoch_v1();
             columnTtl->set_column_name("AccessTime");
-            columnTtl->set_expire_after_seconds(1382400);
             columnTtl->set_column_unit(Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS);
+            auto* deleteTier = ttlSettings->add_tiers();
+            deleteTier->mutable_delete_();
+            deleteTier->set_evict_after_seconds(1382400);
         }
 
         result.emplace_back(new NInitializer::TGenericTableModifier<NRequest::TDialogCreateTable>(request, "create"));

ydb/core/protos/flat_scheme_op.proto

@@ -221,11 +221,24 @@ message TTTLSettings {
         optional uint32 MaxShardsInFlight = 6 [default = 0]; // zero means no limit
     }
 
+    message TEvictionToExternalStorageSettings {
+        optional string StorageName = 1;
+    }
+
+    message TTier {
+        optional uint32 EvictAfterSeconds = 1;
+        oneof Action {
+            google.protobuf.Empty Delete = 2;
+            TEvictionToExternalStorageSettings EvictToExternalStorage = 3;
+        }
+    }
+
     message TEnabled {
         optional string ColumnName = 1;
-        optional uint32 ExpireAfterSeconds = 2;
+        optional uint32 ExpireAfterSeconds = 2 [deprecated = true];
         optional EUnit ColumnUnit = 3;
         optional TSysSettings SysSettings = 4;
+        repeated TTier Tiers = 5;
     }
 
     message TDisabled {
@@ -571,10 +584,11 @@ message TColumnDataLifeCycle {
     message TTtl {
         optional string ColumnName = 1;
         oneof Expire {
-            uint32 ExpireAfterSeconds = 2;
+            uint32 ExpireAfterSeconds = 2 [deprecated = true]; // ignored if Tiers are not empty
             uint64 ExpireAfterBytes = 4;
         }
         optional TTTLSettings.EUnit ColumnUnit = 3;
+        repeated TTTLSettings.TTier Tiers = 5;
     }
 
     message TDisabled {

ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h

@@ -25,6 +25,9 @@ class TConverterModifyToAlter {
                 if (enabled.HasColumnUnit()) {
                     alterEnabled->SetColumnUnit(enabled.GetColumnUnit());
                 }
+                for (const auto& tier : enabled.GetTiers()) {
+                    alterEnabled->AddTiers()->CopyFrom(tier);
+                }
             } else if (tableTtl.HasDisabled()) {
                 alterTtl->MutableDisabled();
             }

ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp

@@ -141,7 +141,21 @@ struct TSchemeShard::TTxRunConditionalErase: public TSchemeShard::TRwTxBase {
         }
 
         const auto& settings = tableInfo->TTLSettings().GetEnabled();
-        const TDuration expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds());
+
+        TDuration expireAfter;
+         if (settings.TiersSize()) {
+            if (settings.TiersSize() > 1 || !settings.GetTiers(0).HasDelete()) {
+                LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Invalid ttl setting: external storage tiers"
+                    << ": shardIdx: " << tableShardInfo.ShardIdx
+                    << ": pathId: " << shardInfo.PathId
+                    << ", at schemeshard: " << Self->TabletID());
+                return false;
+            }
+            expireAfter = TDuration::Seconds(settings.GetTiers(0).GetEvictAfterSeconds());
+         } else {
+            // legacy format
+            expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds());
+         }
         const TInstant wallClock = ctx.Now() - expireAfter;
 
         NKikimrTxDataShard::TEvConditionalEraseRowsRequest request;

ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp

@@ -58,8 +58,20 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl,
             return false;
         }
 
+        ui32 expireInSeconds;
+        if (enabled.TiersSize()) {
+            if (enabled.TiersSize() > 1 || !enabled.GetTiers(0).HasDelete()) {
+                errStr = Sprintf("Only DELETE via TTL is allowed for row-oriented tables");
+                return false;            
+            }
+            expireInSeconds = enabled.GetTiers(0).GetEvictAfterSeconds();
+        } else {
+            // legacy format
+            expireInSeconds = enabled.GetExpireAfterSeconds();
+        }
+
         const TInstant now = TInstant::Now();
-        if (enabled.GetExpireAfterSeconds() > now.Seconds()) {
+        if (expireInSeconds > now.Seconds()) {
             errStr = Sprintf("TTL should be less than %" PRIu64 " seconds (%" PRIu64 " days, %" PRIu64 " years). The ttl behaviour is undefined before 1970.", now.Seconds(), now.Days(), now.Days() / 365);
             return false;            
         }

ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp

@@ -1139,6 +1139,9 @@ TCheckFunc HasTtlEnabled(const TString& columnName, const TDuration& expireAfter
         UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), columnName);
         UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnUnit(), columnUnit);
         UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), expireAfter.Seconds());
+        UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 1);
+        UNIT_ASSERT(ttl.GetEnabled().GetTiers(0).HasDelete());
+        UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetTiers(0).GetEvictAfterSeconds(), expireAfter.Seconds());
     };
 }
 

ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp

@@ -2661,9 +2661,12 @@ Y_UNIT_TEST_SUITE(TImportTests) {
     Y_UNIT_TEST(ShouldRestoreTtlSettingsInDateTypeColumnMode) {
         ShouldRestoreSettings(R"(
             ttl_settings {
-              date_type_column {
+              date_type_column_v1 {
                 column_name: "created_at"
-                expire_after_seconds: 3600
+              }
+              tiers {
+                evict_after_seconds: 3600
+                delete {}
               }
             }
         )", {
@@ -2674,10 +2677,13 @@ Y_UNIT_TEST_SUITE(TImportTests) {
     Y_UNIT_TEST(ShouldRestoreTtlSettingsInValueSinceUnixEpochMode) {
         ShouldRestoreSettings(R"(
             ttl_settings {
-              value_since_unix_epoch {
+              value_since_unix_epoch_v1 {
                 column_name: "modified_at"
                 column_unit: UNIT_SECONDS
-                expire_after_seconds: 7200
+              }
+              tiers {
+                evict_after_seconds: 7200
+                delete {}
               }
             }
         )", {