From 460d5b900e45176ccbe25c91323ca3cbbfc105af Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Sun, 10 Nov 2024 19:13:26 +0000 Subject: [PATCH 01/12] configure tiering on CS via ttl --- ydb/core/grpc_services/rpc_create_table.cpp | 1 - ydb/core/grpc_services/rpc_log_store.cpp | 8 -- ydb/core/kqp/host/kqp_gateway_proxy.cpp | 20 ++- ydb/core/kqp/provider/yql_kikimr_exec.cpp | 7 - ydb/core/kqp/provider/yql_kikimr_gateway.cpp | 32 ++--- ydb/core/kqp/provider/yql_kikimr_gateway.h | 8 +- ydb/core/kqp/provider/yql_kikimr_type_ann.cpp | 8 -- ydb/core/kqp/ut/common/columnshard.cpp | 21 +-- ydb/core/kqp/ut/common/columnshard.h | 3 +- ydb/core/kqp/ut/olap/tiering_ut.cpp | 114 +++++++++-------- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 78 ++++++------ ydb/core/protos/flat_scheme_op.proto | 4 +- ydb/core/tx/columnshard/columnshard.cpp | 8 +- ydb/core/tx/columnshard/columnshard__init.cpp | 1 - .../columnshard__propose_transaction.cpp | 2 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 52 +++++--- ydb/core/tx/columnshard/columnshard_impl.h | 3 +- ydb/core/tx/columnshard/columnshard_schema.h | 6 +- ydb/core/tx/columnshard/columnshard_ttl.h | 91 ------------- .../tx/columnshard/engines/column_engine.h | 5 +- .../engines/column_engine_logs.cpp | 44 ++----- .../columnshard/engines/column_engine_logs.h | 5 +- .../engines/scheme/tiering/tier_info.h | 71 +++++++++++ .../tx/columnshard/hooks/abstract/abstract.h | 4 +- ydb/core/tx/columnshard/loading/stages.cpp | 2 +- ydb/core/tx/columnshard/tables_manager.cpp | 25 ++-- ydb/core/tx/columnshard/tables_manager.h | 21 +-- .../test_helper/columnshard_ut_common.cpp | 10 +- .../test_helper/columnshard_ut_common.h | 25 ++-- ydb/core/tx/schemeshard/common/validation.cpp | 10 +- ydb/core/tx/schemeshard/common/validation.h | 4 +- .../tx/schemeshard/olap/manager/manager.cpp | 55 +++++--- .../tx/schemeshard/olap/manager/manager.h | 6 +- .../operations/alter/abstract/converter.h | 7 +- .../olap/operations/alter_table.cpp | 3 +- ydb/core/tx/schemeshard/olap/ttl/schema.cpp | 3 - .../tx/schemeshard/olap/ttl/validator.cpp | 6 +- .../tx/schemeshard/ut_helpers/ls_checks.cpp | 14 +- .../tx/schemeshard/ut_helpers/ls_checks.h | 2 +- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 29 ++++- ydb/core/tx/tiering/external_data.cpp | 7 +- ydb/core/tx/tiering/external_data.h | 2 +- ydb/core/tx/tiering/manager.cpp | 32 +---- ydb/core/tx/tiering/manager.h | 9 +- ydb/core/tx/tiering/rule/behaviour.cpp | 26 ---- ydb/core/tx/tiering/rule/behaviour.h | 20 --- ydb/core/tx/tiering/rule/checker.cpp | 97 -------------- ydb/core/tx/tiering/rule/checker.h | 44 ------- ydb/core/tx/tiering/rule/initializer.cpp | 41 ------ ydb/core/tx/tiering/rule/initializer.h | 15 --- ydb/core/tx/tiering/rule/manager.cpp | 43 ------- ydb/core/tx/tiering/rule/manager.h | 18 --- ydb/core/tx/tiering/rule/object.cpp | 104 --------------- ydb/core/tx/tiering/rule/object.h | 94 -------------- ydb/core/tx/tiering/rule/ya.make | 24 ---- ydb/core/tx/tiering/snapshot.cpp | 35 +---- ydb/core/tx/tiering/snapshot.h | 8 +- ydb/core/tx/tiering/tier/checker.cpp | 64 +--------- ydb/core/tx/tiering/tier/checker.h | 6 - .../tx/tiering/{rule => tier}/ss_checker.cpp | 0 .../tx/tiering/{rule => tier}/ss_checker.h | 0 .../tx/tiering/{rule => tier}/ss_fetcher.cpp | 0 .../tx/tiering/{rule => tier}/ss_fetcher.h | 0 ydb/core/tx/tiering/tier/ya.make | 2 + ydb/core/tx/tiering/ut/ut_tiers.cpp | 120 +++++++----------- ydb/core/tx/tiering/ya.make | 1 - ydb/core/ydb_convert/table_description.cpp | 16 --- ydb/core/ydb_convert/table_settings.cpp | 10 -- .../api/protos/draft/ydb_logstore.proto | 10 +- ydb/public/api/protos/ydb_table.proto | 9 +- ydb/public/lib/experimental/ydb_logstore.cpp | 2 - ydb/public/lib/experimental/ydb_logstore.h | 20 --- ydb/public/sdk/cpp/client/ydb_table/table.cpp | 20 +-- ydb/public/sdk/cpp/client/ydb_table/table.h | 1 + 74 files changed, 469 insertions(+), 1249 deletions(-) delete mode 100644 ydb/core/tx/columnshard/columnshard_ttl.h delete mode 100644 ydb/core/tx/tiering/rule/behaviour.cpp delete mode 100644 ydb/core/tx/tiering/rule/behaviour.h delete mode 100644 ydb/core/tx/tiering/rule/checker.cpp delete mode 100644 ydb/core/tx/tiering/rule/checker.h delete mode 100644 ydb/core/tx/tiering/rule/initializer.cpp delete mode 100644 ydb/core/tx/tiering/rule/initializer.h delete mode 100644 ydb/core/tx/tiering/rule/manager.cpp delete mode 100644 ydb/core/tx/tiering/rule/manager.h delete mode 100644 ydb/core/tx/tiering/rule/object.cpp delete mode 100644 ydb/core/tx/tiering/rule/object.h delete mode 100644 ydb/core/tx/tiering/rule/ya.make rename ydb/core/tx/tiering/{rule => tier}/ss_checker.cpp (100%) rename ydb/core/tx/tiering/{rule => tier}/ss_checker.h (100%) rename ydb/core/tx/tiering/{rule => tier}/ss_fetcher.cpp (100%) rename ydb/core/tx/tiering/{rule => tier}/ss_fetcher.h (100%) diff --git a/ydb/core/grpc_services/rpc_create_table.cpp b/ydb/core/grpc_services/rpc_create_table.cpp index 971cca82ce05..acad425ae98b 100644 --- a/ydb/core/grpc_services/rpc_create_table.cpp +++ b/ydb/core/grpc_services/rpc_create_table.cpp @@ -136,7 +136,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActorMutableTtlSettings()->SetUseTiering(req.tiering()); return true; } diff --git a/ydb/core/grpc_services/rpc_log_store.cpp b/ydb/core/grpc_services/rpc_log_store.cpp index 08cd8667c99c..2e8e26dc5c90 100644 --- a/ydb/core/grpc_services/rpc_log_store.cpp +++ b/ydb/core/grpc_services/rpc_log_store.cpp @@ -435,8 +435,6 @@ class TCreateLogTableRPC : public TRpcSchemeRequestActorMutableTtlSettings()->MutableEnabled(), req->ttl_settings(), status, error)) { return Reply(status, error, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx); } - } else if (req->has_tiering_settings()) { - create->MutableTtlSettings()->SetUseTiering(req->tiering_settings().tiering_id()); } create->SetColumnShardCount(req->shards_count()); @@ -600,12 +598,6 @@ class TAlterLogTableRPC : public TRpcSchemeRequestActorMutableAlterTtlSettings()->MutableDisabled(); } - if (req->has_set_tiering_settings()) { - alter->MutableAlterTtlSettings()->SetUseTiering(req->set_tiering_settings().tiering_id()); - } else if (req->has_drop_tiering_settings()) { - alter->MutableAlterTtlSettings()->SetUseTiering(""); - } - ctx.Send(MakeTxProxyID(), proposeRequest.release()); } }; diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 1f47b88897dc..e3a4a61366cd 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -276,16 +276,6 @@ bool ConvertCreateTableSettingsToProto(NYql::TKikimrTableMetadataPtr metadata, Y } } - if (const auto& tiering = metadata->TableSettings.Tiering) { - if (tiering.IsSet()) { - proto.set_tiering(tiering.GetValueSet()); - } else { - code = Ydb::StatusIds::BAD_REQUEST; - error = "Can't reset TIERING"; - return false; - } - } - if (metadata->TableSettings.StoreExternalBlobs) { auto& storageSettings = *proto.mutable_storage_settings(); TString value = to_lower(metadata->TableSettings.StoreExternalBlobs.GetRef()); @@ -520,7 +510,15 @@ bool FillCreateColumnTableDesc(NYql::TKikimrTableMetadataPtr metadata, const auto& inputSettings = metadata->TableSettings.TtlSettings.GetValueSet(); auto& resultSettings = *tableDesc.MutableTtlSettings(); resultSettings.MutableEnabled()->SetColumnName(inputSettings.ColumnName); - resultSettings.MutableEnabled()->SetExpireAfterSeconds(inputSettings.ExpireAfter.Seconds()); + for (const auto& tier : inputSettings.Tiers) { + auto* tierProto = resultSettings.MutableEnabled()->AddTiers(); + tierProto->SetApplyAfterSeconds(tier.ApplyAfter.Seconds()); + if (tier.StorageName) { + tierProto->MutableEvictToExternalStorage()->SetStorageName(*tier.StorageName); + } else { + tierProto->MutableDelete(); + } + } if (inputSettings.ColumnUnit) { resultSettings.MutableEnabled()->SetColumnUnit(static_cast(*inputSettings.ColumnUnit)); } diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index b2b205be6e3a..5a11ab401434 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -1657,13 +1657,6 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer().Literal().Cast().Value() - ); - alterTableRequest.set_set_tiering(tieringName); - } else if (name == "resetTiering") { - alterTableRequest.mutable_drop_tiering(); } else { ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), TStringBuilder() << "Unknown table profile setting: " << name)); diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index 233058ee4a01..717b393478a3 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -101,16 +101,6 @@ bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSetti if (name == "columnName") { YQL_ENSURE(field.Value().Maybe()); settings.ColumnName = field.Value().Cast().StringValue(); - } else if (name == "expireAfter") { - // TODO (yentsovsemyon): remove this clause after extending TTL syntax in YQL - YQL_ENSURE(field.Value().Maybe()); - auto value = FromString(field.Value().Cast().Literal().Value()); - if (value < 0) { - error = "Interval value cannot be negative"; - return false; - } - - settings.ExpireAfter = TDuration::FromValue(value); } else if (name == "tiers") { YQL_ENSURE(field.Value().Maybe()); auto listNode = field.Value().Cast(); @@ -118,12 +108,14 @@ bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSetti for (size_t i = 0; i < listNode.Size(); ++i) { auto tierNode = listNode.Item(i); + std::optional storageName; + TDuration evictionDelay; YQL_ENSURE(tierNode.Maybe()); for (const auto& tierField : tierNode.Cast()) { auto tierFieldName = tierField.Name().Value(); if (tierFieldName == "storageName") { - error = "TTL cannot contain tiered storage: tiering in TTL syntax is not supported"; - return false; + YQL_ENSURE(tierField.Value().Maybe()); + storageName = tierField.Value().Cast().StringValue(); } else if (tierFieldName == "evictionDelay") { YQL_ENSURE(tierField.Value().Maybe()); auto value = FromString(tierField.Value().Cast().Literal().Value()); @@ -131,12 +123,14 @@ bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSetti error = "Interval value cannot be negative"; return false; } - settings.ExpireAfter = TDuration::FromValue(value); + evictionDelay = TDuration::FromValue(value); } else { error = TStringBuilder() << "Unknown field: " << tierFieldName; return false; } } + + settings.Tiers.emplace_back(evictionDelay, storageName); } } else if (name == "columnUnit") { YQL_ENSURE(field.Value().Maybe()); @@ -318,9 +312,15 @@ void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::T opts.set_column_name(settings.ColumnName); opts.set_column_unit(static_cast(*settings.ColumnUnit)); } - auto* deleteTier = proto.add_tiers(); - deleteTier->set_apply_after_seconds(settings.ExpireAfter.Seconds()); - deleteTier->mutable_delete_(); + for (const auto& tier : settings.Tiers) { + auto* tierProto = proto.add_tiers(); + tierProto->set_apply_after_seconds(tier.ApplyAfter.Seconds()); + if (tier.StorageName) { + tierProto->mutable_evict_to_external_storage()->set_storage_name(tier.StorageName); + } else { + tierProto->mutable_delete_(); + } + } } Ydb::FeatureFlag::Status GetFlagValue(const TMaybe& value) { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 86e304fb040c..b3c71974011d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -221,9 +221,14 @@ struct TTtlSettings { Nanoseconds = 4, }; + struct TTier { + TDuration ApplyAfter; + std::optional StorageName; + }; + TString ColumnName; - TDuration ExpireAfter; TMaybe ColumnUnit; + std::vector Tiers; static bool TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error); }; @@ -241,7 +246,6 @@ struct TTableSettings { TMaybe KeyBloomFilter; TMaybe ReadReplicasSettings; TResetableSetting TtlSettings; - TResetableSetting Tiering; TMaybe PartitionByHashFunction; TMaybe StoreExternalBlobs; diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 613149823956..d732bfa7064b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -1251,14 +1251,6 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), "Can't reset TTL settings")); return TStatus::Error; - } else if (name == "setTiering") { - meta->TableSettings.Tiering.Set(TString( - setting.Value().Cast().Literal().Cast().Value() - )); - } else if (name == "resetTiering") { - ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()), - "Can't reset TIERING")); - return TStatus::Error; } else if (name == "storeType") { TMaybe storeType = TString(setting.Value().Cast().Value()); if (storeType && to_lower(storeType.GetRef()) == "column") { diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index e5e442d7cfe7..28ffe5f3e7dd 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -68,29 +68,14 @@ namespace NKqp { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - TString TTestHelper::CreateTieringRule(const TString& tierName, const TString& columnName) { - const TString ruleName = tierName + "_" + columnName; - const TString configTieringStr = TStringBuilder() << R"({ - "rules" : [ - { - "tierName" : ")" << tierName << R"(", - "durationForEvict" : "10d" - } - ] - })"; - auto result = GetSession().ExecuteSchemeQuery("CREATE OBJECT IF NOT EXISTS " + ruleName + " (TYPE TIERING_RULE) WITH (defaultColumn = " + columnName + ", description = `" + configTieringStr + "`)").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - return ruleName; - } - - void TTestHelper::SetTiering(const TString& tableName, const TString& ruleName) { - auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` SET (TIERING = '" << ruleName << "')"; + void TTestHelper::SetTiering(const TString& tableName, const TString& tierName, const TString& columnName) { + auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` SET TTL Interval(\"P10D\") TO EXTERNAL DATA SOURCE `" << tierName << "` ON `" << columnName << "`;"; auto result = GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } void TTestHelper::ResetTiering(const TString& tableName) { - auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` RESET (TIERING)"; + auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` RESET (TTL)"; auto result = GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } diff --git a/ydb/core/kqp/ut/common/columnshard.h b/ydb/core/kqp/ut/common/columnshard.h index e7d00ee6a1f7..ba85f6686162 100644 --- a/ydb/core/kqp/ut/common/columnshard.h +++ b/ydb/core/kqp/ut/common/columnshard.h @@ -109,9 +109,10 @@ class TTestHelper { NYdb::NTable::TSession& GetSession(); void CreateTable(const TColumnTableBase& table, const NYdb::EStatus expectedStatus = NYdb::EStatus::SUCCESS); void DropTable(const TString& tableName); + void EnsureSecret(const TString& name, const TString& value); void CreateTier(const TString& tierName); TString CreateTieringRule(const TString& tierName, const TString& columnName); - void SetTiering(const TString& tableName, const TString& ruleName); + void SetTiering(const TString& tableName, const TString& tierName, const TString& columnName); void ResetTiering(const TString& tableName); void BulkUpsert( const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS); diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index 410adeddb67f..f14fa26a325b 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -12,26 +12,39 @@ namespace NKikimr::NKqp { -Y_UNIT_TEST_SUITE(KqpOlapTiering) { - Y_UNIT_TEST(Eviction) { +class TTestEvictionBase { +protected: + std::optional TestHelper; + TString TieringRule; + +protected: + virtual void UnevictAll() = 0; + +public: + void RunTest() { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetSkipSpecialCheckForEvict(true); TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; - TTestHelper testHelper(runnerSettings); - TLocalHelper localHelper(testHelper.GetKikimr()); - NYdb::NTable::TTableClient tableClient = testHelper.GetKikimr().GetTableClient(); - Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize(); + TestHelper.emplace(runnerSettings); + TLocalHelper localHelper(TestHelper->GetKikimr()); + // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG); + // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); + TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_TIERING, NActors::NLog::PRI_DEBUG); + // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::KQP_GATEWAY, NActors::NLog::PRI_DEBUG); + // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NActors::NLog::PRI_DEBUG); + // TestHelper->GetRuntime().SetLogPriority(NKikimrServices::TX_PROXY, NActors::NLog::PRI_DEBUG); + NYdb::NTable::TTableClient tableClient = TestHelper->GetKikimr().GetTableClient(); + Tests::NCommon::TLoggerInit(TestHelper->GetKikimr()).Initialize(); Singleton()->SetSecretKey("fakeSecret"); localHelper.CreateTestOlapTable(); - testHelper.CreateTier("tier1"); - const TString tieringRule = testHelper.CreateTieringRule("tier1", "timestamp"); + TestHelper->CreateTier("tier1"); for (ui64 i = 0; i < 100; ++i) { - WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000); - WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000); + WriteTestData(TestHelper->GetKikimr(), "/Root/olapStore/olapTable", 0, 3600000000 + i * 10000, 1000); + WriteTestData(TestHelper->GetKikimr(), "/Root/olapStore/olapTable", 0, 3600000000 + i * 10000, 1000); } csController->WaitCompactions(TDuration::Seconds(5)); @@ -55,7 +68,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { UNIT_ASSERT_GT(columnRawBytes, 0); } - testHelper.SetTiering("/Root/olapStore/olapTable", tieringRule); + TestHelper->SetTiering("/Root/olapStore/olapTable", "tier1", "timestamp"); csController->WaitActualization(TDuration::Seconds(5)); { @@ -72,10 +85,10 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes - << " after=" << GetUint64(rows[0].at("RawBytes"))); + << " after=" << GetUint64(rows[0].at("RawBytes"))); } - testHelper.ResetTiering("/Root/olapStore/olapTable"); + UnevictAll(); csController->WaitCompactions(TDuration::Seconds(5)); { @@ -92,72 +105,65 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, TStringBuilder() << "RawBytes changed after resetting tiering: before=" << columnRawBytes - << " after=" << GetUint64(rows[0].at("RawBytes"))); + << " after=" << GetUint64(rows[0].at("RawBytes"))); } + + } +}; + +class TTestEvictionResetTiering : public TTestEvictionBase { + private: + void UnevictAll() { + TestHelper->ResetTiering("/Root/olapStore/olapTable"); } +}; + +class TTestEvictionIncreaseDuration : public TTestEvictionBase { + private: + void UnevictAll() { + const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P30000D") TO EXTERNAL DATA SOURCE tier1 ON timestamp)"; + auto result = TestHelper->GetSession().ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } +}; + +Y_UNIT_TEST_SUITE(KqpOlapTiering) { - Y_UNIT_TEST(TieringRuleValidation) { + Y_UNIT_TEST(EvictionResetTiering) { + TTestEvictionResetTiering().RunTest(); + } + + Y_UNIT_TEST(EvictionIncreaseDuration) { + TTestEvictionIncreaseDuration().RunTest(); + } + + Y_UNIT_TEST(TieringValidation) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; TTestHelper testHelper(runnerSettings); TLocalHelper localHelper(testHelper.GetKikimr()); + testHelper.GetRuntime().SetLogPriority(NKikimrServices::TX_TIERING, NActors::NLog::PRI_DEBUG); NYdb::NTable::TTableClient tableClient = testHelper.GetKikimr().GetTableClient(); Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize(); - Singleton()->SetSecretKey("fakeSecret"); localHelper.CreateTestOlapTable(); testHelper.CreateTier("tier1"); { - const TString query = R"( - CREATE OBJECT IF NOT EXISTS empty_tiering_rule (TYPE TIERING_RULE) - WITH (defaultColumn = timestamp, description = `{"rules": []}`))"; + const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1 ON unknown_column;)"; auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); } { - const TString query = R"( - CREATE OBJECT IF NOT EXISTS empty_default_column (TYPE TIERING_RULE) - WITH (defaultColumn = ``, description = `{"rules": [{ "tierName" : "tier1", "durationForEvict" : "10d" }]}`))"; + const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1 ON uid;)"; auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); } - { - const TString query = R"( - CREATE OBJECT IF NOT EXISTS no_default_column (TYPE TIERING_RULE) - WITH (description = `{"rules": [{ "tierName" : "tier1", "durationForEvict" : "10d" }]}`))"; - auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - } - - const TString correctTieringRule = testHelper.CreateTieringRule("tier1", "timestamp"); - { - const TString query = "ALTER OBJECT " + correctTieringRule + R"( (TYPE TIERING_RULE) SET description `{"rules": []}`)"; - auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - } - - { - const TString query = "ALTER OBJECT " + correctTieringRule + R"( (TYPE TIERING_RULE) SET description `{"rules": []}`)"; - auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - } - - { - const TString query = "ALTER OBJECT " + correctTieringRule + R"( (TYPE TIERING_RULE) SET defaultColumn ``)"; - auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - } - - { - const TString query = "ALTER OBJECT " + correctTieringRule + R"( (TYPE TIERING_RULE) RESET defaultColumn)"; - auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync(); - UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - } + testHelper.SetTiering("/Root/olapStore/olapTable", "tier1", "timestamp"); } } diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 73e88c24e594..806adb2cd59b 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5421,15 +5421,18 @@ Y_UNIT_TEST_SUITE(KqpScheme) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; runnerSettings.SetEnableTieringInColumnShard(true); - TKikimrRunner kikimr(runnerSettings); - auto db = kikimr.GetTableClient(); + TTestHelper testHelper(runnerSettings); + auto db = testHelper.GetKikimr().GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); TString tableName = "/Root/ColumnTableTest"; + testHelper.CreateTier("tier1"); + testHelper.CreateTier("tier2"); + auto query = TStringBuilder() << R"( --!syntax_v1 CREATE TABLE `)" << tableName << R"(` ( - Key Uint64 NOT NULL, + Key Timestamp NOT NULL, Value1 String, Value2 Int64 NOT NULL, PRIMARY KEY (Key) @@ -5438,23 +5441,23 @@ Y_UNIT_TEST_SUITE(KqpScheme) { WITH ( STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10, - TIERING = 'tiering1' + TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE tier1 ON Key );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); -#if 0 // TODO { // describe table auto desc = session.DescribeTable(tableName).ExtractValueSync(); UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); - auto tiering = desc.GetTableDescription().GetTiering(); - UNIT_ASSERT(tiering); - UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering1"); + UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); + auto ttl = desc.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier1"); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetEvictionDelay(), TDuration::Seconds(10)); } -#endif auto query2 = TStringBuilder() << R"( --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` SET(TIERING = 'tiering2');)"; + ALTER TABLE `)" << tableName << R"(` SET (TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE tier2 ON Key);)"; result = session.ExecuteSchemeQuery(query2).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -5462,14 +5465,16 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto desc = session.DescribeTable(tableName).ExtractValueSync(); UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); - auto tiering = desc.GetTableDescription().GetTiering(); - UNIT_ASSERT(tiering); - UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering2"); + UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); + auto ttl = desc.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier2"); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetEvictionDelay(), TDuration::Seconds(10)); } auto query3 = TStringBuilder() << R"( --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` RESET (TIERING);)"; + ALTER TABLE `)" << tableName << R"(` RESET (TTL);)"; result = session.ExecuteSchemeQuery(query3).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -5477,13 +5482,13 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto desc = session.DescribeTable(tableName).ExtractValueSync(); UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); - auto tiering = desc.GetTableDescription().GetTiering(); - UNIT_ASSERT(!tiering); + auto ttl = desc.GetTableDescription().GetTtlSettings(); + UNIT_ASSERT(!ttl); } auto query4 = TStringBuilder() << R"( --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` SET (TIERING = 'tiering1');)"; + ALTER TABLE `)" << tableName << R"(` SET (TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE tier1 ON Key);)"; result = session.ExecuteSchemeQuery(query4).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -5491,9 +5496,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto desc = session.DescribeTable(tableName).ExtractValueSync(); UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); - auto tiering = desc.GetTableDescription().GetTiering(); - UNIT_ASSERT(tiering); - UNIT_ASSERT_VALUES_EQUAL(*tiering, "tiering1"); + UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); + auto ttl = desc.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier1"); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetEvictionDelay(), TDuration::Seconds(10)); } auto query5 = TStringBuilder() << R"( @@ -8111,9 +8118,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.BulkUpsert(testTable, tableInserter); } - // const auto ruleName = testHelper.CreateTieringRule("tier1", "created_att"); - const auto ruleName = testHelper.CreateTieringRule("tier1", "created_at"); - testHelper.SetTiering(tableName, ruleName); + testHelper.SetTiering(tableName, "tier1", "created_at"); while (csController->GetTieringUpdates().Val() == 0) { Cout << "Wait tiering..." << Endl; @@ -8138,6 +8143,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TColumnTable testTable; testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id", "id_second"}).SetSharding({"id"}).SetSchema(schema); testHelper.CreateTable(testTable); + testHelper.CreateTier("tier1"); { auto alterQuery = TStringBuilder() << R"( @@ -8179,16 +8185,22 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { UNIT_ASSERT_VALUES_EQUAL(columns.size(), 5); UNIT_ASSERT_VALUES_EQUAL(description.GetTtlSettings()->GetDateTypeColumn().GetExpireAfter(), TDuration::Hours(1)); } - testHelper.SetTiering("/Root/ColumnTableTest", "tiering1"); + { + auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`SET (TTL = Interval(\"PT10S\") TO EXTERNAL DATA SOURCE tier1, Interval(\"PT1H\") DELETE ON created_at);"; + auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } { auto settings = TDescribeTableSettings().WithTableStatistics(true); auto describeResult = testHelper.GetSession().DescribeTable("/Root/ColumnTableTest", settings).GetValueSync(); UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto& description = describeResult.GetTableDescription(); - UNIT_ASSERT(description.GetTiering()); - UNIT_ASSERT_VALUES_EQUAL(*description.GetTiering(), "tiering1"); - UNIT_ASSERT_VALUES_EQUAL(description.GetTtlSettings()->GetDateTypeColumn().GetExpireAfter(), TDuration::Hours(1)); + UNIT_ASSERT(describeResult.GetTableDescription().GetTtlSettings()); + auto ttl = describeResult.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetExpireAfter(), TDuration::Hours(1)); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier1"); } { auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << R"(` RESET (TTL);)"; @@ -8201,18 +8213,6 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto& description = describeResult.GetTableDescription(); - UNIT_ASSERT(description.GetTiering()); - UNIT_ASSERT_VALUES_EQUAL(*description.GetTiering(), "tiering1"); - UNIT_ASSERT(!description.GetTtlSettings()); - } - testHelper.ResetTiering("/Root/ColumnTableTest"); - { - auto settings = TDescribeTableSettings().WithTableStatistics(true); - auto describeResult = testHelper.GetSession().DescribeTable("/Root/ColumnTableTest", settings).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); - - const auto& description = describeResult.GetTableDescription(); - UNIT_ASSERT(!description.GetTiering()); UNIT_ASSERT(!description.GetTtlSettings()); } } diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 50505ae2f27c..797a0de060ec 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -250,7 +250,7 @@ message TTTLSettings { TDisabled Disabled = 2; } - optional string UseTiering = 3; + reserved 3; } message TTableReplicationConfig { @@ -612,7 +612,7 @@ message TColumnDataLifeCycle { // Incremented on each settings change optional uint64 Version = 3 [default = 1]; - optional string UseTiering = 5; + reserved 5; } message TColumnTableTtlSettingsPreset { diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 125335c86bd0..0d5c5766c270 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -60,8 +60,12 @@ void TColumnShard::SwitchToWork(const TActorContext& ctx) { NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("self_id", SelfId())("process", "SwitchToWork"); AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "initialize_shard")("step", "SwitchToWork"); - for (auto&& i : TablesManager.GetTables()) { - ActivateTiering(i.first, i.second.GetTieringUsage()); + for (const auto& [pathId, tiering] : TablesManager.GetTtl()) { + THashSet tiers; + for (const auto& [name, config] : tiering.GetTierByName()) { + tiers.emplace(name); + } + ActivateTiering(pathId, tiers); } Become(&TThis::StateWork); diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index ace229ac6ff1..852c05444105 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -1,7 +1,6 @@ #include "columnshard_impl.h" #include "columnshard_private_events.h" #include "columnshard_schema.h" -#include "columnshard_ttl.h" #include "bg_tasks/adapter/adapter.h" #include "bg_tasks/manager/manager.h" diff --git a/ydb/core/tx/columnshard/columnshard__propose_transaction.cpp b/ydb/core/tx/columnshard/columnshard__propose_transaction.cpp index d4ded82be3d8..f3bc2aa80e58 100644 --- a/ydb/core/tx/columnshard/columnshard__propose_transaction.cpp +++ b/ydb/core/tx/columnshard/columnshard__propose_transaction.cpp @@ -164,7 +164,7 @@ class TTxProposeTransaction: public NTabletFlatExecutor::TTransactionBaseSetupTtl(pathTtls)) { return TTxController::TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "TTL not started"); } - Self->TablesManager.MutablePrimaryIndex().OnTieringModified(Self->Tiers, Self->TablesManager.GetTtl(), {}); + Self->TablesManager.MutablePrimaryIndex().OnTieringModified(Self->TablesManager.GetTtl()); return TTxController::TProposeResult(); } diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index f38dad89222e..3070a3bb41d1 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -406,20 +406,18 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl } { - bool needTieringActivation = false; + THashSet usedTiers; TTableInfo table(pathId); if (tableProto.HasTtlSettings()) { const auto& ttlSettings = tableProto.GetTtlSettings(); *tableVerProto.MutableTtlSettings() = ttlSettings; - if (ttlSettings.HasUseTiering()) { - table.SetTieringUsage(ttlSettings.GetUseTiering()); - needTieringActivation = true; + if (ttlSettings.HasEnabled()) { + usedTiers = NOlap::TTiering::GetUsedTiers(ttlSettings.GetEnabled()); } } - const TString tieringName = table.GetTieringUsage(); TablesManager.RegisterTable(std::move(table), db); - if (needTieringActivation) { - ActivateTiering(pathId, tieringName); + if (!usedTiers.empty()) { + ActivateTiering(pathId, usedTiers); } } @@ -430,7 +428,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, TablesManager.GetTables().size()); Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_PRESETS, TablesManager.GetSchemaPresets().size()); - Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_TTLS, TablesManager.GetTtl().PathsCount()); + Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_TTLS, TablesManager.GetTtl().size()); } void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterProto, const NOlap::TSnapshot& version, @@ -454,14 +452,16 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP schema = alterProto.GetSchema(); } - const auto& ttlSettings = alterProto.GetTtlSettings(); // Note: Not valid behaviour for full alter implementation - const TString& tieringUsage = ttlSettings.GetUseTiering(); + THashSet usedTiers; if (alterProto.HasTtlSettings()) { const auto& ttlSettings = alterProto.GetTtlSettings(); *tableVerProto.MutableTtlSettings() = ttlSettings; + + if (ttlSettings.HasEnabled()) { + usedTiers = NOlap::TTiering::GetUsedTiers(ttlSettings.GetEnabled()); + } } - ActivateTiering(pathId, tieringUsage); - Schema::SaveTableInfo(db, pathId, tieringUsage); + ActivateTiering(pathId, usedTiers); tableVerProto.SetSchemaPresetVersionAdj(alterProto.GetSchemaPresetVersionAdj()); TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db, Tiers); @@ -1552,13 +1552,13 @@ void TColumnShard::Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& Tiers->TakeConfigs(ev->Get()->GetSnapshot(), nullptr); } -void TColumnShard::ActivateTiering(const ui64 pathId, const TString& useTiering) { +void TColumnShard::ActivateTiering(const ui64 pathId, const THashSet& usedTiers) { AFL_VERIFY(Tiers); - if (useTiering) { - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "activate_tiering")("path_id", pathId)("tiering", useTiering); + if (!usedTiers.empty()) { + AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "activate_tiering")("path_id", pathId)("tiers", JoinStrings(usedTiers.begin(), usedTiers.end(), ",")); } - if (useTiering) { - Tiers->EnablePathId(pathId, useTiering); + if (!usedTiers.empty()) { + Tiers->EnablePathId(pathId, usedTiers); } else { Tiers->DisablePathId(pathId); } @@ -1580,10 +1580,20 @@ void TColumnShard::Enqueue(STFUNC_SIG) { void TColumnShard::OnTieringModified(const std::optional pathId) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnTieringModified")("path_id", pathId); - if (Tiers->IsReady()) { - StoragesManager->OnTieringModified(Tiers); - if (TablesManager.HasPrimaryIndex()) { - TablesManager.MutablePrimaryIndex().OnTieringModified(Tiers, TablesManager.GetTtl(), pathId); + if (!Tiers->IsReady()) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_reload_tiering")("reason", "manager_not_ready")("path_id", pathId); + return; + } + StoragesManager->OnTieringModified(Tiers); + if (TablesManager.HasPrimaryIndex()) { + if (pathId) { + std::optional tableTtl; + if (auto* findTtl = TablesManager.GetTtl().FindPtr(*pathId)) { + tableTtl = *findTtl; + } + TablesManager.MutablePrimaryIndex().OnTieringModified(tableTtl, *pathId); + } else { + TablesManager.MutablePrimaryIndex().OnTieringModified(TablesManager.GetTtl()); } } } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index ffa9a9c7cc23..844a3e20b80c 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -2,7 +2,6 @@ #include "background_controller.h" #include "columnshard.h" #include "columnshard_private_events.h" -#include "columnshard_ttl.h" #include "counters.h" #include "defs.h" #include "inflight_request_tracker.h" @@ -323,7 +322,7 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa putStatus.OnYellowChannels(Executor()); } - void ActivateTiering(const ui64 pathId, const TString& useTiering); + void ActivateTiering(const ui64 pathId, const THashSet& usedTiers); void OnTieringModified(const std::optional pathId = {}); public: diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index 0bea4bcfd18f..2a507e45c6fc 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -781,10 +781,8 @@ struct Schema : NIceDb::Schema { db.Table().Key(id).Delete(); } - static void SaveTableInfo(NIceDb::TNiceDb& db, const ui64 pathId, const TString tieringUsage) { - db.Table().Key(pathId).Update( - NIceDb::TUpdate(tieringUsage) - ); + static void SaveTableInfo(NIceDb::TNiceDb& db, const ui64 pathId) { + db.Table().Key(pathId).Update(); } diff --git a/ydb/core/tx/columnshard/columnshard_ttl.h b/ydb/core/tx/columnshard/columnshard_ttl.h deleted file mode 100644 index c8c99d638118..000000000000 --- a/ydb/core/tx/columnshard/columnshard_ttl.h +++ /dev/null @@ -1,91 +0,0 @@ -#pragma once -#include "defs.h" - -namespace NKikimr::NColumnShard { - -class TTtl { -public: - struct TEviction { - TDuration EvictAfter; - TString ColumnName; - ui32 UnitsInSecond = 0; // 0 means auto (data type specific) - }; - - struct TDescription { - std::optional Eviction; - - TDescription() = default; - - TDescription(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl) { - auto expireSec = TDuration::Seconds(ttl.GetExpireAfterSeconds()); - - Eviction = TEviction{expireSec, ttl.GetColumnName()}; - Y_ABORT_UNLESS(!Eviction->ColumnName.empty()); - - switch (ttl.GetColumnUnit()) { - case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: - Eviction->UnitsInSecond = 1; - break; - case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: - Eviction->UnitsInSecond = 1000; - break; - case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: - Eviction->UnitsInSecond = 1000 * 1000; - break; - case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: - Eviction->UnitsInSecond = 1000 * 1000 * 1000; - break; - case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: - default: - break; - } - } - }; - - ui64 PathsCount() const { - return PathTtls.size(); - } - - void SetPathTtl(ui64 pathId, TDescription&& descr) { - if (descr.Eviction) { - PathTtls[pathId] = descr; - } else { - PathTtls.erase(pathId); - } - } - - void DropPathTtl(ui64 pathId) { - PathTtls.erase(pathId); - } - - bool AddTtls(THashMap& eviction) const { - for (auto& [pathId, descr] : PathTtls) { - if (!eviction[pathId].Add(Convert(descr))) { - return false; - } - } - return true; - } - - THashSet TtlColumns() const { - THashSet columns; - for (const auto& [pathId, settings] : PathTtls) { - columns.insert(settings.Eviction->ColumnName); - } - return columns; - } - -private: - THashMap PathTtls; // pathId -> ttl - - std::shared_ptr Convert(const TDescription& descr) const - { - if (descr.Eviction) { - auto& evict = descr.Eviction; - return NOlap::TTierInfo::MakeTtl(evict->EvictAfter, evict->ColumnName, evict->UnitsInSecond); - } - return {}; - } -}; - -} diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 58581bab51e2..06db832429df 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -13,7 +13,6 @@ namespace NKikimr::NColumnShard { class TTiersManager; -class TTtl; } // namespace NKikimr::NColumnShard namespace NKikimr::NOlap { @@ -365,8 +364,8 @@ class IColumnEngine { virtual TSnapshot LastUpdate() const { return TSnapshot::Zero(); } - virtual void OnTieringModified( - const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; + virtual void OnTieringModified(const std::optional& ttl, const ui64 pathId) = 0; + virtual void OnTieringModified(const THashMap& ttl) = 0; }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 7a1511a3acf1..6ae4ba1cad19 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -153,12 +152,7 @@ void TColumnEngineForLogs::RegisterSchemaVersion(const TSnapshot& snapshot, TInd const bool isCriticalScheme = indexInfo.GetSchemeNeedActualization(); auto* indexInfoActual = VersionedIndex.AddIndex(snapshot, std::move(indexInfo)); if (isCriticalScheme) { - if (!ActualizationStarted) { - ActualizationStarted = true; - for (auto&& i : GranulesStorage->GetTables()) { - i.second->StartActualizationIndex(); - } - } + StartActualization({}); for (auto&& i : GranulesStorage->GetTables()) { i.second->RefreshScheme(); } @@ -548,35 +542,25 @@ bool TColumnEngineForLogs::StartActualization(const THashMap& sp ActualizationStarted = true; return true; } +void TColumnEngineForLogs::OnTieringModified(const std::optional& ttl, const ui64 pathId) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnTieringModified")("path_id", pathId); + StartActualization({}); + + auto g = GetGranulePtrVerified(pathId); + g->RefreshTiering(ttl); +} -void TColumnEngineForLogs::OnTieringModified( - const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) { +void TColumnEngineForLogs::OnTieringModified(const THashMap& ttl) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnTieringModified")("new_count_tierings", ttl.size()); StartActualization({}); - AFL_VERIFY(manager); - THashMap tierings = manager->GetTiering(); - ttl.AddTtls(tierings); - - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnTieringModified")("new_count_tierings", tierings.size())( - "new_count_ttls", ttl.PathsCount()); - // some string - - if (pathId) { - auto g = GetGranulePtrVerified(*pathId); - auto it = tierings.find(*pathId); - if (it == tierings.end()) { + + for (auto&& [gPathId, g] : GranulesStorage->GetTables()) { + auto it = ttl.find(gPathId); + if (it == ttl.end()) { g->RefreshTiering({}); } else { g->RefreshTiering(it->second); } - } else { - for (auto&& [gPathId, g] : GranulesStorage->GetTables()) { - auto it = tierings.find(gPathId); - if (it == tierings.end()) { - g->RefreshTiering({}); - } else { - g->RefreshTiering(it->second); - } - } } } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 92f1a4ed5e8a..3db0681a93e7 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -8,7 +8,6 @@ #include "storage/granule/granule.h" #include "storage/granule/storage.h" -#include #include #include #include @@ -95,8 +94,8 @@ class TColumnEngineForLogs: public IColumnEngine { TColumnEngineForLogs(const ui64 tabletId, const std::shared_ptr& dataAccessorsManager, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema); - virtual void OnTieringModified( - const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) override; + void OnTieringModified(const std::optional& ttl, const ui64 pathId) override; + void OnTieringModified(const THashMap& ttl) override; virtual std::shared_ptr CopyVersionedIndexPtr() const override { return std::make_shared(VersionedIndex); diff --git a/ydb/core/tx/columnshard/engines/scheme/tiering/tier_info.h b/ydb/core/tx/columnshard/engines/scheme/tiering/tier_info.h index 77f090e3b993..8e1cb9169495 100644 --- a/ydb/core/tx/columnshard/engines/scheme/tiering/tier_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/tiering/tier_info.h @@ -53,6 +53,21 @@ class TTierInfo { return std::make_shared(NTiering::NCommon::DeleteTierName, evictDuration, ttlColumn, unitsInSecond); } + static ui32 GetUnitsInSecond(const NKikimrSchemeOp::TTTLSettings::EUnit timeUnit) { + switch (timeUnit) { + case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS: + return 1; + case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS: + return 1000; + case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS: + return 1000 * 1000; + case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: + return 1000 * 1000 * 1000; + case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: + return 0; + } + } + TString GetDebugString() const { TStringBuilder sb; sb << "name=" << Name << ";duration=" << EvictDuration << ";column=" << EvictColumnName << ";serializer="; @@ -106,6 +121,7 @@ class TTierRef { }; class TTiering { + using TProto = NKikimrSchemeOp::TColumnDataLifeCycle::TTtl; using TTiersMap = THashMap>; TTiersMap TierByName; TSet OrderedTiers; @@ -199,6 +215,46 @@ class TTiering { return {}; } + TConclusionStatus DeserializeFromProto(const TProto& serialized) { + if (serialized.HasExpireAfterBytes()) { + return TConclusionStatus::Fail("TTL by size is not supported."); + } + if (!serialized.HasColumnName()) { + return TConclusionStatus::Fail("Missing column name in TTL settings"); + } + + const TString ttlColumnName = serialized.GetColumnName(); + const ui32 unitsInSecond = TTierInfo::GetUnitsInSecond(serialized.GetColumnUnit()); + + if (!serialized.TiersSize()) { + // legacy schema + if (!Add(TTierInfo::MakeTtl(TDuration::Seconds(serialized.GetExpireAfterSeconds()), ttlColumnName, unitsInSecond))) { + return TConclusionStatus::Fail("Invalid ttl settings"); + } + } + for (const auto& tier : serialized.GetTiers()) { + if (!tier.HasApplyAfterSeconds()) { + return TConclusionStatus::Fail("Missing eviction delay in tier description"); + } + std::shared_ptr tierInfo; + switch (tier.GetActionCase()) { + case NKikimrSchemeOp::TTTLSettings_TTier::kDelete: + tierInfo = TTierInfo::MakeTtl(TDuration::Seconds(tier.GetApplyAfterSeconds()), ttlColumnName, unitsInSecond); + break; + case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage: + tierInfo = std::make_shared(tier.GetEvictToExternalStorage().GetStorageName(), + TDuration::Seconds(tier.GetApplyAfterSeconds()), ttlColumnName, unitsInSecond); + break; + case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET: + return TConclusionStatus::Fail("No action in tier"); + } + if (!Add(tierInfo)) { + return TConclusionStatus::Fail("Invalid tier settings"); + } + } + return TConclusionStatus::Success(); + } + const TString& GetEvictColumnName() const { AFL_VERIFY(TTLColumnName); return *TTLColumnName; @@ -211,6 +267,21 @@ class TTiering { } return sb; } + + static THashSet GetUsedTiers(const TProto& ttlSettings) { + THashSet usedTiers; + for (const auto& tier : ttlSettings.GetTiers()) { + switch (tier.GetActionCase()) { + case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage: + usedTiers.emplace(tier.GetEvictToExternalStorage().GetStorageName()); + break; + case NKikimrSchemeOp::TTTLSettings_TTier::kDelete: + case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET: + break; + } + } + return usedTiers; + } }; } diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index 506020186346..28470aca8bda 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -285,8 +285,8 @@ class ICSController { } virtual NMetadata::NFetcher::ISnapshot::TPtr GetFallbackTiersSnapshot() const { - static std::shared_ptr result = - std::make_shared(TInstant::Now()); + static std::shared_ptr result = + std::make_shared(TInstant::Now()); return result; } diff --git a/ydb/core/tx/columnshard/loading/stages.cpp b/ydb/core/tx/columnshard/loading/stages.cpp index ed9a5fd6f87c..ac4ab092d4db 100644 --- a/ydb/core/tx/columnshard/loading/stages.cpp +++ b/ydb/core/tx/columnshard/loading/stages.cpp @@ -203,7 +203,7 @@ bool TTablesManagerInitializer::DoExecute(NTabletFlatExecutor::TTransactionConte } Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, tablesManagerLocal.GetTables().size()); Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_PRESETS, tablesManagerLocal.GetSchemaPresets().size()); - Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_TTLS, tablesManagerLocal.GetTtl().PathsCount()); + Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_TTLS, tablesManagerLocal.GetTtl().size()); Self->TablesManager = std::move(tablesManagerLocal); return true; diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 762583fc8ae6..1706563ffe65 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -132,8 +132,9 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { vIt = lastVersion.emplace(pathId, version).first; } if (vIt->second <= version) { - TTtl::TDescription description(ttlSettings.GetEnabled()); - Ttl.SetPathTtl(pathId, std::move(description)); + NOlap::TTiering deserializedTtl; + AFL_VERIFY(deserializedTtl.DeserializeFromProto(ttlSettings.GetEnabled()).IsSuccess()); + Ttl[pathId] = std::move(deserializedTtl); vIt->second = version; } } @@ -228,7 +229,7 @@ const TTableInfo& TTablesManager::GetTable(const ui64 pathId) const { } ui64 TTablesManager::GetMemoryUsage() const { - ui64 memory = Tables.size() * sizeof(TTableInfo) + PathsToDrop.size() * sizeof(ui64) + Ttl.PathsCount() * sizeof(TTtl::TDescription); + ui64 memory = Tables.size() * sizeof(TTableInfo) + PathsToDrop.size() * sizeof(ui64) + Ttl.size() * sizeof(NOlap::TTiering); if (PrimaryIndex) { memory += PrimaryIndex->MemoryUsage(); } @@ -240,7 +241,7 @@ void TTablesManager::DropTable(const ui64 pathId, const NOlap::TSnapshot& versio auto& table = Tables[pathId]; table.SetDropVersion(version); PathsToDrop.insert(pathId); - Ttl.DropPathTtl(pathId); + Ttl.erase(pathId); Schema::SaveTableDropVersion(db, pathId, version.GetPlanStep(), version.GetTxId()); } @@ -254,7 +255,7 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) { Y_ABORT_UNLESS(!HasTable(table.GetPathId())); Y_ABORT_UNLESS(table.IsEmpty()); - Schema::SaveTableInfo(db, table.GetPathId(), table.GetTieringUsage()); + Schema::SaveTableInfo(db, table.GetPathId()); const ui64 pathId = table.GetPathId(); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("method", "RegisterTable")("path_id", pathId); AFL_VERIFY(Tables.emplace(pathId, std::move(table)).second)("path_id", pathId)("size", Tables.size()); @@ -298,7 +299,7 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho PrimaryIndex->RegisterTable(i.first); } if (manager->IsReady()) { - PrimaryIndex->OnTieringModified(manager, Ttl, {}); + PrimaryIndex->OnTieringModified(Ttl); } } else { PrimaryIndex->RegisterSchemaVersion(version, NOlap::IColumnEngine::TSchemaInitializationData(versionInfo)); @@ -322,9 +323,11 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& isTtlModified = true; const auto& ttlSettings = versionInfo.GetTtlSettings(); if (ttlSettings.HasEnabled()) { - Ttl.SetPathTtl(pathId, TTtl::TDescription(ttlSettings.GetEnabled())); + NOlap::TTiering deserializedTtl; + AFL_VERIFY(deserializedTtl.DeserializeFromProto(ttlSettings.GetEnabled()).IsSuccess()); + Ttl[pathId] = std::move(deserializedTtl); } else { - Ttl.DropPathTtl(pathId); + Ttl.erase(pathId); } } @@ -343,7 +346,11 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& if (isTtlModified) { if (PrimaryIndex && manager->IsReady()) { - PrimaryIndex->OnTieringModified(manager, Ttl, pathId); + if (auto findTtl = Ttl.FindPtr(pathId)) { + PrimaryIndex->OnTieringModified(*findTtl, pathId); + } else { + PrimaryIndex->OnTieringModified({}, pathId); + } } } Schema::SaveTableVersionInfo(db, pathId, version, versionInfo); diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 5f6928f7cce6..f44ca4c872ce 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -1,7 +1,6 @@ #pragma once #include "columnshard_schema.h" -#include "columnshard_ttl.h" #include "blobs_action/abstract/storages_manager.h" #include "data_accessor/manager.h" @@ -10,6 +9,7 @@ #include #include #include +#include #include @@ -93,20 +93,10 @@ class TSchemaPreset: public TVersionedSchema DropVersion; YDB_READONLY_DEF(TSet, Versions); public: - const TString& GetTieringUsage() const { - return TieringUsage; - } - - TTableInfo& SetTieringUsage(const TString& data) { - TieringUsage = data; - return *this; - } - bool IsEmpty() const { return Versions.empty(); } @@ -136,7 +126,6 @@ class TTableInfo { template bool InitFromDB(const TRow& rowset) { PathId = rowset.template GetValue(); - TieringUsage = rowset.template GetValue(); if (rowset.template HaveValue() && rowset.template HaveValue()) { DropVersion.emplace( rowset.template GetValue(), rowset.template GetValue()); @@ -151,7 +140,7 @@ class TTablesManager { THashSet SchemaPresetsIds; THashMap ActualSchemaForPreset; THashSet PathsToDrop; - TTtl Ttl; + THashMap Ttl; std::unique_ptr PrimaryIndex; std::shared_ptr StoragesManager; std::shared_ptr DataAccessorsManager; @@ -171,14 +160,10 @@ class TTablesManager { bool TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const; bool TryFinalizeDropPathOnComplete(const ui64 pathId); - const TTtl& GetTtl() const { + const THashMap& GetTtl() const { return Ttl; } - bool AddTtls(THashMap& eviction) { - return Ttl.AddTtls(eviction); - } - const THashSet& GetPathsToDrop() const { return PathsToDrop; } diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index 8e08a0fd331f..e8d016f45d17 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -372,17 +372,11 @@ TSerializedTableRange MakeTestRange(std::pair range, bool inclusiveF } NMetadata::NFetcher::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecials& specials) { - std::unique_ptr cs(new NColumnShard::NTiers::TConfigsSnapshot(Now())); + std::unique_ptr cs(new NColumnShard::NTiers::TTiersSnapshot(Now())); if (specials.Tiers.empty()) { return cs; } - NColumnShard::NTiers::TTieringRule tRule; - tRule.SetTieringRuleId("Tiering1"); for (auto&& tier : specials.Tiers) { - if (!tRule.GetDefaultColumn()) { - tRule.SetDefaultColumn(tier.TtlColumn); - } - UNIT_ASSERT(tRule.GetDefaultColumn() == tier.TtlColumn); { NKikimrSchemeOp::TStorageTierConfig cProto; cProto.SetName(tier.Name); @@ -396,9 +390,7 @@ NMetadata::NFetcher::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpec NColumnShard::NTiers::TTierConfig tConfig(tier.Name, cProto); cs->MutableTierConfigs().emplace(tConfig.GetTierName(), tConfig); } - tRule.AddInterval(tier.Name, TDuration::Seconds((*tier.EvictAfter).Seconds())); } - cs->MutableTableTierings().emplace(tRule.GetTieringRuleId(), tRule); return cs; } diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 3dd60dcb8bb0..20418f574cf9 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -246,22 +246,25 @@ struct TTestSchema { const TTableSpecials& specials, NKikimrSchemeOp::TColumnTableSchema* schema); - static void InitTtl(const TTableSpecials& specials, NKikimrSchemeOp::TColumnDataLifeCycle::TTtl* ttl) { - Y_ABORT_UNLESS(specials.HasTtl()); - Y_ABORT_UNLESS(!specials.TtlColumn.empty()); - ttl->SetColumnName(specials.TtlColumn); - ttl->SetExpireAfterSeconds((*specials.EvictAfter).Seconds()); - } - static bool InitTiersAndTtl(const TTableSpecials& specials, NKikimrSchemeOp::TColumnDataLifeCycle* ttlSettings) { ttlSettings->SetVersion(1); - if (specials.HasTiers()) { - ttlSettings->SetUseTiering("Tiering1"); + if (!specials.HasTiers() && !specials.HasTtl()) { + return false; + } + ttlSettings->MutableEnabled()->SetColumnName(specials.TtlColumn); + for (const auto& tier : specials.Tiers) { + UNIT_ASSERT(tier.EvictAfter); + UNIT_ASSERT_EQUAL(specials.TtlColumn, tier.TtlColumn); + auto* tierSettings = ttlSettings->MutableEnabled()->AddTiers(); + tierSettings->MutableEvictToExternalStorage()->SetStorageName(tier.Name); + tierSettings->SetApplyAfterSeconds(tier.EvictAfter->Seconds()); } if (specials.HasTtl()) { - InitTtl(specials, ttlSettings->MutableEnabled()); + auto* tier = ttlSettings->MutableEnabled()->AddTiers(); + tier->MutableDelete(); + tier->SetApplyAfterSeconds((*specials.EvictAfter).Seconds()); } - return specials.HasTiers() || specials.HasTtl(); + return true; } static TString CreateTableTxBody(ui64 pathId, const std::vector& columns, diff --git a/ydb/core/tx/schemeshard/common/validation.cpp b/ydb/core/tx/schemeshard/common/validation.cpp index 1e52ec848195..fdbe79d264ea 100644 --- a/ydb/core/tx/schemeshard/common/validation.cpp +++ b/ydb/core/tx/schemeshard/common/validation.cpp @@ -58,16 +58,16 @@ bool TTTLValidator::ValidateUnit(const NScheme::TTypeInfo columnType, NKikimrSch return true; } -bool TTTLValidator::ValidateTiers(const NKikimrSchemeOp::TTTLSettings::TEnabled ttlSettings, TString& errStr) { - for (ui64 i = 0; i < ttlSettings.TiersSize(); ++i) { - const auto& tier = ttlSettings.GetTiers(i); +bool TTTLValidator::ValidateTiers(const google::protobuf::RepeatedPtrField& tiers, TString& errStr) { + for (ui64 i = 0; i < tiers.size(); ++i) { + const auto& tier = tiers[i]; if (!tier.HasApplyAfterSeconds()) { errStr = TStringBuilder() << "Tier " << i << ": missing ApplyAfterSeconds"; return false; } - if (i != 0 && tier.GetApplyAfterSeconds() <= ttlSettings.GetTiers(i - 1).GetApplyAfterSeconds()) { + if (i != 0 && tier.GetApplyAfterSeconds() <= tiers[i - 1].GetApplyAfterSeconds()) { errStr = TStringBuilder() << "Tiers in the sequence must have increasing ApplyAfterSeconds: " - << ttlSettings.GetTiers(i - 1).GetApplyAfterSeconds() << " (tier " << i - 1 + << tiers[i - 1].GetApplyAfterSeconds() << " (tier " << i - 1 << ") >= " << tier.GetApplyAfterSeconds() << " (tier " << i << ")"; return false; } diff --git a/ydb/core/tx/schemeshard/common/validation.h b/ydb/core/tx/schemeshard/common/validation.h index 0ee739982ea7..e110a72e85fd 100644 --- a/ydb/core/tx/schemeshard/common/validation.h +++ b/ydb/core/tx/schemeshard/common/validation.h @@ -9,6 +9,8 @@ namespace NKikimr::NSchemeShard::NValidation { class TTTLValidator { public: static bool ValidateUnit(const NScheme::TTypeInfo columnType, NKikimrSchemeOp::TTTLSettings::EUnit unit, TString& errStr); - static bool ValidateTiers(const NKikimrSchemeOp::TTTLSettings::TEnabled ttlSettings, TString& errStr); + static bool ValidateTiers(const google::protobuf::RepeatedPtrField& tiers, TString& errStr); + +private: }; } diff --git a/ydb/core/tx/schemeshard/olap/manager/manager.cpp b/ydb/core/tx/schemeshard/olap/manager/manager.cpp index 4d91e3c07f8e..0df8e96f0e6b 100644 --- a/ydb/core/tx/schemeshard/olap/manager/manager.cpp +++ b/ydb/core/tx/schemeshard/olap/manager/manager.cpp @@ -3,25 +3,43 @@ namespace NKikimr::NSchemeShard { void TTablesStorage::OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr object) { - const TString& tieringId = object->Description.GetTtlSettings().GetUseTiering(); - if (!!tieringId) { - PathsByTieringId[tieringId].emplace(pathId); + for (const auto& tier : object->Description.GetTtlSettings().GetEnabled().GetTiers()) { + std::optional usedExternalStorage; + switch (tier.GetActionCase()) { + case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage: + usedExternalStorage = tier.GetEvictToExternalStorage().GetStorageName(); + break; + case NKikimrSchemeOp::TTTLSettings_TTier::kDelete: + case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET: + break; + } + if (usedExternalStorage) { + AFL_VERIFY(PathsByTier[*usedExternalStorage].emplace(pathId).second); + } } for (auto&& s : object->GetColumnShards()) { - TablesByShard[s].AddId(pathId); + AFL_VERIFY(TablesByShard[s].AddId(pathId)); } } void TTablesStorage::OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPtr object) { - const TString& tieringId = object->Description.GetTtlSettings().GetUseTiering(); - if (!!tieringId) { - auto it = PathsByTieringId.find(tieringId); - if (PathsByTieringId.end() == it) { - return; + for (const auto& tier : object->Description.GetTtlSettings().GetEnabled().GetTiers()) { + std::optional usedExternalStorage; + switch (tier.GetActionCase()) { + case NKikimrSchemeOp::TTTLSettings_TTier::kEvictToExternalStorage: + usedExternalStorage = tier.GetEvictToExternalStorage().GetStorageName(); + break; + case NKikimrSchemeOp::TTTLSettings_TTier::kDelete: + case NKikimrSchemeOp::TTTLSettings_TTier::ACTION_NOT_SET: + break; } - it->second.erase(pathId); - if (it->second.empty()) { - PathsByTieringId.erase(it); + if (usedExternalStorage) { + auto findTier = PathsByTier.find(*usedExternalStorage); + AFL_VERIFY(findTier); + AFL_VERIFY(findTier->second.erase(pathId)); + if (findTier->second.empty()) { + PathsByTier.erase(findTier); + } } } for (auto&& s : object->GetColumnShards()) { @@ -29,9 +47,9 @@ void TTablesStorage::OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPt } } -const THashSet& TTablesStorage::GetTablesWithTiering(const TString& tieringId) const { - auto it = PathsByTieringId.find(tieringId); - if (it != PathsByTieringId.end()) { +const THashSet& TTablesStorage::GetTablesWithTier(const TString& storageId) const { + auto it = PathsByTier.find(storageId); + if (it != PathsByTier.end()) { return it->second; } else { return Default>(); @@ -78,13 +96,14 @@ TTablesStorage::TTableCreatedGuard TTablesStorage::BuildNew(const TPathId& id) { return TTableCreatedGuard(*this, id); } -size_t TTablesStorage::Drop(const TPathId& id) { +bool TTablesStorage::Drop(const TPathId& id) { auto it = Tables.find(id); if (it == Tables.end()) { - return 0; + return false; } else { OnRemoveObject(id, it->second); - return Tables.erase(id); + Tables.erase(it); + return true; } } diff --git a/ydb/core/tx/schemeshard/olap/manager/manager.h b/ydb/core/tx/schemeshard/olap/manager/manager.h index 0873a12da22d..8c025690e97a 100644 --- a/ydb/core/tx/schemeshard/olap/manager/manager.h +++ b/ydb/core/tx/schemeshard/olap/manager/manager.h @@ -9,7 +9,7 @@ namespace NKikimr::NSchemeShard { class TTablesStorage { private: THashMap Tables; - THashMap> PathsByTieringId; + THashMap> PathsByTier; THashMap TablesByShard; void OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr object); @@ -20,7 +20,7 @@ class TTablesStorage { TColumnTablesLayout GetTablesLayout(const std::vector& tabletIds) const; - const THashSet& GetTablesWithTiering(const TString& tieringId) const; + const THashSet& GetTablesWithTier(const TString& storageId) const; class TTableReadGuard { protected: @@ -115,7 +115,7 @@ class TTablesStorage { TTableReadGuard at(const TPathId& id) const { return TTableReadGuard(Tables.at(id)); } - size_t Drop(const TPathId& id); + bool Drop(const TPathId& id); }; } diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h index 6a36133245e9..da5db2ad1117 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h @@ -26,15 +26,10 @@ class TConverterModifyToAlter { if (enabled.HasColumnUnit()) { alterEnabled->SetColumnUnit(enabled.GetColumnUnit()); } - for (const auto& tier : enabled.GetTiers()) { - alterEnabled->AddTiers()->CopyFrom(tier); - } + *alterEnabled->MutableTiers() = enabled.GetTiers(); } else if (tableTtl.HasDisabled()) { alterTtl->MutableDisabled(); } - if (tableTtl.HasUseTiering()) { - alterTtl->SetUseTiering(tableTtl.GetUseTiering()); - } } for (auto&& dsColumn : dsDescription.GetColumns()) { diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp index 4fb76b4a75a0..850151c51dd3 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp @@ -271,8 +271,7 @@ class TAlterColumnTable: public TSubOperation { return result; } - const bool hasTiering = Transaction.HasAlterColumnTable() && Transaction.GetAlterColumnTable().HasAlterTtlSettings() && - Transaction.GetAlterColumnTable().GetAlterTtlSettings().HasUseTiering(); + const bool hasTiering = Transaction.GetAlterColumnTable().GetAlterTtlSettings().GetEnabled().TiersSize(); if (hasTiering && HasAppData() && !AppDataVerified().FeatureFlags.GetEnableTieringInColumnShard()) { result->SetError(NKikimrScheme::StatusPreconditionFailed, "Tiering functionality is disabled for OLAP tables"); return result; diff --git a/ydb/core/tx/schemeshard/olap/ttl/schema.cpp b/ydb/core/tx/schemeshard/olap/ttl/schema.cpp index 379f35012d24..f1b6e73c93ff 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/schema.cpp @@ -5,9 +5,6 @@ namespace NKikimr::NSchemeShard::NOlap::NAlter { TConclusionStatus TOlapTTL::Update(const TOlapTTLUpdate& update) { const ui64 currentTtlVersion = Proto.GetVersion(); const auto& ttlUpdate = update.GetPatch(); - if (ttlUpdate.HasUseTiering()) { - Proto.SetUseTiering(ttlUpdate.GetUseTiering()); - } if (ttlUpdate.HasEnabled()) { *Proto.MutableEnabled() = ttlUpdate.GetEnabled(); } diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp index 8cae2a0b4fdb..e4a3e2b5d389 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -44,7 +44,7 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif return false; } - if (!ttl.HasExpireAfterSeconds()) { + if (!ttl.HasExpireAfterSeconds() && ttl.GetTiers().empty()) { errors.AddError("TTL without eviction time"); return false; } @@ -66,6 +66,10 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif errors.AddError(errStr); return false; } + if (!NValidation::TTTLValidator::ValidateTiers(ttl.GetTiers(), errStr)) { + errors.AddError(errStr); + return false; + } { bool correct = false; if (column->GetKeyOrder() && *column->GetKeyOrder() == 0) { diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 7d86703aab85..edf4e8a5506e 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -1218,12 +1218,22 @@ TCheckFunc HasColumnTableTtlSettingsDisabled() { }; } -TCheckFunc HasColumnTableTtlSettingsTiering(const TString& tieringName) { +TCheckFunc HasColumnTableTtlSettingsTier(const TString& columnName, const TDuration& evictAfter, const std::optional& storageName) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { const auto& table = record.GetPathDescription().GetColumnTableDescription(); UNIT_ASSERT(table.HasTtlSettings()); const auto& ttl = table.GetTtlSettings(); - UNIT_ASSERT_EQUAL(ttl.GetUseTiering(), tieringName); + UNIT_ASSERT(ttl.HasEnabled()); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), columnName); + UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 1); + const auto& tier = ttl.GetEnabled().GetTiers(0); + UNIT_ASSERT_VALUES_EQUAL(tier.GetApplyAfterSeconds(), evictAfter.Seconds()); + if (storageName) { + UNIT_ASSERT(tier.HasEvictToExternalStorage()); + UNIT_ASSERT_VALUES_EQUAL(tier.GetEvictToExternalStorage().GetStorageName(), storageName); + } else { + UNIT_ASSERT(tier.HasDelete()); + } }; } diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index bcd1e2c0e163..c46f6bc5c610 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -134,7 +134,7 @@ namespace NLs { TCheckFunc HasColumnTableTtlSettingsVersion(ui64 ttlSettingsVersion); TCheckFunc HasColumnTableTtlSettingsEnabled(const TString& columnName, const TDuration& expireAfter); TCheckFunc HasColumnTableTtlSettingsDisabled(); - TCheckFunc HasColumnTableTtlSettingsTiering(const TString& tierName); + TCheckFunc HasColumnTableTtlSettingsTier(const TString& columnName, const TDuration& evictAfter, const std::optional& storageName); TCheckFunc CheckPartCount(const TString& name, ui32 partCount, ui32 maxParts, ui32 tabletCount, ui32 groupCount, NKikimrSchemeOp::EPathState pathState = NKikimrSchemeOp::EPathState::EPathStateNoChanges); diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index f0f5fb0832ee..5545ce39eda3 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -470,7 +470,14 @@ Y_UNIT_TEST_SUITE(TOlap) { Name: "Table3" ColumnShardCount: 1 TtlSettings { - UseTiering : "Tiering1" + Enabled: { + ColumnName: "timestamp" + ColumnUnit: UNIT_AUTO + Tiers: { + EvictAfterSeconds: 360 + StorageName: "Tier1" + } + } } )"; @@ -481,13 +488,20 @@ Y_UNIT_TEST_SUITE(TOlap) { NLs::HasColumnTableSchemaPreset("default"), NLs::HasColumnTableSchemaVersion(1), NLs::HasColumnTableTtlSettingsVersion(1), - NLs::HasColumnTableTtlSettingsTiering("Tiering1"))); + NLs::HasColumnTableTtlSettingsTier("timestamp", TDuration::Seconds(360), "Tier1"))); TString tableSchema4 = R"( Name: "Table4" ColumnShardCount: 1 TtlSettings { - UseTiering : "Tiering1" + Enabled: { + ColumnName: "timestamp" + ColumnUnit: UNIT_AUTO + Tiers: { + EvictAfterSeconds: 3600000000 + StorageName: "Tier1" + } + } } )"; @@ -631,7 +645,14 @@ Y_UNIT_TEST_SUITE(TOlap) { TestAlterColumnTable(runtime, ++txId, "/MyRoot/OlapStore", R"( Name: "ColumnTable" AlterTtlSettings { - UseTiering : "Tiering1" + Enabled: { + ColumnName: "timestamp" + ColumnUnit: UNIT_AUTO + Tiers: { + EvictAfterSeconds: 3600000000 + StorageName: "Tier1" + } + } } )"); env.TestWaitNotification(runtime, txId); diff --git a/ydb/core/tx/tiering/external_data.cpp b/ydb/core/tx/tiering/external_data.cpp index b6616bc760aa..812215b01ff3 100644 --- a/ydb/core/tx/tiering/external_data.cpp +++ b/ydb/core/tx/tiering/external_data.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -19,11 +18,7 @@ TSnapshotConstructor::TSnapshotConstructor() { } std::vector TSnapshotConstructor::DoGetManagers() const { - std::vector result = { - TTierConfig::GetBehaviour(), - TTieringRule::GetBehaviour() - }; - return result; + return { TTierConfig::GetBehaviour() }; } } diff --git a/ydb/core/tx/tiering/external_data.h b/ydb/core/tx/tiering/external_data.h index 02b963ab6d4b..456ad1ff59e3 100644 --- a/ydb/core/tx/tiering/external_data.h +++ b/ydb/core/tx/tiering/external_data.h @@ -9,7 +9,7 @@ namespace NKikimr::NColumnShard::NTiers { -class TSnapshotConstructor: public NMetadata::NFetcher::TSnapshotsFetcher { +class TSnapshotConstructor: public NMetadata::NFetcher::TSnapshotsFetcher { private: using TNavigate = NSchemeCache::TSchemeCacheNavigate; using TBaseActor = TActor; diff --git a/ydb/core/tx/tiering/manager.cpp b/ydb/core/tx/tiering/manager.cpp index 57462d745d3a..dbe06df7e0d4 100644 --- a/ydb/core/tx/tiering/manager.cpp +++ b/ydb/core/tx/tiering/manager.cpp @@ -12,7 +12,7 @@ class TTiersManager::TActor: public TActorBootstrapped { std::shared_ptr Owner; NMetadata::NFetcher::ISnapshotsFetcher::TPtr SecretsFetcher; std::shared_ptr SecretsSnapshot; - std::shared_ptr ConfigsSnapshot; + std::shared_ptr ConfigsSnapshot; TActorId GetExternalDataActorId() const { return NMetadata::NProvider::MakeServiceId(SelfId().NodeId()); } @@ -45,7 +45,7 @@ class TTiersManager::TActor: public TActorBootstrapped { void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { auto snapshot = ev->Get()->GetSnapshot(); - if (auto configs = std::dynamic_pointer_cast(snapshot)) { + if (auto configs = std::dynamic_pointer_cast(snapshot)) { AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "TEvRefreshSubscriberData")("snapshot", "configs"); ConfigsSnapshot = configs; if (SecretsSnapshot) { @@ -123,7 +123,7 @@ void TTiersManager::TakeConfigs(NMetadata::NFetcher::ISnapshot::TPtr snapshotExt ALS_INFO(NKikimrServices::TX_TIERING) << "Take configs:" << (snapshotExt ? " snapshots" : "") << (secrets ? " secrets" : "") << " at tablet " << TabletId; - auto snapshotPtr = std::dynamic_pointer_cast(snapshotExt); + auto snapshotPtr = std::dynamic_pointer_cast(snapshotExt); Y_ABORT_UNLESS(snapshotPtr); Snapshot = snapshotExt; Secrets = secrets; @@ -192,32 +192,6 @@ NMetadata::NFetcher::ISnapshotsFetcher::TPtr TTiersManager::GetExternalDataManip return ExternalDataManipulation; } -THashMap TTiersManager::GetTiering() const { - THashMap result; - AFL_VERIFY(IsReady()); - auto snapshotPtr = std::dynamic_pointer_cast(Snapshot); - Y_ABORT_UNLESS(snapshotPtr); - auto& tierConfigs = snapshotPtr->GetTierConfigs(); - for (auto&& i : PathIdTiering) { - auto* tieringRule = snapshotPtr->GetTieringById(i.second); - if (tieringRule) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("path_id", i.first)("tiering_name", i.second)("event", "activation"); - NOlap::TTiering tiering = tieringRule->BuildOlapTiers(); - for (auto& [name, tier] : tiering.GetTierByName()) { - AFL_VERIFY(name != NOlap::NTiering::NCommon::DeleteTierName); - auto it = tierConfigs.find(name); - if (it != tierConfigs.end()) { - tier->SetSerializer(NTiers::ConvertCompression(it->second.GetCompression())); - } - } - result.emplace(i.first, std::move(tiering)); - } else { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("path_id", i.first)("tiering_name", i.second)("event", "not_found"); - } - } - return result; -} - TActorId TTiersManager::GetActorId() const { if (Actor) { return Actor->SelfId(); diff --git a/ydb/core/tx/tiering/manager.h b/ydb/core/tx/tiering/manager.h index 147ee27f54f8..d0a464e40e61 100644 --- a/ydb/core/tx/tiering/manager.h +++ b/ydb/core/tx/tiering/manager.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -53,7 +54,6 @@ class TTiersManager: public ITiersManager { const TActorId TabletActorId; std::function ShardCallback; TActor* Actor = nullptr; - std::unordered_map PathIdTiering; TManagers Managers; std::shared_ptr Secrets; @@ -69,13 +69,10 @@ class TTiersManager: public ITiersManager { { } TActorId GetActorId() const; - THashMap GetTiering() const; void TakeConfigs(NMetadata::NFetcher::ISnapshot::TPtr snapshot, std::shared_ptr secrets); - void EnablePathId(const ui64 pathId, const TString& tieringId) { - PathIdTiering.emplace(pathId, tieringId); + void EnablePathId(const ui64 /*pathId*/, const THashSet& /*usedTiers*/) { } - void DisablePathId(const ui64 pathId) { - PathIdTiering.erase(pathId); + void DisablePathId(const ui64 /*pathId*/) { } bool IsReady() const { diff --git a/ydb/core/tx/tiering/rule/behaviour.cpp b/ydb/core/tx/tiering/rule/behaviour.cpp deleted file mode 100644 index df7ad1973101..000000000000 --- a/ydb/core/tx/tiering/rule/behaviour.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "behaviour.h" -#include "initializer.h" -#include "checker.h" -#include "manager.h" - -namespace NKikimr::NColumnShard::NTiers { - -TTieringRuleBehaviour::TFactory::TRegistrator TTieringRuleBehaviour::Registrator(TTieringRule::GetTypeId()); - -TString TTieringRuleBehaviour::GetInternalStorageTablePath() const { - return "tiering/rules"; -} - -NMetadata::NInitializer::IInitializationBehaviour::TPtr TTieringRuleBehaviour::ConstructInitializer() const { - return std::make_shared(); -} - -NMetadata::NModifications::IOperationsManager::TPtr TTieringRuleBehaviour::ConstructOperationsManager() const { - return std::make_shared(); -} - -TString TTieringRuleBehaviour::GetTypeId() const { - return TTieringRule::GetTypeId(); -} - -} diff --git a/ydb/core/tx/tiering/rule/behaviour.h b/ydb/core/tx/tiering/rule/behaviour.h deleted file mode 100644 index c10f2f24d73f..000000000000 --- a/ydb/core/tx/tiering/rule/behaviour.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "object.h" -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTieringRuleBehaviour: public NMetadata::TClassBehaviour { -private: - static TFactory::TRegistrator Registrator; -protected: - virtual std::shared_ptr ConstructInitializer() const override; - virtual std::shared_ptr ConstructOperationsManager() const override; - - virtual TString GetInternalStorageTablePath() const override; - virtual TString GetTypeId() const override; - -}; - -} diff --git a/ydb/core/tx/tiering/rule/checker.cpp b/ydb/core/tx/tiering/rule/checker.cpp deleted file mode 100644 index 1210b66ed0b4..000000000000 --- a/ydb/core/tx/tiering/rule/checker.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "checker.h" -#include "ss_checker.h" - -#include -#include -#include -#include -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -void TRulePreparationActor::StartChecker() { - if (!Tierings || !Secrets || !SSCheckResult) { - return; - } - auto g = PassAwayGuard(); - if (!SSCheckResult->GetContent().GetOperationAllow()) { - Controller->OnPreparationProblem(SSCheckResult->GetContent().GetDenyReason()); - return; - } - - for (auto&& tiering : Objects) { - for (auto&& interval : tiering.GetIntervals()) { - auto tier = Tierings->GetTierById(interval.GetTierName()); - if (!tier) { - Controller->OnPreparationProblem("unknown tier usage: " + interval.GetTierName()); - return; - } else if (!Secrets->CheckSecretAccess(tier->GetAccessKey(), Context.GetExternalData().GetUserToken())) { - Controller->OnPreparationProblem("no access for secret: " + tier->GetAccessKey().DebugString()); - return; - } else if (!Secrets->CheckSecretAccess(tier->GetSecretKey(), Context.GetExternalData().GetUserToken())) { - Controller->OnPreparationProblem("no access for secret: " + tier->GetSecretKey().DebugString()); - return; - } - } - } - Controller->OnPreparationFinished(std::move(Objects)); -} - -void TRulePreparationActor::Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev) { - auto& proto = ev->Get()->Record; - if (proto.HasError()) { - Controller->OnPreparationProblem(proto.GetError().GetErrorMessage()); - PassAway(); - } else if (proto.HasContent()) { - SSCheckResult = SSFetcher->UnpackResult(ev->Get()->Record.GetContent().GetData()); - if (!SSCheckResult) { - Controller->OnPreparationProblem("cannot unpack ss-fetcher result for class " + SSFetcher->GetClassName()); - PassAway(); - } else { - StartChecker(); - } - } else { - Y_ABORT_UNLESS(false); - } -} - -void TRulePreparationActor::Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { - if (auto snapshot = ev->Get()->GetSnapshotPtrAs()) { - Tierings = snapshot; - } else if (auto snapshot = ev->Get()->GetSnapshotPtrAs()) { - Secrets = snapshot; - } else { - Y_ABORT_UNLESS(false); - } - StartChecker(); -} - -void TRulePreparationActor::Bootstrap() { - Become(&TThis::StateMain); - Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), - new NMetadata::NProvider::TEvAskSnapshot(std::make_shared())); - Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), - new NMetadata::NProvider::TEvAskSnapshot(std::make_shared())); - { - SSFetcher = std::make_shared(); - SSFetcher->SetUserToken(Context.GetExternalData().GetUserToken()); - SSFetcher->SetActivityType(Context.GetActivityType()); - for (auto&& i : Objects) { - SSFetcher->MutableTieringRuleIds().emplace(i.GetTieringRuleId()); - } - Register(new TSSFetchingActor(SSFetcher, std::make_shared(SelfId()), TDuration::Seconds(10))); - } -} - -TRulePreparationActor::TRulePreparationActor(std::vector&& objects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const NMetadata::NModifications::IOperationsManager::TInternalModificationContext& context) - : Objects(std::move(objects)) - , Controller(controller) - , Context(context) -{ - -} - -} diff --git a/ydb/core/tx/tiering/rule/checker.h b/ydb/core/tx/tiering/rule/checker.h deleted file mode 100644 index ec6e0f3d66e7..000000000000 --- a/ydb/core/tx/tiering/rule/checker.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include "object.h" -#include "ss_fetcher.h" - -#include -#include - -#include -#include -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TRulePreparationActor: public NActors::TActorBootstrapped { -private: - std::vector Objects; - NMetadata::NModifications::IAlterPreparationController::TPtr Controller; - NMetadata::NModifications::IOperationsManager::TInternalModificationContext Context; - std::shared_ptr Tierings; - std::shared_ptr Secrets; - std::shared_ptr SSFetcher; - std::optional SSCheckResult; - void StartChecker(); -protected: - void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev); - void Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev); -public: - STATEFN(StateMain) { - switch (ev->GetTypeRewrite()) { - hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); - hFunc(NSchemeShard::TEvSchemeShard::TEvProcessingResponse, Handle); - default: - break; - } - } - void Bootstrap(); - - TRulePreparationActor(std::vector&& objects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const NMetadata::NModifications::IOperationsManager::TInternalModificationContext& context); -}; - -} diff --git a/ydb/core/tx/tiering/rule/initializer.cpp b/ydb/core/tx/tiering/rule/initializer.cpp deleted file mode 100644 index 96c1c3cff550..000000000000 --- a/ydb/core/tx/tiering/rule/initializer.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "initializer.h" -#include "object.h" - -namespace NKikimr::NColumnShard::NTiers { - -TVector TTierRulesInitializer::BuildModifiers() const { - TVector result; - { - Ydb::Table::CreateTableRequest request; - request.set_session_id(""); - request.set_path(TTieringRule::GetBehaviour()->GetStorageTablePath()); - request.add_primary_key("tieringRuleId"); - { - auto& column = *request.add_columns(); - column.set_name("tieringRuleId"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); - } - { - auto& column = *request.add_columns(); - column.set_name("defaultColumn"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); - } - { - auto& column = *request.add_columns(); - column.set_name("description"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); - } - result.emplace_back(new NMetadata::NInitializer::TGenericTableModifier(request, "create")); - auto hRequest = TTieringRule::AddHistoryTableScheme(request); - result.emplace_back(new NMetadata::NInitializer::TGenericTableModifier(hRequest, "create_history")); - } - result.emplace_back(NMetadata::NInitializer::TACLModifierConstructor::GetReadOnlyModifier(TTieringRule::GetBehaviour()->GetStorageTablePath(), "acl")); - result.emplace_back(NMetadata::NInitializer::TACLModifierConstructor::GetReadOnlyModifier(TTieringRule::GetBehaviour()->GetStorageHistoryTablePath(), "acl_history")); - return result; -} - -void TTierRulesInitializer::DoPrepare(NMetadata::NInitializer::IInitializerInput::TPtr controller) const { - controller->OnPreparationFinished(BuildModifiers()); -} - -} diff --git a/ydb/core/tx/tiering/rule/initializer.h b/ydb/core/tx/tiering/rule/initializer.h deleted file mode 100644 index 93f15e78f9c4..000000000000 --- a/ydb/core/tx/tiering/rule/initializer.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once -#include -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTierRulesInitializer: public NMetadata::NInitializer::IInitializationBehaviour { -protected: - TVector BuildModifiers() const; - virtual void DoPrepare(NMetadata::NInitializer::IInitializerInput::TPtr controller) const override; -public: -}; - -} diff --git a/ydb/core/tx/tiering/rule/manager.cpp b/ydb/core/tx/tiering/rule/manager.cpp deleted file mode 100644 index 88ff0533c444..000000000000 --- a/ydb/core/tx/tiering/rule/manager.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "manager.h" -#include "initializer.h" -#include "checker.h" - -namespace NKikimr::NColumnShard::NTiers { - -void TTieringRulesManager::DoPrepareObjectsBeforeModification(std::vector&& objects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const TInternalModificationContext& context, const NMetadata::NModifications::TAlterOperationContext& /*alterContext*/) const { - TActivationContext::Register(new TRulePreparationActor(std::move(objects), controller, context)); -} - -NMetadata::NModifications::TOperationParsingResult TTieringRulesManager::DoBuildPatchFromSettings( - const NYql::TObjectSettingsImpl& settings, - TInternalModificationContext& /*context*/) const { - if (HasAppData() && !AppDataVerified().FeatureFlags.GetEnableTieringInColumnShard()) { - return TConclusionStatus::Fail("Tiering functionality is disabled for OLAP tables."); - } - - NMetadata::NInternal::TTableRecord result; - result.SetColumn(TTieringRule::TDecoder::TieringRuleId, NMetadata::NInternal::TYDBValue::Utf8(settings.GetObjectId())); - if (settings.GetObjectId().StartsWith("$") || settings.GetObjectId().StartsWith("_")) { - return TConclusionStatus::Fail("tiering rule cannot start with '$', '_' characters"); - } - { - auto fValue = settings.GetFeaturesExtractor().Extract(TTieringRule::TDecoder::DefaultColumn); - if (fValue) { - if (fValue->empty()) { - return TConclusionStatus::Fail("defaultColumn cannot be empty"); - } - result.SetColumn(TTieringRule::TDecoder::DefaultColumn, NMetadata::NInternal::TYDBValue::Utf8(*fValue)); - } - } - { - auto fValue = settings.GetFeaturesExtractor().Extract(TTieringRule::TDecoder::Description); - if (fValue) { - result.SetColumn(TTieringRule::TDecoder::Description, NMetadata::NInternal::TYDBValue::Utf8(*fValue)); - } - } - return result; -} - -} diff --git a/ydb/core/tx/tiering/rule/manager.h b/ydb/core/tx/tiering/rule/manager.h deleted file mode 100644 index d5646dbf3002..000000000000 --- a/ydb/core/tx/tiering/rule/manager.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once -#include "object.h" - -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTieringRulesManager: public NMetadata::NModifications::TGenericOperationsManager { -protected: - virtual void DoPrepareObjectsBeforeModification(std::vector&& objects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const TInternalModificationContext& context, const NMetadata::NModifications::TAlterOperationContext& alterContext) const override; - - virtual NMetadata::NModifications::TOperationParsingResult DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, - TInternalModificationContext& context) const override; -}; - -} diff --git a/ydb/core/tx/tiering/rule/object.cpp b/ydb/core/tx/tiering/rule/object.cpp deleted file mode 100644 index 18dfc9288bc1..000000000000 --- a/ydb/core/tx/tiering/rule/object.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "object.h" -#include "behaviour.h" - -#include -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -NJson::TJsonValue TTieringRule::GetDebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue(TDecoder::TieringRuleId, TieringRuleId); - result.InsertValue(TDecoder::DefaultColumn, DefaultColumn); - result.InsertValue(TDecoder::Description, SerializeDescriptionToJson()); - return result; -} - -NJson::TJsonValue TTieringRule::SerializeDescriptionToJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - auto& jsonRules = result.InsertValue("rules", NJson::JSON_ARRAY); - for (auto&& i : Intervals) { - jsonRules.AppendValue(i.SerializeToJson()); - } - return result; -} - -bool TTieringRule::DeserializeDescriptionFromJson(const NJson::TJsonValue& jsonInfo) { - const NJson::TJsonValue::TArray* rules; - if (!jsonInfo["rules"].GetArrayPointer(&rules)) { - return false; - } - if (rules->empty()) { - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "tiering_rule_deserialization_failed")("reason", "empty_rules"); - return false; - } - for (auto&& i : *rules) { - TTieringInterval interval; - if (!interval.DeserializeFromJson(i)) { - return false; - } - Intervals.emplace_back(std::move(interval)); - } - std::sort(Intervals.begin(), Intervals.end()); - return true; -} - -NMetadata::NInternal::TTableRecord TTieringRule::SerializeToRecord() const { - NMetadata::NInternal::TTableRecord result; - result.SetColumn(TDecoder::TieringRuleId, NMetadata::NInternal::TYDBValue::Utf8(TieringRuleId)); - result.SetColumn(TDecoder::DefaultColumn, NMetadata::NInternal::TYDBValue::Utf8(DefaultColumn)); - { - auto jsonDescription = SerializeDescriptionToJson(); - NJsonWriter::TBuf sout; - sout.WriteJsonValue(&jsonDescription, true); - result.SetColumn(TDecoder::Description, NMetadata::NInternal::TYDBValue::Utf8(sout.Str())); - } - return result; -} - -bool TTieringRule::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r) { - if (!decoder.Read(decoder.GetTieringRuleIdIdx(), TieringRuleId, r)) { - return false; - } - if (!decoder.Read(decoder.GetDefaultColumnIdx(), DefaultColumn, r)) { - return false; - } - if (DefaultColumn.empty()) { - return false; - } - NJson::TJsonValue jsonDescription; - if (!decoder.ReadJson(decoder.GetDescriptionIdx(), jsonDescription, r)) { - return false; - } - if (!DeserializeDescriptionFromJson(jsonDescription)) { - return false; - } - return true; -} - -NKikimr::NOlap::TTiering TTieringRule::BuildOlapTiers() const { - AFL_VERIFY(!Intervals.empty()); - NOlap::TTiering result; - for (auto&& r : Intervals) { - AFL_VERIFY(result.Add(std::make_shared(r.GetTierName(), r.GetDurationForEvict(), GetDefaultColumn()))); - } - return result; -} - -bool TTieringRule::ContainsTier(const TString& tierName) const { - for (auto&& i : Intervals) { - if (i.GetTierName() == tierName) { - return true; - } - } - return false; -} - -NMetadata::IClassBehaviour::TPtr TTieringRule::GetBehaviour() { - static std::shared_ptr result = std::make_shared(); - return result; -} - -} diff --git a/ydb/core/tx/tiering/rule/object.h b/ydb/core/tx/tiering/rule/object.h deleted file mode 100644 index 566f10e5efc8..000000000000 --- a/ydb/core/tx/tiering/rule/object.h +++ /dev/null @@ -1,94 +0,0 @@ -#pragma once -#include -#include - -#include -#include -#include -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTieringInterval { -private: - YDB_ACCESSOR_DEF(TString, TierName); - YDB_ACCESSOR_DEF(TDuration, DurationForEvict); -public: - TTieringInterval() = default; - TTieringInterval(const TString& name, const TDuration d) - : TierName(name) - , DurationForEvict(d) - { - - } - - bool operator<(const TTieringInterval& item) const { - return DurationForEvict < item.DurationForEvict; - } - - NJson::TJsonValue SerializeToJson() const { - NJson::TJsonValue result; - result.InsertValue("tierName", TierName); - result.InsertValue("durationForEvict", DurationForEvict.ToString()); - return result; - } - - bool DeserializeFromJson(const NJson::TJsonValue& jsonInfo) { - if (!jsonInfo["tierName"].GetString(&TierName)) { - return false; - } - const TString dStr = jsonInfo["durationForEvict"].GetStringRobust(); - if (!TDuration::TryParse(dStr, DurationForEvict)) { - return false; - } - return true; - } -}; - -class TTieringRule: public NMetadata::NModifications::TObject { -private: - YDB_ACCESSOR_DEF(TString, TieringRuleId); - YDB_ACCESSOR_DEF(TString, DefaultColumn); - YDB_ACCESSOR_DEF(TVector, Intervals); -protected: - NJson::TJsonValue SerializeDescriptionToJson() const; - bool DeserializeDescriptionFromJson(const NJson::TJsonValue& jsonInfo); -public: - static NMetadata::IClassBehaviour::TPtr GetBehaviour(); - - bool ContainsTier(const TString& tierName) const; - - void AddInterval(const TString& name, const TDuration evDuration) { - Intervals.emplace_back(TTieringInterval(name, evDuration)); - } - - static TString GetTypeId() { - return "TIERING_RULE"; - } - - NJson::TJsonValue GetDebugJson() const; - - class TDecoder: public NMetadata::NInternal::TDecoderBase { - private: - YDB_READONLY(i32, TieringRuleIdIdx, -1); - YDB_READONLY(i32, DefaultColumnIdx, -1); - YDB_READONLY(i32, DescriptionIdx, -1); - public: - static inline const TString TieringRuleId = "tieringRuleId"; - static inline const TString DefaultColumn = "defaultColumn"; - static inline const TString Description = "description"; - - TDecoder(const Ydb::ResultSet& rawData) { - TieringRuleIdIdx = GetFieldIndex(rawData, TieringRuleId); - DefaultColumnIdx = GetFieldIndex(rawData, DefaultColumn); - DescriptionIdx = GetFieldIndex(rawData, Description); - } - }; - NMetadata::NInternal::TTableRecord SerializeToRecord() const; - bool DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r); - NKikimr::NOlap::TTiering BuildOlapTiers() const; -}; - -} diff --git a/ydb/core/tx/tiering/rule/ya.make b/ydb/core/tx/tiering/rule/ya.make deleted file mode 100644 index 80b625857c1f..000000000000 --- a/ydb/core/tx/tiering/rule/ya.make +++ /dev/null @@ -1,24 +0,0 @@ -LIBRARY() - -SRCS( - manager.cpp - object.cpp - GLOBAL behaviour.cpp - initializer.cpp - checker.cpp - ss_checker.cpp - GLOBAL ss_fetcher.cpp -) - -PEERDIR( - ydb/services/metadata/abstract - ydb/services/metadata/common - ydb/services/metadata/initializer - ydb/services/metadata/manager - ydb/services/bg_tasks/abstract - ydb/core/tx/schemeshard -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/ydb/core/tx/tiering/snapshot.cpp b/ydb/core/tx/tiering/snapshot.cpp index cfc003d7ba7b..d64987b5b62e 100644 --- a/ydb/core/tx/tiering/snapshot.cpp +++ b/ydb/core/tx/tiering/snapshot.cpp @@ -9,14 +9,13 @@ namespace NKikimr::NColumnShard::NTiers { -bool TConfigsSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawDataResult) { - Y_ABORT_UNLESS(rawDataResult.result_sets().size() == 2); +bool TTiersSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawDataResult) { + Y_ABORT_UNLESS(rawDataResult.result_sets().size() == 1); ParseSnapshotObjects(rawDataResult.result_sets()[0], [this](TTierConfig&& s) {TierConfigs.emplace(s.GetTierName(), s); }); - ParseSnapshotObjects(rawDataResult.result_sets()[1], [this](TTieringRule&& s) {TableTierings.emplace(s.GetTieringRuleId(), s); }); return true; } -std::optional TConfigsSnapshot::GetTierById(const TString& tierName) const { +std::optional TTiersSnapshot::GetTierById(const TString& tierName) const { auto it = TierConfigs.find(tierName); if (it == TierConfigs.end()) { return {}; @@ -25,38 +24,12 @@ std::optional TConfigsSnapshot::GetTierById(const TString& tierName } } -const TTieringRule* TConfigsSnapshot::GetTieringById(const TString& tieringId) const { - auto it = TableTierings.find(tieringId); - if (it == TableTierings.end()) { - return nullptr; - } else { - return &it->second; - } -} - -std::set TConfigsSnapshot::GetTieringIdsForTier(const TString& tierName) const { - std::set result; - for (auto&& i : TableTierings) { - for (auto&& t : i.second.GetIntervals()) { - if (t.GetTierName() == tierName) { - result.emplace(i.second.GetTieringRuleId()); - break; - } - } - } - return result; -} - -TString NTiers::TConfigsSnapshot::DoSerializeToString() const { +TString NTiers::TTiersSnapshot::DoSerializeToString() const { NJson::TJsonValue result = NJson::JSON_MAP; auto& jsonTiers = result.InsertValue("tiers", NJson::JSON_MAP); for (auto&& i : TierConfigs) { jsonTiers.InsertValue(i.first, i.second.GetDebugJson()); } - auto& jsonTiering = result.InsertValue("rules", NJson::JSON_MAP); - for (auto&& i : TableTierings) { - jsonTiering.InsertValue(i.first, i.second.GetDebugJson()); - } return result.GetStringRobust(); } diff --git a/ydb/core/tx/tiering/snapshot.h b/ydb/core/tx/tiering/snapshot.h index db323d11f253..4eea9921c56a 100644 --- a/ydb/core/tx/tiering/snapshot.h +++ b/ydb/core/tx/tiering/snapshot.h @@ -2,7 +2,6 @@ #include #include #include -#include #include @@ -10,20 +9,15 @@ namespace NKikimr::NColumnShard::NTiers { -class TConfigsSnapshot: public NMetadata::NFetcher::ISnapshot { +class TTiersSnapshot: public NMetadata::NFetcher::ISnapshot { private: using TBase = NMetadata::NFetcher::ISnapshot; using TConfigsMap = TMap; YDB_ACCESSOR_DEF(TConfigsMap, TierConfigs); - using TTieringMap = TMap; - YDB_ACCESSOR_DEF(TTieringMap, TableTierings); protected: virtual bool DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawData) override; virtual TString DoSerializeToString() const override; public: - - std::set GetTieringIdsForTier(const TString& tierName) const; - const TTieringRule* GetTieringById(const TString& tieringId) const; std::optional GetTierById(const TString& tierName) const; using TBase::TBase; }; diff --git a/ydb/core/tx/tiering/tier/checker.cpp b/ydb/core/tx/tiering/tier/checker.cpp index 1fd719069d43..32ec8da0e7b4 100644 --- a/ydb/core/tx/tiering/tier/checker.cpp +++ b/ydb/core/tx/tiering/tier/checker.cpp @@ -1,36 +1,16 @@ #include "checker.h" #include -#include #include namespace NKikimr::NColumnShard::NTiers { void TTierPreparationActor::StartChecker() { - if (!Tierings || !Secrets || !SSCheckResult) { + if (!Secrets) { return; } auto g = PassAwayGuard(); - if (!SSCheckResult->GetContent().GetOperationAllow()) { - Controller->OnPreparationProblem(SSCheckResult->GetContent().GetDenyReason()); - return; - } for (auto&& tier : Objects) { - if (Context.GetActivityType() == NMetadata::NModifications::IOperationsManager::EActivityType::Drop) { - std::set tieringsWithTiers; - for (auto&& i : Tierings->GetTableTierings()) { - if (i.second.ContainsTier(tier.GetTierName())) { - tieringsWithTiers.emplace(i.first); - if (tieringsWithTiers.size() > 10) { - break; - } - } - } - if (tieringsWithTiers.size()) { - Controller->OnPreparationProblem("tier in usage for tierings: " + JoinSeq(", ", tieringsWithTiers)); - return; - } - } if (!Secrets->CheckSecretAccess(tier.GetAccessKey(), Context.GetExternalData().GetUserToken())) { Controller->OnPreparationProblem("no access for secret: " + tier.GetAccessKey().DebugString()); return; @@ -42,49 +22,9 @@ void TTierPreparationActor::StartChecker() { Controller->OnPreparationFinished(std::move(Objects)); } -void TTierPreparationActor::Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev) { - auto& proto = ev->Get()->Record; - if (proto.HasError()) { - Controller->OnPreparationProblem(proto.GetError().GetErrorMessage()); - PassAway(); - } else if (proto.HasContent()) { - SSCheckResult = SSFetcher->UnpackResult(ev->Get()->Record.GetContent().GetData()); - if (!SSCheckResult) { - Controller->OnPreparationProblem("cannot unpack ss-fetcher result for class " + SSFetcher->GetClassName()); - PassAway(); - } else { - StartChecker(); - } - } else { - Y_ABORT_UNLESS(false); - } -} - void TTierPreparationActor::Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { if (auto snapshot = ev->Get()->GetSnapshotPtrAs()) { Secrets = snapshot; - } else if (auto snapshot = ev->Get()->GetSnapshotPtrAs()) { - Tierings = snapshot; - std::set tieringIds; - std::set tiersChecked; - for (auto&& tier : Objects) { - if (!tiersChecked.emplace(tier.GetTierName()).second) { - continue; - } - auto tIds = Tierings->GetTieringIdsForTier(tier.GetTierName()); - if (tieringIds.empty()) { - tieringIds = std::move(tIds); - } else { - tieringIds.insert(tIds.begin(), tIds.end()); - } - } - { - SSFetcher = std::make_shared(); - SSFetcher->SetUserToken(Context.GetExternalData().GetUserToken()); - SSFetcher->SetActivityType(Context.GetActivityType()); - SSFetcher->MutableTieringRuleIds() = tieringIds; - Register(new TSSFetchingActor(SSFetcher, std::make_shared(SelfId()), TDuration::Seconds(10))); - } } else { Y_ABORT_UNLESS(false); } @@ -95,8 +35,6 @@ void TTierPreparationActor::Bootstrap() { Become(&TThis::StateMain); Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), new NMetadata::NProvider::TEvAskSnapshot(std::make_shared())); - Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), - new NMetadata::NProvider::TEvAskSnapshot(std::make_shared())); } TTierPreparationActor::TTierPreparationActor(std::vector&& objects, diff --git a/ydb/core/tx/tiering/tier/checker.h b/ydb/core/tx/tiering/tier/checker.h index 2b1d5ffd2e7a..109c9de208aa 100644 --- a/ydb/core/tx/tiering/tier/checker.h +++ b/ydb/core/tx/tiering/tier/checker.h @@ -2,7 +2,6 @@ #include "object.h" #include -#include #include #include @@ -18,18 +17,13 @@ class TTierPreparationActor: public NActors::TActorBootstrapped::TPtr Controller; NMetadata::NModifications::IOperationsManager::TInternalModificationContext Context; std::shared_ptr Secrets; - std::shared_ptr Tierings; - std::shared_ptr SSFetcher; - std::optional SSCheckResult; void StartChecker(); protected: void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev); - void Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev); public: STATEFN(StateMain) { switch (ev->GetTypeRewrite()) { hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); - hFunc(NSchemeShard::TEvSchemeShard::TEvProcessingResponse, Handle); default: break; } diff --git a/ydb/core/tx/tiering/rule/ss_checker.cpp b/ydb/core/tx/tiering/tier/ss_checker.cpp similarity index 100% rename from ydb/core/tx/tiering/rule/ss_checker.cpp rename to ydb/core/tx/tiering/tier/ss_checker.cpp diff --git a/ydb/core/tx/tiering/rule/ss_checker.h b/ydb/core/tx/tiering/tier/ss_checker.h similarity index 100% rename from ydb/core/tx/tiering/rule/ss_checker.h rename to ydb/core/tx/tiering/tier/ss_checker.h diff --git a/ydb/core/tx/tiering/rule/ss_fetcher.cpp b/ydb/core/tx/tiering/tier/ss_fetcher.cpp similarity index 100% rename from ydb/core/tx/tiering/rule/ss_fetcher.cpp rename to ydb/core/tx/tiering/tier/ss_fetcher.cpp diff --git a/ydb/core/tx/tiering/rule/ss_fetcher.h b/ydb/core/tx/tiering/tier/ss_fetcher.h similarity index 100% rename from ydb/core/tx/tiering/rule/ss_fetcher.h rename to ydb/core/tx/tiering/tier/ss_fetcher.h diff --git a/ydb/core/tx/tiering/tier/ya.make b/ydb/core/tx/tiering/tier/ya.make index f319e8e28af1..822435a2ec4c 100644 --- a/ydb/core/tx/tiering/tier/ya.make +++ b/ydb/core/tx/tiering/tier/ya.make @@ -6,9 +6,11 @@ SRCS( initializer.cpp checker.cpp GLOBAL behaviour.cpp + ss_checker.cpp ) PEERDIR( + ydb/services/bg_tasks/abstract ydb/services/metadata/initializer ydb/services/metadata/abstract ydb/services/metadata/secret diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index d3b707a24c0a..0afdde164162 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -75,9 +75,6 @@ class TLocalHelper: public Tests::NCS::THelper { TBase::CreateTestOlapTable(sender, storeName, Sprintf(R"( Name: "%s" ColumnShardCount: %d - TtlSettings: { - UseTiering: "tiering1" - } Sharding { HashSharding { Function: %s @@ -219,7 +216,6 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { TActorId ProviderId; TInstant Start; YDB_READONLY_FLAG(Found, false); - YDB_ACCESSOR(ui32, ExpectedTieringsCount, 1); YDB_ACCESSOR(ui32, ExpectedTiersCount, 1); using TKeyCheckers = TMap; @@ -261,17 +257,12 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { } void CheckFound(NMetadata::NProvider::TEvRefreshSubscriberData* event) { - auto snapshot = event->GetSnapshotAs(); + auto snapshot = event->GetSnapshotAs(); if (!snapshot) { Cerr << "incorrect snapshot" << Endl; return; } Cerr << "SNAPSHOT: " << snapshot->SerializeToString() << Endl; - const auto& tierings = snapshot->GetTableTierings(); - if (tierings.size() != ExpectedTieringsCount) { - Cerr << "TieringsCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTieringsCount << Endl; - return; - } if (ExpectedTiersCount != snapshot->GetTierConfigs().size()) { Cerr << "TiersCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTiersCount << Endl; return; @@ -281,9 +272,6 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { if (i.first.StartsWith("TIER.")) { auto value = snapshot->GetTierById(i.first.substr(5)); jsonData = value->SerializeConfigToJson(); - } else if (i.first.StartsWith("TIERING_RULE.")) { - auto value = snapshot->GetTierById(i.first.substr(13)); - jsonData = value->SerializeConfigToJson(); } else { Y_ABORT_UNLESS(false); } @@ -344,25 +332,23 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { Tests::TClient client(serverSettings); auto& runtime = *server->GetRuntime(); + runtime.SetLogPriority(NKikimrServices::TX_TIERING, NLog::PRI_DEBUG); auto sender = runtime.AllocateEdgeActor(); server->SetupRootStoragePools(sender); TLocalHelper lHelper(*server); - lHelper.CreateTestOlapTable(); { - TTestCSEmulator* emulator = new TTestCSEmulator; + lHelper.CreateTestOlapTable(); + lHelper.StartSchemaRequest("CREATE OBJECT tier1 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc") + "`"); + lHelper.StartSchemaRequest("CREATE OBJECT tier2 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc") + "`"); + lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1, Interval("P20D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); + + TTestCSEmulator* emulator = new TTestCSEmulator(); emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc")); emulator->SetExpectedTiersCount(2); runtime.Register(emulator); runtime.SimulateSleep(TDuration::Seconds(10)); Cerr << "Initialization finished" << Endl; - - lHelper.StartSchemaRequest("CREATE OBJECT tier1 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc") + "`"); - lHelper.StartSchemaRequest("CREATE OBJECT tiering1 (" - "TYPE TIERING_RULE) WITH (defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )", false); - lHelper.StartSchemaRequest("CREATE OBJECT tier2 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc") + "`"); - lHelper.StartSchemaRequest("CREATE OBJECT tiering1 (" - "TYPE TIERING_RULE) WITH (defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )"); { const TInstant start = Now(); while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) { @@ -387,13 +373,11 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { } { emulator->ResetConditions(); - emulator->SetExpectedTieringsCount(0); emulator->SetExpectedTiersCount(0); - lHelper.StartSchemaRequest("DROP OBJECT tier1(TYPE TIER)", false); - lHelper.StartSchemaRequest("DROP OBJECT tiering1(TYPE TIERING_RULE)", false); + // TODO: add validation + // lHelper.StartSchemaRequest("DROP OBJECT tier1(TYPE TIER)", false); lHelper.StartSchemaRequest("DROP TABLE `/Root/olapStore/olapTable`"); - lHelper.StartSchemaRequest("DROP OBJECT tiering1(TYPE TIERING_RULE)"); lHelper.StartSchemaRequest("DROP OBJECT tier1(TYPE TIER)"); lHelper.StartSchemaRequest("DROP OBJECT tier2(TYPE TIER)"); @@ -438,56 +422,52 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { TLocalHelper lHelper(*server); lHelper.SetUseQueryService(useQueryService); - lHelper.CreateTestOlapTable("olapTable"); - runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_NOTICE); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NLog::PRI_INFO); + runtime.SetLogPriority(NKikimrServices::TX_TIERING, NLog::PRI_DEBUG); // runtime.SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NLog::PRI_DEBUG); runtime.SimulateSleep(TDuration::Seconds(10)); Cerr << "Initialization finished" << Endl; lHelper.StartSchemaRequest("CREATE OBJECT tier1 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc1") + "`", true, false); { - TTestCSEmulator emulator; - emulator.MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); - emulator.SetExpectedTieringsCount(0); - emulator.SetExpectedTiersCount(1); - emulator.CheckRuntime(runtime); + TTestCSEmulator* emulator = new TTestCSEmulator; + runtime.Register(emulator); + emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); + emulator->SetExpectedTiersCount(1); + emulator->CheckRuntime(runtime); } lHelper.StartSchemaRequest("CREATE OBJECT tier2 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc2") + "`"); - lHelper.StartSchemaRequest("CREATE OBJECT IF NOT EXISTS tiering1 (TYPE TIERING_RULE) " - "WITH (defaultColumn = timestamp, description = `" + ConfigTiering1Str + "`)"); - lHelper.StartSchemaRequest("CREATE OBJECT tiering2 (TYPE TIERING_RULE) " - "WITH (defaultColumn = timestamp, description = `" + ConfigTiering2Str + "` )", true, false); { - TTestCSEmulator emulator; - emulator.MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); - emulator.MutableCheckers().emplace("TIER.tier2", TJsonChecker("Name", "abc2")); - emulator.SetExpectedTieringsCount(2); - emulator.SetExpectedTiersCount(2); - emulator.CheckRuntime(runtime); - } - - lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)", false); - lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)", false); - lHelper.StartSchemaRequest("DROP OBJECT tiering2 (TYPE TIERING_RULE)"); - lHelper.StartSchemaRequest("DROP OBJECT tiering1 (TYPE TIERING_RULE)", false); + TTestCSEmulator* emulator = new TTestCSEmulator(); + runtime.Register(emulator); + emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); + emulator->MutableCheckers().emplace("TIER.tier2", TJsonChecker("Name", "abc2")); + emulator->SetExpectedTiersCount(2); + emulator->CheckRuntime(runtime); + } + + lHelper.CreateTestOlapTable("olapTable"); + lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1, Interval("P20D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); + + // TODO: add validation + // lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)", false); + // lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)", false); lHelper.StartSchemaRequest("DROP TABLE `/Root/olapStore/olapTable`"); - lHelper.StartSchemaRequest("DROP OBJECT tiering1 (TYPE TIERING_RULE)", true, false); { - TTestCSEmulator emulator; - emulator.SetExpectedTieringsCount(0); - emulator.SetExpectedTiersCount(2); - emulator.CheckRuntime(runtime); + TTestCSEmulator* emulator = new TTestCSEmulator; + runtime.Register(emulator); + emulator->SetExpectedTiersCount(2); + emulator->CheckRuntime(runtime); } lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)"); lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)", true, false); { - TTestCSEmulator emulator; - emulator.SetExpectedTieringsCount(0); - emulator.SetExpectedTiersCount(0); - emulator.CheckRuntime(runtime); + TTestCSEmulator* emulator = new TTestCSEmulator; + runtime.Register(emulator); + emulator->SetExpectedTiersCount(0); + emulator->CheckRuntime(runtime); } //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE); @@ -573,6 +553,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { // runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_NOTICE); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::BG_TASKS, NLog::PRI_DEBUG); + // runtime.SetLogPriority(NKikimrServices::TX_TIERING, NLog::PRI_DEBUG); // runtime.SetLogPriority(NKikimrServices::TX_PROXY_SCHEME_CACHE, NLog::PRI_DEBUG); TLocalHelper lHelper(*server); @@ -587,21 +568,17 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)"); lHelper.StartSchemaRequest("CREATE OBJECT tier2 ( " "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)"); - - lHelper.StartSchemaRequest("CREATE OBJECT tiering1 (" - "TYPE TIERING_RULE) WITH (defaultColumn = timestamp, description = `" + ConfigTiering1Str + "` )"); - lHelper.StartSchemaRequest("CREATE OBJECT tiering2 (" - "TYPE TIERING_RULE) WITH (defaultColumn = timestamp, description = `" + ConfigTiering2Str + "` )"); { - TTestCSEmulator* emulator = new TTestCSEmulator; + TTestCSEmulator* emulator = new TTestCSEmulator(); runtime.Register(emulator); emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "fakeTier")); emulator->MutableCheckers().emplace("TIER.tier2", TJsonChecker("ObjectStorage.Endpoint", TierEndpoint)); - emulator->SetExpectedTieringsCount(2); emulator->SetExpectedTiersCount(2); emulator->CheckRuntime(runtime); } + lHelper.CreateTestOlapTable("olapTable", 2); + lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1, Interval("P20D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); Cerr << "Wait tables" << Endl; runtime.SimulateSleep(TDuration::Seconds(20)); Cerr << "Initialization tables" << Endl; @@ -647,8 +624,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { lHelper.DropTable("/Root/olapStore/olapTable"); lHelper.StartDataRequest("DELETE FROM `/Root/olapStore/olapTable`"); */ - lHelper.StartSchemaRequest("UPSERT OBJECT tiering1 (" - "TYPE TIERING_RULE) WITH (defaultColumn = timestamp, description = `" + ConfigTieringNothingStr + "` )"); + lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10000D") TO EXTERNAL DATA SOURCE tier1, Interval("P20000D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); { const TInstant start = Now(); bool check = false; @@ -939,11 +915,11 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { { TVector> result; lHelper.StartScanRequest("SELECT MAX(timestamp) as a, MIN(timestamp) as b, COUNT(*) as c FROM `/Root/olapStore/olapTable`", true, &result); - UNIT_ASSERT(result.size() == 1); - UNIT_ASSERT(result.front().size() == 3); - UNIT_ASSERT(GetValueResult(result.front(), "c")->GetProto().uint64_value() == 600000); - UNIT_ASSERT(GetValueResult(result.front(), "a")->GetProto().uint64_value() == 599999000000); - UNIT_ASSERT(GetValueResult(result.front(), "b")->GetProto().uint64_value() == 0); + UNIT_ASSERT_VALUES_EQUAL(result.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.front().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(GetValueResult(result.front(), "c")->GetProto().uint64_value(), 600000); + UNIT_ASSERT_VALUES_EQUAL(GetValueResult(result.front(), "a")->GetProto().uint64_value(), 599999000000); + UNIT_ASSERT_VALUES_EQUAL(GetValueResult(result.front(), "b")->GetProto().uint64_value(), 0); } const ui32 reduceStepsCount = 1; for (ui32 i = 0; i < reduceStepsCount; ++i) { diff --git a/ydb/core/tx/tiering/ya.make b/ydb/core/tx/tiering/ya.make index b7412d358938..4090ce51fb6d 100644 --- a/ydb/core/tx/tiering/ya.make +++ b/ydb/core/tx/tiering/ya.make @@ -19,7 +19,6 @@ PEERDIR( ydb/core/blobstorage ydb/core/protos ydb/core/tx/schemeshard - ydb/core/tx/tiering/rule ydb/core/tx/tiering/tier ydb/core/tablet_flat/protos ydb/core/wrappers diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index ba5207084a3c..e35b05368cfd 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -49,8 +49,6 @@ THashSet GetAlterOperationKinds(const Ydb::Table::AlterTabl req->alter_columns_size() || req->ttl_action_case() != Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET || - req->tiering_action_case() != - Ydb::Table::AlterTableRequest::TIERING_ACTION_NOT_SET || req->has_alter_storage_settings() || req->add_column_families_size() || req->alter_column_families_size() || req->set_compaction_policy() || req->has_alter_partitioning_settings() || @@ -545,10 +543,6 @@ void FillColumnDescriptionImpl(TYdbProto& out, ythrow yexception() << "invalid TTL settings: " << error; } } - - if (in.GetTTLSettings().HasUseTiering()) { - out.set_tiering(in.GetTTLSettings().GetUseTiering()); - } } } @@ -589,10 +583,6 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc ythrow yexception() << "invalid TTL settings: " << error; } } - - if (in.GetTtlSettings().HasUseTiering()) { - out.set_tiering(in.GetTtlSettings().GetUseTiering()); - } } out.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN); @@ -867,12 +857,6 @@ bool BuildAlterColumnTableModifyScheme(const TString& path, const Ydb::Table::Al } else if (req->has_drop_ttl_settings()) { alterColumnTable->MutableAlterTtlSettings()->MutableDisabled(); } - - if (req->has_set_tiering()) { - alterColumnTable->MutableAlterTtlSettings()->SetUseTiering(req->set_tiering()); - } else if (req->has_drop_tiering()) { - alterColumnTable->MutableAlterTtlSettings()->SetUseTiering(""); - } } return true; diff --git a/ydb/core/ydb_convert/table_settings.cpp b/ydb/core/ydb_convert/table_settings.cpp index 367eb7a8db89..9047391c958c 100644 --- a/ydb/core/ydb_convert/table_settings.cpp +++ b/ydb/core/ydb_convert/table_settings.cpp @@ -228,10 +228,6 @@ bool FillCreateTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, } } - if (proto.tiering().size()) { - tableDesc.MutableTTLSettings()->SetUseTiering(proto.tiering()); - } - if (proto.has_storage_settings()) { TColumnFamilyManager families(tableDesc.MutablePartitionConfig()); if (!families.ApplyStorageSettings(proto.storage_settings(), &code, &error)) { @@ -391,12 +387,6 @@ bool FillAlterTableSettingsDesc(NKikimrSchemeOp::TTableDescription& tableDesc, tableDesc.MutableTTLSettings()->MutableDisabled(); } - if (proto.has_set_tiering()) { - tableDesc.MutableTTLSettings()->SetUseTiering(proto.set_tiering()); - } else if (proto.has_drop_tiering()) { - tableDesc.MutableTTLSettings()->SetUseTiering(""); - } - if (!changed && !hadPartitionConfig) { tableDesc.ClearPartitionConfig(); } diff --git a/ydb/public/api/protos/draft/ydb_logstore.proto b/ydb/public/api/protos/draft/ydb_logstore.proto index 8c0a24c53aaa..34705aa0f593 100644 --- a/ydb/public/api/protos/draft/ydb_logstore.proto +++ b/ydb/public/api/protos/draft/ydb_logstore.proto @@ -60,10 +60,6 @@ message Tier { Ydb.Table.TtlSettings eviction = 2; // When to evict data to the next tier (or remove if none) } -message TieringSettings { - optional string tiering_id = 2; -} - message CreateLogStoreRequest { Ydb.Operations.OperationParams operation_params = 1; @@ -135,8 +131,8 @@ message CreateLogTableRequest { }; oneof ttl_specification { Ydb.Table.TtlSettings ttl_settings = 5; - TieringSettings tiering_settings = 6; }; + reserved 6; // Specifies the desired number of ColumnShards for this table uint32 shards_count = 7; @@ -160,9 +156,9 @@ message DescribeLogTableResult { string schema_preset_name = 2; Schema schema = 3; + reserved 4; oneof ttl_specification { Ydb.Table.TtlSettings ttl_settings = 5; - TieringSettings tiering_settings = 4; } // Specifies the desired number of ColumnShards for this table @@ -195,9 +191,9 @@ message AlterLogTableRequest { oneof ttl_action { google.protobuf.Empty drop_ttl_settings = 3; Ydb.Table.TtlSettings set_ttl_settings = 4; - TieringSettings set_tiering_settings = 5; google.protobuf.Empty drop_tiering_settings = 6; } + reserved 5; } message AlterLogTableResponse { diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index cb6b86286579..172c97ceb3d0 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -667,8 +667,7 @@ message CreateTableRequest { Ydb.FeatureFlag.Status key_bloom_filter = 16; // Read replicas settings for table ReadReplicasSettings read_replicas_settings = 17; - // Tiering rules name. It specifies how data migrates from one tier (logical storage) to another. - string tiering = 18; + reserved 18; // Is temporary table bool temporary = 19; // Is table column or row oriented @@ -747,11 +746,7 @@ message AlterTableRequest { repeated string drop_changefeeds = 20; // Rename existed index repeated RenameIndexItem rename_indexes = 21; - // Setup or remove tiering - oneof tiering_action { - string set_tiering = 22; - google.protobuf.Empty drop_tiering = 23; - } + reserved 22, 23; } message AlterTableResponse { diff --git a/ydb/public/lib/experimental/ydb_logstore.cpp b/ydb/public/lib/experimental/ydb_logstore.cpp index 758e14986afd..bbfe436889a1 100644 --- a/ydb/public/lib/experimental/ydb_logstore.cpp +++ b/ydb/public/lib/experimental/ydb_logstore.cpp @@ -186,8 +186,6 @@ void TLogTableDescription::SerializeTo(Ydb::LogStore::CreateLogTableRequest& req if (TtlSettings) { TtlSettings->SerializeTo(*request.mutable_ttl_settings()); - } else if (TieringSettings) { - TieringSettings->SerializeTo(*request.mutable_tiering_settings()); } } diff --git a/ydb/public/lib/experimental/ydb_logstore.h b/ydb/public/lib/experimental/ydb_logstore.h index 633730e015da..be8148b023f3 100644 --- a/ydb/public/lib/experimental/ydb_logstore.h +++ b/ydb/public/lib/experimental/ydb_logstore.h @@ -152,21 +152,6 @@ struct TLogTableSharding { TLogTableSharding(const Ydb::LogStore::DescribeLogTableResult& desc); }; -class TTieringSettings { -private: - TString TieringId; -public: - TTieringSettings(const TString& tieringId) - : TieringId(tieringId) { - - } - - void SerializeTo(Ydb::LogStore::TieringSettings& proto) const { - proto.set_tiering_id(TieringId); - } - -}; - class TLogTableDescription { public: TLogTableDescription(const TString& schemaPresetName, const TLogTableSharding& sharding); @@ -200,16 +185,11 @@ class TLogTableDescription { TtlSettings = settings; return *this; } - TLogTableDescription& SetTieringSettings(const TTieringSettings& settings) { - TieringSettings = settings; - return *this; - } private: const TString SchemaPresetName; const TSchema Schema; const TLogTableSharding Sharding; TMaybe TtlSettings; - TMaybe TieringSettings; TString Owner; TVector Permissions; TVector EffectivePermissions; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index ca4a031cb063..2ef786076dfc 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -331,11 +331,6 @@ class TTableDescription::TImpl { TtlSettings_ = std::move(*ttlSettings); } - // tiering - if (proto.tiering().size()) { - Tiering_ = proto.tiering(); - } - if (proto.store_type()) { StoreType_ = (proto.store_type() == Ydb::Table::STORE_TYPE_COLUMN) ? EStoreType::Column : EStoreType::Row; } @@ -407,9 +402,7 @@ class TTableDescription::TImpl { } for (const auto& shardStats : Proto_.table_stats().partition_stats()) { - PartitionStats_.emplace_back( - TPartitionStats{shardStats.rows_estimate(), shardStats.store_size(), shardStats.leader_node_id()} - ); + PartitionStats_.emplace_back(TPartitionStats{ shardStats.rows_estimate(), shardStats.store_size(), shardStats.leader_node_id() }); } TableStats.Rows = Proto_.table_stats().rows_estimate(); @@ -566,10 +559,6 @@ class TTableDescription::TImpl { return TtlSettings_; } - const TMaybe& GetTiering() const { - return Tiering_; - } - EStoreType GetStoreType() const { return StoreType_; } @@ -650,7 +639,6 @@ class TTableDescription::TImpl { TVector Indexes_; TVector Changefeeds_; TMaybe TtlSettings_; - TMaybe Tiering_; TString Owner_; TVector Permissions_; TVector EffectivePermissions_; @@ -717,7 +705,7 @@ TMaybe TTableDescription::GetTtlSettings() const { } TMaybe TTableDescription::GetTiering() const { - return Impl_->GetTiering(); + return Nothing(); } EStoreType TTableDescription::GetStoreType() const { @@ -940,10 +928,6 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con ttl->SerializeTo(*request.mutable_ttl_settings()); } - if (const auto& tiering = Impl_->GetTiering()) { - request.set_tiering(*tiering); - } - if (Impl_->GetStoreType() == EStoreType::Column) { request.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN); } diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index f885240587ee..bb7215e8a9db 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -660,6 +660,7 @@ class TTableDescription { TVector GetIndexDescriptions() const; TVector GetChangefeedDescriptions() const; TMaybe GetTtlSettings() const; + // Deprecated. Use GetTtlSettings() instead TMaybe GetTiering() const; EStoreType GetStoreType() const; From c6a0f5f8d25195c35f9e954d72b42089faf0b5a5 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Thu, 28 Nov 2024 12:27:37 +0000 Subject: [PATCH 02/12] fix build --- ydb/core/kqp/provider/yql_kikimr_gateway.cpp | 2 +- ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index 717b393478a3..d4533f7d8960 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -316,7 +316,7 @@ void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::T auto* tierProto = proto.add_tiers(); tierProto->set_apply_after_seconds(tier.ApplyAfter.Seconds()); if (tier.StorageName) { - tierProto->mutable_evict_to_external_storage()->set_storage_name(tier.StorageName); + tierProto->mutable_evict_to_external_storage()->set_storage_name(*tier.StorageName); } else { tierProto->mutable_delete_(); } diff --git a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp index ac27997fe55f..6ca31686c344 100644 --- a/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp @@ -57,7 +57,7 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, return false; } - if (!NValidation::TTTLValidator::ValidateTiers(enabled, errStr)) { + if (!NValidation::TTTLValidator::ValidateTiers(enabled.GetTiers(), errStr)) { return false; } From 8cd82bfd868bdfe70d37f6e36a08f5ed87ecc775 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Thu, 28 Nov 2024 15:04:50 +0000 Subject: [PATCH 03/12] fix build --- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 37 +++++++++++-------- ydb/public/sdk/cpp/client/ydb_table/table.cpp | 12 +++--- ydb/public/sdk/cpp/client/ydb_table/table.h | 4 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 806adb2cd59b..acb9892f1ba8 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5450,10 +5450,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); - auto ttl = desc.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier1"); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetEvictionDelay(), TDuration::Seconds(10)); + auto ttl = desc.GetTableDescription().GetTtlSettings(); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "tier1"); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10)); } auto query2 = TStringBuilder() << R"( --!syntax_v1 @@ -5466,10 +5466,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); - auto ttl = desc.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier2"); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetEvictionDelay(), TDuration::Seconds(10)); + auto ttl = desc.GetTableDescription().GetTtlSettings(); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "tier2"); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10)); } auto query3 = TStringBuilder() << R"( @@ -5497,10 +5497,10 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_C(desc.IsSuccess(), desc.GetIssues().ToString()); UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); - auto ttl = desc.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier1"); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetEvictionDelay(), TDuration::Seconds(10)); + auto ttl = desc.GetTableDescription().GetTtlSettings(); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "tier1"); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10)); } auto query5 = TStringBuilder() << R"( @@ -8197,10 +8197,15 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { const auto& description = describeResult.GetTableDescription(); UNIT_ASSERT(describeResult.GetTableDescription().GetTtlSettings()); - auto ttl = describeResult.GetTableDescription().GetTtlSettings()->GetDateTypeColumn(); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetExpireAfter(), TDuration::Hours(1)); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(ttl.GetTiers()[0].GetStorageName(), "tier1"); + auto ttl = describeResult.GetTableDescription().GetTtlSettings(); + UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 2); + auto evictTier = ttl->GetTiers()[0]; + UNIT_ASSERT(std::holds_alternative(evictTier.GetAction())); + UNIT_ASSERT_VALUES_EQUAL(std::get(evictTier.GetAction()).StorageName, "tier1"); + UNIT_ASSERT_VALUES_EQUAL(evictTier.GetApplyAfter(), TDuration::Seconds(10)); + auto deleteTier = ttl->GetTiers()[1]; + UNIT_ASSERT(std::holds_alternative(deleteTier.GetAction())); + UNIT_ASSERT_VALUES_EQUAL(deleteTier.GetApplyAfter(), TDuration::Hours(1)); } { auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << R"(` RESET (TTL);)"; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 2ef786076dfc..4cde4085bfc7 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -2917,12 +2917,12 @@ bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription& //////////////////////////////////////////////////////////////////////////////// TTtlTierSettings::TTtlTierSettings(TDuration evictionDelay, const TAction& action) - : EvictAfter_(evictionDelay) + : ApplyAfter_(evictionDelay) , Action_(action) { } TTtlTierSettings::TTtlTierSettings(const Ydb::Table::TtlTier& tier) - : EvictAfter_(TDuration::Seconds(tier.apply_after_seconds())) { + : ApplyAfter_(TDuration::Seconds(tier.apply_after_seconds())) { switch (tier.action_case()) { case Ydb::Table::TtlTier::kDelete: Action_ = TTtlDeleteAction(); @@ -2936,7 +2936,7 @@ TTtlTierSettings::TTtlTierSettings(const Ydb::Table::TtlTier& tier) } void TTtlTierSettings::SerializeTo(Ydb::Table::TtlTier& proto) const { - proto.set_apply_after_seconds(EvictAfter_.Seconds()); + proto.set_apply_after_seconds(ApplyAfter_.Seconds()); std::visit(TOverloaded{ [&proto](const TTtlDeleteAction&) { proto.mutable_delete_(); }, @@ -2948,8 +2948,8 @@ void TTtlTierSettings::SerializeTo(Ydb::Table::TtlTier& proto) const { Action_); } -TDuration TTtlTierSettings::GetEvictAfter() const { - return EvictAfter_; +TDuration TTtlTierSettings::GetApplyAfter() const { + return ApplyAfter_; } const TTtlTierSettings::TAction& TTtlTierSettings::GetAction() const { @@ -3150,7 +3150,7 @@ std::optional TTtlSettings::GetExpireAfter() const { std::optional TTtlSettings::GetExpireAfterFrom(const TVector& tiers) { for (const auto& tier : tiers) { if (std::holds_alternative(tier.GetAction())) { - return tier.GetEvictAfter(); + return tier.GetApplyAfter(); } } return std::nullopt; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index bb7215e8a9db..e010337106ac 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -443,11 +443,11 @@ class TTtlTierSettings { explicit TTtlTierSettings(const Ydb::Table::TtlTier& tier); void SerializeTo(Ydb::Table::TtlTier& proto) const; - TDuration GetEvictAfter() const; + TDuration GetApplyAfter() const; const TAction& GetAction() const; private: - TDuration EvictAfter_; + TDuration ApplyAfter_; TAction Action_; }; From 9757e4a44cc34a3a9ec414828c6147dbfbd1659b Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Fri, 29 Nov 2024 09:40:28 +0000 Subject: [PATCH 04/12] fix ff vaildation --- ydb/core/tx/schemeshard/olap/operations/alter_table.cpp | 6 ------ ydb/core/tx/schemeshard/olap/ttl/validator.cpp | 8 ++++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp index 850151c51dd3..0409fad44a3c 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp @@ -271,12 +271,6 @@ class TAlterColumnTable: public TSubOperation { return result; } - const bool hasTiering = Transaction.GetAlterColumnTable().GetAlterTtlSettings().GetEnabled().TiersSize(); - if (hasTiering && HasAppData() && !AppDataVerified().FeatureFlags.GetEnableTieringInColumnShard()) { - result->SetError(NKikimrScheme::StatusPreconditionFailed, "Tiering functionality is disabled for OLAP tables"); - return result; - } - const TString& parentPathStr = Transaction.GetWorkingDir(); const TString& name = Transaction.HasAlterColumnTable() ? Transaction.GetAlterColumnTable().GetName() : Transaction.GetAlterTable().GetName(); LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp index e4a3e2b5d389..bb9b0230aada 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -70,6 +70,14 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif errors.AddError(errStr); return false; } + if (!AppDataVerified().FeatureFlags.GetEnableTieringInColumnShard()) { + for (const auto& tier : ttl.GetTiers()) { + if (tier.HasEvictToExternalStorage()) { + errors.AddError(NKikimrScheme::StatusPreconditionFailed, "Tiering functionality is disabled for OLAP tables"); + return false; + } + } + } { bool correct = false; if (column->GetKeyOrder() && *column->GetKeyOrder() == 0) { From f729dfa71ce6d86fbc2710d4d4082fce9956dad3 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Fri, 29 Nov 2024 09:45:38 +0000 Subject: [PATCH 05/12] fix build --- ydb/core/tx/schemeshard/common/validation.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/common/validation.cpp b/ydb/core/tx/schemeshard/common/validation.cpp index fdbe79d264ea..5467fae3d330 100644 --- a/ydb/core/tx/schemeshard/common/validation.cpp +++ b/ydb/core/tx/schemeshard/common/validation.cpp @@ -59,7 +59,7 @@ bool TTTLValidator::ValidateUnit(const NScheme::TTypeInfo columnType, NKikimrSch } bool TTTLValidator::ValidateTiers(const google::protobuf::RepeatedPtrField& tiers, TString& errStr) { - for (ui64 i = 0; i < tiers.size(); ++i) { + for (i64 i = 0; i < tiers.size(); ++i) { const auto& tier = tiers[i]; if (!tier.HasApplyAfterSeconds()) { errStr = TStringBuilder() << "Tier " << i << ": missing ApplyAfterSeconds"; @@ -73,7 +73,7 @@ bool TTTLValidator::ValidateTiers(const google::protobuf::RepeatedPtrField Date: Fri, 29 Nov 2024 11:08:05 +0000 Subject: [PATCH 06/12] fix build --- ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp index ba658e23fef5..4cdbefacb452 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp +++ b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl_utility.cpp @@ -8,9 +8,9 @@ using namespace NSchemeShard; Y_UNIT_TEST_SUITE(TSchemeShardTTLUtility) { void TestValidateTiers(const std::vector& tiers, const TConclusionStatus& expectedResult) { - NKikimrSchemeOp::TTTLSettings::TEnabled input; + google::protobuf::RepeatedPtrField input; for (const auto& tier : tiers) { - *input.AddTiers() = tier; + input.Add()->CopyFrom(tier); } TString error; From 39423f0f3007170b42d356cb9f927f0b4d059c82 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Fri, 29 Nov 2024 16:33:31 +0000 Subject: [PATCH 07/12] fix tests --- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 6 +- ydb/public/lib/experimental/ydb_logstore.cpp | 2 +- ydb/public/sdk/cpp/client/ydb_table/table.cpp | 61 ++++++++++++------- ydb/public/sdk/cpp/client/ydb_table/table.h | 8 +-- 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index 5545ce39eda3..d3dd052a1f11 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -474,7 +474,7 @@ Y_UNIT_TEST_SUITE(TOlap) { ColumnName: "timestamp" ColumnUnit: UNIT_AUTO Tiers: { - EvictAfterSeconds: 360 + ApplyAfterSeconds: 360 StorageName: "Tier1" } } @@ -498,7 +498,7 @@ Y_UNIT_TEST_SUITE(TOlap) { ColumnName: "timestamp" ColumnUnit: UNIT_AUTO Tiers: { - EvictAfterSeconds: 3600000000 + ApplyAfterSeconds: 3600000000 StorageName: "Tier1" } } @@ -649,7 +649,7 @@ Y_UNIT_TEST_SUITE(TOlap) { ColumnName: "timestamp" ColumnUnit: UNIT_AUTO Tiers: { - EvictAfterSeconds: 3600000000 + ApplyAfterSeconds: 3600000000 StorageName: "Tier1" } } diff --git a/ydb/public/lib/experimental/ydb_logstore.cpp b/ydb/public/lib/experimental/ydb_logstore.cpp index bbfe436889a1..510f50bb3f70 100644 --- a/ydb/public/lib/experimental/ydb_logstore.cpp +++ b/ydb/public/lib/experimental/ydb_logstore.cpp @@ -16,7 +16,7 @@ namespace NYdb { namespace NLogStore { TMaybe TtlSettingsFromProto(const Ydb::Table::TtlSettings& proto) { - if (auto settings = TTtlSettings::DeserializeFromProto(proto)) { + if (auto settings = TTtlSettings::FromProto(proto)) { return *settings; } return Nothing(); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 4cde4085bfc7..b228be106340 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -327,7 +327,7 @@ class TTableDescription::TImpl { } // ttl settings - if (auto ttlSettings = TTtlSettings::DeserializeFromProto(proto.ttl_settings())) { + if (auto ttlSettings = TTtlSettings::FromProto(proto.ttl_settings())) { TtlSettings_ = std::move(*ttlSettings); } @@ -2918,8 +2918,8 @@ bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription& TTtlTierSettings::TTtlTierSettings(TDuration evictionDelay, const TAction& action) : ApplyAfter_(evictionDelay) - , Action_(action) { -} + , Action_(action) +{ } TTtlTierSettings::TTtlTierSettings(const Ydb::Table::TtlTier& tier) : ApplyAfter_(TDuration::Seconds(tier.apply_after_seconds())) { @@ -3083,14 +3083,12 @@ const TValueSinceUnixEpochModeSettings& TTtlSettings::GetValueSinceUnixEpoch() c return std::get(Mode_); } -std::optional TTtlSettings::DeserializeFromProto(const Ydb::Table::TtlSettings& proto) { - TDuration legacyExpireAfter = TDuration::Max(); +std::optional TTtlSettings::FromProto(const Ydb::Table::TtlSettings& proto) { + TVector tiers; for (const auto& tier : proto.tiers()) { - if (tier.has_delete_()) { - legacyExpireAfter = TDuration::Seconds(tier.apply_after_seconds()); - break; - } + tiers.emplace_back(tier); } + TDuration legacyExpireAfter = GetExpireAfterFrom(tiers).value_or(TDuration::Max()); switch(proto.mode_case()) { case Ydb::Table::TtlSettings::kDateTypeColumn: @@ -3098,9 +3096,12 @@ std::optional TTtlSettings::DeserializeFromProto(const Ydb::Table: case Ydb::Table::TtlSettings::kValueSinceUnixEpoch: return TTtlSettings(proto.value_since_unix_epoch(), proto.run_interval_seconds()); case Ydb::Table::TtlSettings::kDateTypeColumnV1: - return TTtlSettings(TDateTypeColumnModeSettings(proto.date_type_column_v1().column_name(), legacyExpireAfter), proto.run_interval_seconds()); + return TTtlSettings( + TDateTypeColumnModeSettings(proto.date_type_column_v1().column_name(), legacyExpireAfter), tiers, proto.run_interval_seconds()); case Ydb::Table::TtlSettings::kValueSinceUnixEpochV1: - return TTtlSettings(TValueSinceUnixEpochModeSettings(proto.value_since_unix_epoch_v1().column_name(), TProtoAccessor::FromProto(proto.value_since_unix_epoch_v1().column_unit()), legacyExpireAfter), proto.run_interval_seconds()); + return TTtlSettings(TValueSinceUnixEpochModeSettings(proto.value_since_unix_epoch_v1().column_name(), + TProtoAccessor::FromProto(proto.value_since_unix_epoch_v1().column_unit()), legacyExpireAfter), + tiers, proto.run_interval_seconds()); case Ydb::Table::TtlSettings::MODE_NOT_SET: return std::nullopt; break; @@ -3108,17 +3109,29 @@ std::optional TTtlSettings::DeserializeFromProto(const Ydb::Table: } void TTtlSettings::SerializeTo(Ydb::Table::TtlSettings& proto) const { - switch (GetMode()) { - case EMode::DateTypeColumn: - GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column_v1()); - break; - case EMode::ValueSinceUnixEpoch: - GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch_v1()); - break; - } + if (Tiers_.size() == 1 && std::holds_alternative(Tiers_.back().GetAction())) { + // serialize DELETE-only TTL to legacy format for backwards-compatibility + switch (GetMode()) { + case EMode::DateTypeColumn: + GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column()); + break; + case EMode::ValueSinceUnixEpoch: + GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch()); + break; + } + } else { + switch (GetMode()) { + case EMode::DateTypeColumn: + GetDateTypeColumn().SerializeTo(*proto.mutable_date_type_column_v1()); + break; + case EMode::ValueSinceUnixEpoch: + GetValueSinceUnixEpoch().SerializeTo(*proto.mutable_value_since_unix_epoch_v1()); + break; + } - for (const auto& tier : Tiers_) { - tier.SerializeTo(*proto.add_tiers()); + for (const auto& tier : Tiers_) { + tier.SerializeTo(*proto.add_tiers()); + } } if (RunInterval_) { @@ -3156,7 +3169,11 @@ std::optional TTtlSettings::GetExpireAfterFrom(const TVector& tiers, ui32 runIntervalSeconds) + : Mode_(std::move(mode)) + , Tiers_(tiers) + , RunInterval_(TDuration::Seconds(runIntervalSeconds)) +{} TAlterTtlSettings::EAction TAlterTtlSettings::GetAction() const { return static_cast(Action_.index()); diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index e010337106ac..621ca89d1fd6 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -516,16 +516,16 @@ class TTtlSettings { explicit TTtlSettings(const TString& columnName, const TVector& tiers); explicit TTtlSettings(const TString& columnName, const TDuration& expireAfter); const TDateTypeColumnModeSettings& GetDateTypeColumn() const; - // Deprecated. Use DeserializeFromProto() + // Deprecated. Use FromProto() explicit TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds); explicit TTtlSettings(const TString& columnName, EUnit columnUnit, const TVector& tiers); explicit TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter); const TValueSinceUnixEpochModeSettings& GetValueSinceUnixEpoch() const; - // Deprecated. Use DeserializeFromProto() + // Deprecated. Use FromProto() explicit TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds); - static std::optional DeserializeFromProto(const Ydb::Table::TtlSettings& proto); + static std::optional FromProto(const Ydb::Table::TtlSettings& proto); void SerializeTo(Ydb::Table::TtlSettings& proto) const; EMode GetMode() const; @@ -536,7 +536,7 @@ class TTtlSettings { std::optional GetExpireAfter() const; private: - explicit TTtlSettings(TMode mode, ui32 runIntervalSeconds); + explicit TTtlSettings(TMode mode, const TVector& tiers, ui32 runIntervalSeconds); static std::optional GetExpireAfterFrom(const TVector& tiers); private: From c4c3e55e9697602231d27b9a3f2f3a2b76aaf028 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Sat, 30 Nov 2024 17:19:59 +0000 Subject: [PATCH 08/12] fix test --- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index d3dd052a1f11..72428f9deec5 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -475,7 +475,9 @@ Y_UNIT_TEST_SUITE(TOlap) { ColumnUnit: UNIT_AUTO Tiers: { ApplyAfterSeconds: 360 - StorageName: "Tier1" + EvictToExternalStorage { + StorageName: "Tier1" + } } } } @@ -499,7 +501,9 @@ Y_UNIT_TEST_SUITE(TOlap) { ColumnUnit: UNIT_AUTO Tiers: { ApplyAfterSeconds: 3600000000 - StorageName: "Tier1" + EvictToExternalStorage { + StorageName: "Tier1" + } } } } @@ -650,7 +654,9 @@ Y_UNIT_TEST_SUITE(TOlap) { ColumnUnit: UNIT_AUTO Tiers: { ApplyAfterSeconds: 3600000000 - StorageName: "Tier1" + EvictToExternalStorage { + StorageName: "Tier1" + } } } } From 615a38729a64d35ec60556d894fc5753a1837dfb Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Sat, 30 Nov 2024 19:23:02 +0000 Subject: [PATCH 09/12] fix test --- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index 72428f9deec5..2b4658cd3b30 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -426,7 +426,9 @@ Y_UNIT_TEST_SUITE(TOlap) { Y_UNIT_TEST(CreateTableTtl) { TTestBasicRuntime runtime; - TTestEnv env(runtime); + TTestEnvOptions options; + options.EnableTieringInColumnShard(true); + TTestEnv env(runtime, options); ui64 txId = 100; TestCreateOlapStore(runtime, ++txId, "/MyRoot", defaultStoreSchema); From ade6979b2d275f6a2ee950e2b9f9732e4ba5e802 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Fri, 1 Nov 2024 13:55:18 +0000 Subject: [PATCH 10/12] squashed --- ydb/core/kqp/ut/common/columnshard.cpp | 18 +- .../columnshard/blobs_action/tier/storage.cpp | 2 +- ydb/core/tx/columnshard/columnshard.cpp | 10 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 21 +- ydb/core/tx/columnshard/columnshard_impl.h | 3 - .../tx/columnshard/hooks/abstract/abstract.h | 10 +- ydb/core/tx/columnshard/tables_manager.cpp | 19 +- ydb/core/tx/columnshard/tables_manager.h | 6 +- .../test_helper/columnshard_ut_common.cpp | 42 ++- .../test_helper/columnshard_ut_common.h | 6 +- .../columnshard/test_helper/controllers.cpp | 8 +- .../tx/columnshard/test_helper/controllers.h | 16 +- ydb/core/tx/tiering/common.h | 5 + ydb/core/tx/tiering/external_data.cpp | 24 -- ydb/core/tx/tiering/external_data.h | 25 -- ydb/core/tx/tiering/fetcher.h | 226 +++++++++++++++ ydb/core/tx/tiering/manager.cpp | 274 +++++++++++++----- ydb/core/tx/tiering/manager.h | 90 ++++-- ydb/core/tx/tiering/snapshot.cpp | 36 --- ydb/core/tx/tiering/snapshot.h | 25 -- ydb/core/tx/tiering/tier/behaviour.cpp | 33 --- ydb/core/tx/tiering/tier/behaviour.h | 20 -- ydb/core/tx/tiering/tier/checker.cpp | 50 ---- ydb/core/tx/tiering/tier/checker.h | 38 --- ydb/core/tx/tiering/tier/initializer.cpp | 36 --- ydb/core/tx/tiering/tier/initializer.h | 15 - ydb/core/tx/tiering/tier/manager.cpp | 77 ----- ydb/core/tx/tiering/tier/manager.h | 19 -- ydb/core/tx/tiering/tier/object.cpp | 104 ++++--- ydb/core/tx/tiering/tier/object.h | 61 +--- ydb/core/tx/tiering/tier/ss_checker.cpp | 26 -- ydb/core/tx/tiering/tier/ss_checker.h | 68 ----- ydb/core/tx/tiering/tier/ss_fetcher.cpp | 77 ----- ydb/core/tx/tiering/tier/ss_fetcher.h | 79 ----- ydb/core/tx/tiering/tier/ya.make | 9 - ydb/core/tx/tiering/ya.make | 4 +- 36 files changed, 645 insertions(+), 937 deletions(-) delete mode 100644 ydb/core/tx/tiering/external_data.cpp delete mode 100644 ydb/core/tx/tiering/external_data.h create mode 100644 ydb/core/tx/tiering/fetcher.h delete mode 100644 ydb/core/tx/tiering/snapshot.cpp delete mode 100644 ydb/core/tx/tiering/snapshot.h delete mode 100644 ydb/core/tx/tiering/tier/behaviour.cpp delete mode 100644 ydb/core/tx/tiering/tier/behaviour.h delete mode 100644 ydb/core/tx/tiering/tier/checker.cpp delete mode 100644 ydb/core/tx/tiering/tier/checker.h delete mode 100644 ydb/core/tx/tiering/tier/initializer.cpp delete mode 100644 ydb/core/tx/tiering/tier/initializer.h delete mode 100644 ydb/core/tx/tiering/tier/manager.cpp delete mode 100644 ydb/core/tx/tiering/tier/manager.h delete mode 100644 ydb/core/tx/tiering/tier/ss_checker.cpp delete mode 100644 ydb/core/tx/tiering/tier/ss_checker.h delete mode 100644 ydb/core/tx/tiering/tier/ss_fetcher.cpp delete mode 100644 ydb/core/tx/tiering/tier/ss_fetcher.h diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 28ffe5f3e7dd..2383a075aa6e 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -5,6 +5,8 @@ #include #include +#include + extern "C" { #include } @@ -39,6 +41,9 @@ namespace NKqp { if (!kikimrSettings.FeatureFlags.HasEnableTieringInColumnShard()) { kikimrSettings.SetEnableTieringInColumnShard(true); } + if (!kikimrSettings.FeatureFlags.HasEnableExternalDataSources()) { + kikimrSettings.SetEnableExternalDataSources(true); + } Kikimr = std::make_unique(kikimrSettings); TableClient = std::make_unique(Kikimr->GetTableClient()); @@ -64,7 +69,18 @@ namespace NKqp { } void TTestHelper::CreateTier(const TString& tierName) { - auto result = GetSession().ExecuteSchemeQuery("CREATE OBJECT " + tierName + " (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName(tierName) + "`").GetValueSync(); + auto result = GetSession().ExecuteSchemeQuery(R"( + UPSERT OBJECT `accessKey` (TYPE SECRET) WITH (value = `secretAccessKey`); + UPSERT OBJECT `secretKey` (TYPE SECRET) WITH (value = `fakeSecret`); + CREATE EXTERNAL DATA SOURCE `)" + tierName + R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="fake/fake", + AUTH_METHOD="AWS", + AWS_ACCESS_KEY_ID_SECRET_NAME="accessKey", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="secretKey", + AWS_REGION="ru-central1" + ); + )").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp index cf842edbd411..9188630e10a3 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp @@ -55,7 +55,7 @@ void TOperator::DoStartGCAction(const std::shared_ptr& action) c void TOperator::InitNewExternalOperator(const NColumnShard::NTiers::TManager* tierManager) { NKikimrSchemeOp::TS3Settings settings; - if (tierManager) { + if (tierManager && tierManager->IsReady()) { settings = tierManager->GetS3Settings(); } else { settings.SetEndpoint("nowhere"); diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 0d5c5766c270..f9273888ab11 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -100,8 +100,8 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { ctx.Send(selfActorId, new TEvPrivate::TEvTieringModified); }); Tiers->Start(Tiers); - if (!NMetadata::NProvider::TServiceOperator::IsEnabled()) { - Tiers->TakeConfigs(NYDBTest::TControllers::GetColumnShardController()->GetFallbackTiersSnapshot(), nullptr); + if (!NYDBTest::TControllers::GetColumnShardController()->GetOverrideTierConfigs().empty()) { + Send(SelfId(), new TEvPrivate::TEvTieringModified()); } BackgroundSessionsManager = std::make_shared( std::make_shared(selfActorId, (NOlap::TTabletId)TabletID(), *this)); @@ -120,6 +120,12 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { } void TColumnShard::Handle(TEvPrivate::TEvTieringModified::TPtr& /*ev*/, const TActorContext& /*ctx*/) { + if (const auto& tiersSnapshot = NYDBTest::TControllers::GetColumnShardController()->GetOverrideTierConfigs(); !tiersSnapshot.empty()) { + for (const auto& [id, tier] : tiersSnapshot) { + Tiers->UpdateTierConfig(tier, id, false); + } + } + OnTieringModified(); NYDBTest::TControllers::GetColumnShardController()->OnTieringModified(Tiers); } diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 3070a3bb41d1..2042f64db570 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -46,7 +46,6 @@ #include #include #include -#include #include #include @@ -398,7 +397,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl tableVerProto.SetSchemaPresetId(preset.GetId()); if (TablesManager.RegisterSchemaPreset(preset, db)) { - TablesManager.AddSchemaVersion(tableProto.GetSchemaPreset().GetId(), version, tableProto.GetSchemaPreset().GetSchema(), db, Tiers); + TablesManager.AddSchemaVersion(tableProto.GetSchemaPreset().GetId(), version, tableProto.GetSchemaPreset().GetSchema(), db); } } else { Y_ABORT_UNLESS(tableProto.HasSchema(), "Tables has either schema or preset"); @@ -423,7 +422,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl tableVerProto.SetSchemaPresetVersionAdj(tableProto.GetSchemaPresetVersionAdj()); - TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db, Tiers); + TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db); InsertTable->RegisterPathInfo(pathId); Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, TablesManager.GetTables().size()); @@ -447,7 +446,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP std::optional schema; if (alterProto.HasSchemaPreset()) { tableVerProto.SetSchemaPresetId(alterProto.GetSchemaPreset().GetId()); - TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db, Tiers); + TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db); } else if (alterProto.HasSchema()) { schema = alterProto.GetSchema(); } @@ -464,7 +463,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP ActivateTiering(pathId, usedTiers); tableVerProto.SetSchemaPresetVersionAdj(alterProto.GetSchemaPresetVersionAdj()); - TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db, Tiers); + TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db); } void TColumnShard::RunDropTable(const NKikimrTxColumnShard::TDropTable& dropProto, const NOlap::TSnapshot& version, @@ -501,7 +500,7 @@ void TColumnShard::RunAlterStore(const NKikimrTxColumnShard::TAlterStore& proto, if (!TablesManager.HasPreset(presetProto.GetId())) { continue; // we don't update presets that we don't use } - TablesManager.AddSchemaVersion(presetProto.GetId(), version, presetProto.GetSchema(), db, Tiers); + TablesManager.AddSchemaVersion(presetProto.GetId(), version, presetProto.GetSchema(), db); } } @@ -1546,12 +1545,6 @@ void TColumnShard::Handle(NOlap::NBlobOperations::NEvents::TEvDeleteSharedBlobs: Execute(new TTxRemoveSharedBlobs(this, blobs, NActors::ActorIdFromProto(ev->Get()->Record.GetSourceActorId()), ev->Get()->Record.GetStorageId()), ctx); } -void TColumnShard::Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { - Y_ABORT_UNLESS(Tiers); - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "TEvRefreshSubscriberData")("snapshot", ev->Get()->GetSnapshot()->SerializeToString()); - Tiers->TakeConfigs(ev->Get()->GetSnapshot(), nullptr); -} - void TColumnShard::ActivateTiering(const ui64 pathId, const THashSet& usedTiers) { AFL_VERIFY(Tiers); if (!usedTiers.empty()) { @@ -1580,10 +1573,6 @@ void TColumnShard::Enqueue(STFUNC_SIG) { void TColumnShard::OnTieringModified(const std::optional pathId) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnTieringModified")("path_id", pathId); - if (!Tiers->IsReady()) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_reload_tiering")("reason", "manager_not_ready")("path_id", pathId); - return; - } StoragesManager->OnTieringModified(Tiers); if (TablesManager.HasPrimaryIndex()) { if (pathId) { diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 844a3e20b80c..d21551bc8d35 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -261,7 +261,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void Handle(TEvPrivate::TEvPingSnapshotsUsage::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvWriteIndex::TPtr& ev, const TActorContext& ctx); - void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev); void Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvWriteDraft::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvGarbageCollectionFinished::TPtr& ev, const TActorContext& ctx); @@ -395,8 +394,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa "self_id", SelfId())("ev", ev->GetTypeName()); TRACE_EVENT(NKikimrServices::TX_COLUMNSHARD); switch (ev->GetTypeRewrite()) { - hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); - HFunc(TEvTxProcessing::TEvReadSet, Handle); HFunc(TEvTxProcessing::TEvReadSetAck, Handle); diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index 28470aca8bda..8c7b94f8f66b 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -1,10 +1,10 @@ #pragma once #include +#include #include #include -#include -#include +#include #include #include @@ -284,10 +284,8 @@ class ICSController { return nullptr; } - virtual NMetadata::NFetcher::ISnapshot::TPtr GetFallbackTiersSnapshot() const { - static std::shared_ptr result = - std::make_shared(TInstant::Now()); - return result; + virtual THashMap GetOverrideTierConfigs() const { + return {}; } virtual void OnSwitchToWork(const ui64 tabletId) { diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 1706563ffe65..5e1ca8738d03 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -273,8 +273,8 @@ bool TTablesManager::RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIc return true; } -void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, - NIceDb::TNiceDb& db, std::shared_ptr& manager) { +void TTablesManager::AddSchemaVersion( + const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db) { Y_ABORT_UNLESS(SchemaPresetsIds.contains(presetId)); TSchemaPreset::TSchemaPresetVersionInfo versionInfo; @@ -298,9 +298,7 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho for (auto&& i : Tables) { PrimaryIndex->RegisterTable(i.first); } - if (manager->IsReady()) { - PrimaryIndex->OnTieringModified(Ttl); - } + PrimaryIndex->OnTieringModified(Ttl); } else { PrimaryIndex->RegisterSchemaVersion(version, NOlap::IColumnEngine::TSchemaInitializationData(versionInfo)); } @@ -312,15 +310,12 @@ std::unique_ptr TTablesManager::CreateAddShar } void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, - const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, const std::optional& schema, - NIceDb::TNiceDb& db, std::shared_ptr& manager) { + const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, const std::optional& schema, NIceDb::TNiceDb& db) { auto it = Tables.find(pathId); AFL_VERIFY(it != Tables.end()); auto& table = it->second; - bool isTtlModified = false; if (versionInfo.HasTtlSettings()) { - isTtlModified = true; const auto& ttlSettings = versionInfo.GetTtlSettings(); if (ttlSettings.HasEnabled()) { NOlap::TTiering deserializedTtl; @@ -341,11 +336,11 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& } else { Y_ABORT_UNLESS(SchemaPresetsIds.contains(fakePreset.GetId())); } - AddSchemaVersion(fakePreset.GetId(), version, *schema, db, manager); + AddSchemaVersion(fakePreset.GetId(), version, *schema, db); } - if (isTtlModified) { - if (PrimaryIndex && manager->IsReady()) { + if (versionInfo.HasTtlSettings()) { + if (PrimaryIndex) { if (auto findTtl = Ttl.FindPtr(pathId)) { PrimaryIndex->OnTieringModified(*findTtl, pathId); } else { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index f44ca4c872ce..05f1872c9234 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -244,10 +244,10 @@ class TTablesManager { void RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db); bool RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIceDb::TNiceDb& db); - void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, - NIceDb::TNiceDb& db, std::shared_ptr& manager); + void AddSchemaVersion( + const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db); void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, - const std::optional& schema, NIceDb::TNiceDb& db, std::shared_ptr& manager); + const std::optional& schema, NIceDb::TNiceDb& db); bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); [[nodiscard]] std::unique_ptr CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index e8d016f45d17..1317b2a65c38 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -1,18 +1,18 @@ #include "columnshard_ut_common.h" #include "shard_reader.h" -#include +#include +#include +#include +#include #include #include +#include #include #include -#include - -#include -#include -#include -#include +#include #include + #include namespace NKikimr::NTxUT { @@ -48,8 +48,8 @@ void TTester::Setup(TTestActorRuntime& runtime) { runtime.UpdateCurrentTime(TInstant::Now()); } -void ProvideTieringSnapshot(TTestBasicRuntime& runtime, const TActorId& sender, NMetadata::NFetcher::ISnapshot::TPtr snapshot) { - auto event = std::make_unique(snapshot); +void RefreshTiering(TTestBasicRuntime& runtime, const TActorId& sender) { + auto event = std::make_unique(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, event.release()); } @@ -371,27 +371,21 @@ TSerializedTableRange MakeTestRange(std::pair range, bool inclusiveF TConstArrayRef(cellsTo), inclusiveTo); } -NMetadata::NFetcher::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpecials& specials) { - std::unique_ptr cs(new NColumnShard::NTiers::TTiersSnapshot(Now())); +THashMap TTestSchema::BuildSnapshot(const TTableSpecials& specials) { if (specials.Tiers.empty()) { - return cs; + return {}; } + THashMap tiers; for (auto&& tier : specials.Tiers) { { - NKikimrSchemeOp::TStorageTierConfig cProto; - cProto.SetName(tier.Name); - *cProto.MutableObjectStorage() = tier.S3; - if (tier.Codec) { - cProto.MutableCompression()->SetCodec(tier.GetCodecId()); - } - if (tier.CompressionLevel) { - cProto.MutableCompression()->SetLevel(*tier.CompressionLevel); - } - NColumnShard::NTiers::TTierConfig tConfig(tier.Name, cProto); - cs->MutableTierConfigs().emplace(tConfig.GetTierName(), tConfig); + NKikimrSchemeOp::TCompressionOptions compressionProto; + compressionProto.SetCodec(tier.GetCodecId()); + compressionProto.SetLevel(*tier.CompressionLevel); + NColumnShard::NTiers::TTierConfig tConfig(tier.S3, compressionProto); + tiers.emplace(tier.Name, tConfig); } } - return cs; + return tiers; } void TTestSchema::InitSchema(const std::vector& columns, const std::vector& pk, diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 20418f574cf9..4cc86fa9d642 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -358,7 +359,7 @@ struct TTestSchema { return out; } - static NMetadata::NFetcher::ISnapshot::TPtr BuildSnapshot(const TTableSpecials& specials); + static THashMap BuildSnapshot(const TTableSpecials& specials); static TString CommitTxBody(ui64, const std::vector& writeIds) { NKikimrTxColumnShard::TCommitTxBody proto; @@ -403,8 +404,9 @@ struct TTestSchema { } }; +void RefreshTiering(TTestBasicRuntime& runtime, const TActorId& sender); + bool ProposeSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, const TString& txBody, NOlap::TSnapshot snap); -void ProvideTieringSnapshot(TTestBasicRuntime& runtime, const TActorId& sender, NMetadata::NFetcher::ISnapshot::TPtr snapshot); void PlanSchemaTx(TTestBasicRuntime& runtime, const TActorId& sender, NOlap::TSnapshot snap); void PlanWriteTx(TTestBasicRuntime& runtime, const TActorId& sender, NOlap::TSnapshot snap, bool waitResult = true); diff --git a/ydb/core/tx/columnshard/test_helper/controllers.cpp b/ydb/core/tx/columnshard/test_helper/controllers.cpp index 997a700d901b..6a5a43773de5 100644 --- a/ydb/core/tx/columnshard/test_helper/controllers.cpp +++ b/ydb/core/tx/columnshard/test_helper/controllers.cpp @@ -10,13 +10,13 @@ void TWaitCompactionController::OnTieringModified(const std::shared_ptr tiers) { + OverrideTiers = std::move(tiers); ui32 startCount = TiersModificationsCount; - NTxUT::ProvideTieringSnapshot(runtime, tabletActorId, snapshot); + NTxUT::RefreshTiering(runtime, tabletActorId); while (TiersModificationsCount == startCount) { runtime.SimulateSleep(TDuration::Seconds(1)); } } - } \ No newline at end of file diff --git a/ydb/core/tx/columnshard/test_helper/controllers.h b/ydb/core/tx/columnshard/test_helper/controllers.h index 8886700d452f..b1e8e881d174 100644 --- a/ydb/core/tx/columnshard/test_helper/controllers.h +++ b/ydb/core/tx/columnshard/test_helper/controllers.h @@ -1,6 +1,7 @@ #pragma once -#include #include +#include +#include namespace NKikimr::NOlap { @@ -8,7 +9,7 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController { private: using TBase = NKikimr::NYDBTest::ICSController; TAtomicCounter ExportsFinishedCount = 0; - NMetadata::NFetcher::ISnapshot::TPtr CurrentConfig; + THashMap OverrideTiers; ui32 TiersModificationsCount = 0; YDB_READONLY(TAtomicCounter, StatisticsUsageCount, 0); YDB_READONLY(TAtomicCounter, MaxValueUsageCount, 0); @@ -59,14 +60,11 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController { virtual void OnMaxValueUsage() override { MaxValueUsageCount.Inc(); } - void SetTiersSnapshot(TTestBasicRuntime& runtime, const TActorId& tabletActorId, const NMetadata::NFetcher::ISnapshot::TPtr& snapshot); + void OverrideTierConfigs( + TTestBasicRuntime& runtime, const TActorId& tabletActorId, THashMap tiers); - virtual NMetadata::NFetcher::ISnapshot::TPtr GetFallbackTiersSnapshot() const override { - if (CurrentConfig) { - return CurrentConfig; - } else { - return TBase::GetFallbackTiersSnapshot(); - } + THashMap GetOverrideTierConfigs() const override { + return OverrideTiers; } }; diff --git a/ydb/core/tx/tiering/common.h b/ydb/core/tx/tiering/common.h index 1eb341b78b80..cd7fa14b82bc 100644 --- a/ydb/core/tx/tiering/common.h +++ b/ydb/core/tx/tiering/common.h @@ -13,6 +13,11 @@ enum EEvents { EvSSFetchingProblem, EvTimeout, EvTiersManagerReadyForUsage, + EvWatchSchemeObject, + EvNotifySchemeObjectUpdated, + EvNotifySchemeObjectDeleted, + EvSchemeObjectResulutionFailed, + EvListTieredStoragesResult, EvEnd }; diff --git a/ydb/core/tx/tiering/external_data.cpp b/ydb/core/tx/tiering/external_data.cpp deleted file mode 100644 index 812215b01ff3..000000000000 --- a/ydb/core/tx/tiering/external_data.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "external_data.h" - -#include -#include - -#include -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -void TSnapshotConstructor::EnrichSnapshotData(ISnapshot::TPtr original, NMetadata::NFetcher::ISnapshotAcceptorController::TPtr controller) const { - controller->OnSnapshotEnriched(original); -} - -TSnapshotConstructor::TSnapshotConstructor() { -} - -std::vector TSnapshotConstructor::DoGetManagers() const { - return { TTierConfig::GetBehaviour() }; -} - -} diff --git a/ydb/core/tx/tiering/external_data.h b/ydb/core/tx/tiering/external_data.h deleted file mode 100644 index 456ad1ff59e3..000000000000 --- a/ydb/core/tx/tiering/external_data.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include "snapshot.h" - -#include -#include -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TSnapshotConstructor: public NMetadata::NFetcher::TSnapshotsFetcher { -private: - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TBaseActor = TActor; - using ISnapshot = NMetadata::NFetcher::ISnapshot; -protected: - virtual std::vector DoGetManagers() const override; -public: - virtual void EnrichSnapshotData(ISnapshot::TPtr original, NMetadata::NFetcher::ISnapshotAcceptorController::TPtr controller) const override; - - TSnapshotConstructor(); -}; - -} diff --git a/ydb/core/tx/tiering/fetcher.h b/ydb/core/tx/tiering/fetcher.h new file mode 100644 index 000000000000..5a7baa048316 --- /dev/null +++ b/ydb/core/tx/tiering/fetcher.h @@ -0,0 +1,226 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include + +namespace NKikimr::NColumnShard { + +namespace NTiers { + +class TEvWatchSchemeObject: public TEventLocal { +private: + YDB_READONLY_DEF(std::vector, ObjectIds); + +public: + TEvWatchSchemeObject(std::vector names) + : ObjectIds(std::move(names)) { + } +}; + +class TEvNotifySchemeObjectUpdated: public TEventLocal { +private: + YDB_READONLY_DEF(TString, ObjectId); + YDB_READONLY_DEF(NKikimrSchemeOp::TPathDescription, Description); + +public: + TEvNotifySchemeObjectUpdated(const TString& path, NKikimrSchemeOp::TPathDescription description) + : ObjectId(path) + , Description(std::move(description)) { + } +}; + +class TEvNotifySchemeObjectDeleted: public TEventLocal { +private: + YDB_READONLY_DEF(TString, ObjectId); + +public: + TEvNotifySchemeObjectDeleted(TString name) + : ObjectId(std::move(name)) { + } +}; + +class TEvSchemeObjectResolutionFailed: public TEventLocal { +public: + enum EReason { + NOT_FOUND = 0, + LOOKUP_ERROR = 1 + }; + +private: + YDB_READONLY_DEF(TString, ObjectId); + YDB_READONLY_DEF(EReason, Reason); + +public: + TEvSchemeObjectResolutionFailed(TString name, const EReason reason) + : ObjectId(std::move(name)) + , Reason(reason) { + } +}; + +} // namespace NTiers + +class TSchemeObjectWatcher: public TActorBootstrapped { +private: + TActorId Owner; + THashSet WatchedPathIds; + +private: + void WatchObjects(const std::vector& objectIds) { + RequestPaths(objectIds); + } + + THolder BuildSchemeCacheNavigateRequest( + const TVector>& paths, TIntrusiveConstPtr userToken) { + auto request = MakeHolder(); + request->DatabaseName = AppDataVerified().TenantName; + if (userToken && !userToken->GetSerializedToken().empty()) { + request->UserToken = userToken; + } + + for (const auto& pathComponents : paths) { + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.RequestType = NSchemeCache::TSchemeCacheNavigate::TEntry::ERequestType::ByPath; + entry.ShowPrivatePath = true; + entry.Path = pathComponents; + } + + return request; + } + + void RequestPaths(const std::vector& paths) { + TVector> splitPaths; + for (const TString& path : paths) { + splitPaths.emplace_back(SplitPath(path)); + } + + auto event = + BuildSchemeCacheNavigateRequest(std::move(splitPaths), MakeIntrusive(BUILTIN_ACL_METADATA, TVector{})); + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(event.Release()), IEventHandle::FlagTrackDelivery); + } + + void OnPathFetched(const TVector pathComponents, const TPathId& pathId) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "path_fetched")("path", JoinPath(pathComponents)); + if (WatchedPathIds.emplace(pathId).second) { + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchPathId(pathId), IEventHandle::FlagTrackDelivery); + } else { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "skip_watch_path_id")("reason", "already_subscribed")("path", JoinPath(pathComponents)); + } + } + + void OnPathNotFound(const TVector& path) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "path_not_found")("path", JoinPath(path)); + OnObjectResolutionFailure(path, NTiers::TEvSchemeObjectResolutionFailed::EReason::NOT_FOUND); + } + + void OnLookupError(const TVector& path) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "lookup_error")("path", JoinPath(path)); + OnObjectResolutionFailure(path, NTiers::TEvSchemeObjectResolutionFailed::EReason::LOOKUP_ERROR); + } + + void OnObjectResolutionFailure(const TVector& pathComponents, const NTiers::TEvSchemeObjectResolutionFailed::EReason reason) { + Send(Owner, new NTiers::TEvSchemeObjectResolutionFailed(JoinPath(pathComponents), reason)); + } + + void OnObjectFetched(const NKikimrSchemeOp::TPathDescription& description, const TString& path) { + Send(Owner, new NTiers::TEvNotifySchemeObjectUpdated(path, description)); + } + + void OnObjectDeleted(const TString& path, const TPathId& pathId) { + AFL_VERIFY(WatchedPathIds.erase(pathId)); + Send(Owner, new NTiers::TEvNotifySchemeObjectDeleted(path)); + } + +public: + TSchemeObjectWatcher(TActorId owner) + : Owner(owner) { + } + + STATEFN(StateMain) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); + hFunc(TEvTxProxySchemeCache::TEvWatchNotifyDeleted, Handle); + hFunc(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable, Handle); + hFunc(NTiers::TEvWatchSchemeObject, Handle); + hFunc(NActors::TEvents::TEvPoison, Handle); + hFunc(NActors::TEvents::TEvUndelivered, Handle); + default: + break; + } + } + + void Bootstrap() { + Become(&TSchemeObjectWatcher::StateMain); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const NSchemeCache::TSchemeCacheNavigate* result = ev->Get()->Request.Get(); + for (auto entry : result->ResultSet) { + switch (entry.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: + OnPathFetched(entry.Path, entry.TableId.PathId); + break; + + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + OnPathNotFound(entry.Path); + break; + + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + OnLookupError(entry.Path); + break; + + case NSchemeCache::TSchemeCacheNavigate::EStatus::AccessDenied: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotTable: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathNotPath: + case NSchemeCache::TSchemeCacheNavigate::EStatus::TableCreationNotComplete: + case NSchemeCache::TSchemeCacheNavigate::EStatus::Unknown: + AFL_VERIFY(false)("entry", entry.ToString()); + } + } + } + + void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "object_fetched")("path", ev->Get()->Path); + const auto& describeResult = *ev->Get()->Result; + OnObjectFetched(describeResult.GetPathDescription(), describeResult.GetPath()); + } + + void Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev) { + const auto& record = ev->Get(); + const TString name = TString(ExtractBase(record->Path)); + const TString storageDir = TString(ExtractParent(record->Path)); + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "object_deleted")("path", record->Path); + OnObjectDeleted(record->Path, record->PathId); + } + + void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUnavailable::TPtr& ev) { + const auto& record = ev->Get(); + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "object_unavailable")("path", record->Path); + } + + void Handle(NTiers::TEvWatchSchemeObject::TPtr& ev) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "watch_scheme_objects")( + "names", JoinStrings(ev->Get()->GetObjectIds().begin(), ev->Get()->GetObjectIds().end(), ",")); + WatchObjects(ev->Get()->GetObjectIds()); + } + + void Handle(NActors::TEvents::TEvPoison::TPtr& /*ev*/) { + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvWatchRemove()); + PassAway(); + } + + void Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { + AFL_WARN(NKikimrServices::TX_TIERING)("error", "event_undelivered_to_scheme_cache")("reason", ev->Get()->Reason); + } +}; + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/tiering/manager.cpp b/ydb/core/tx/tiering/manager.cpp index dbe06df7e0d4..1fc5948b5d8e 100644 --- a/ydb/core/tx/tiering/manager.cpp +++ b/ydb/core/tx/tiering/manager.cpp @@ -1,109 +1,139 @@ #include "common.h" #include "manager.h" -#include "external_data.h" #include +#include + +#include #include +#include + namespace NKikimr::NColumnShard { class TTiersManager::TActor: public TActorBootstrapped { private: std::shared_ptr Owner; NMetadata::NFetcher::ISnapshotsFetcher::TPtr SecretsFetcher; - std::shared_ptr SecretsSnapshot; - std::shared_ptr ConfigsSnapshot; + TActorId TieredStorageFetcher; + +private: TActorId GetExternalDataActorId() const { return NMetadata::NProvider::MakeServiceId(SelfId().NodeId()); } -public: - TActor(std::shared_ptr owner) - : Owner(owner) - , SecretsFetcher(std::make_shared()) - { - } - ~TActor() { - Owner->Stop(false); + void ScheduleRetryWatchObjects(std::unique_ptr ev) const { + AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiers_manager")("event", "retry_watch_objects"); + constexpr static const TDuration RetryInterval = TDuration::Seconds(1); + ActorContext().Schedule(RetryInterval, std::make_unique(SelfId(), TieredStorageFetcher, ev.release())); } STATEFN(StateMain) { switch (ev->GetTypeRewrite()) { hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); hFunc(NActors::TEvents::TEvPoison, Handle); + hFunc(NTiers::TEvNotifySchemeObjectUpdated, Handle); + hFunc(NTiers::TEvNotifySchemeObjectDeleted, Handle); + hFunc(NTiers::TEvSchemeObjectResolutionFailed, Handle); + hFunc(NTiers::TEvWatchSchemeObject, Handle); default: break; } } - void Bootstrap() { - Become(&TThis::StateMain); - AFL_INFO(NKikimrServices::TX_TIERING)("event", "start_subscribing_metadata"); - Send(GetExternalDataActorId(), new NMetadata::NProvider::TEvSubscribeExternal(Owner->GetExternalDataManipulation())); - Send(GetExternalDataActorId(), new NMetadata::NProvider::TEvSubscribeExternal(SecretsFetcher)); - } - void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { auto snapshot = ev->Get()->GetSnapshot(); - if (auto configs = std::dynamic_pointer_cast(snapshot)) { - AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "TEvRefreshSubscriberData")("snapshot", "configs"); - ConfigsSnapshot = configs; - if (SecretsSnapshot) { - Owner->TakeConfigs(ConfigsSnapshot, SecretsSnapshot); - } else { - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Waiting secrets for update at tablet " << Owner->TabletId; - } - } else if (auto secrets = std::dynamic_pointer_cast(snapshot)) { + if (auto secrets = std::dynamic_pointer_cast(snapshot)) { AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "TEvRefreshSubscriberData")("snapshot", "secrets"); - SecretsSnapshot = secrets; - if (ConfigsSnapshot) { - Owner->TakeConfigs(ConfigsSnapshot, SecretsSnapshot); - } else { - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Waiting configs for update at tablet " << Owner->TabletId; - } + Owner->UpdateSecretsSnapshot(secrets); } else { - Y_ABORT_UNLESS(false, "unexpected behaviour"); + Y_ABORT_UNLESS(false, "unexpected snapshot"); } } void Handle(NActors::TEvents::TEvPoison::TPtr& /*ev*/) { - Send(GetExternalDataActorId(), new NMetadata::NProvider::TEvUnsubscribeExternal(Owner->GetExternalDataManipulation())); Send(GetExternalDataActorId(), new NMetadata::NProvider::TEvUnsubscribeExternal(SecretsFetcher)); PassAway(); } + + void Handle(NTiers::TEvNotifySchemeObjectUpdated::TPtr& ev) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiering_manager")("event", "object_updated")("path", ev->Get()->GetObjectId())("type", "TIERED_STORAGE"); + const TString& objectId = ev->Get()->GetObjectId(); + const auto& description = ev->Get()->GetDescription(); + if (description.GetSelf().GetPathType() == NKikimrSchemeOp::EPathTypeExternalDataSource) { + NTiers::TTierConfig tier; + AFL_VERIFY(tier.DeserializeFromProto(description.GetExternalDataSourceDescription())); + Owner->UpdateTierConfig(tier, objectId); + } else { + AFL_WARN(NKikimrServices::TX_TIERING)("error", "invalid_object_type")("type", static_cast(description.GetSelf().GetPathType()))("path", objectId); + } + } + + void Handle(NTiers::TEvNotifySchemeObjectDeleted::TPtr& ev) { + AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiering_manager")("event", "object_deleted")("name", ev->Get()->GetObjectId())("type", "TIERED_STORAGE"); + } + + void Handle(NTiers::TEvSchemeObjectResolutionFailed::TPtr& ev) { + const TString objectId = ev->Get()->GetObjectId(); + switch (ev->Get()->GetReason()) { + case NTiers::TEvSchemeObjectResolutionFailed::NOT_FOUND: + AFL_WARN(NKikimrServices::TX_TIERING)("event", "object_not_found")("name", objectId)("type", "TIERING_RULE"); + break; + case NTiers::TEvSchemeObjectResolutionFailed::LOOKUP_ERROR: + ScheduleRetryWatchObjects(std::make_unique(std::vector({ objectId }))); + break; + } + } + + void Handle(NTiers::TEvWatchSchemeObject::TPtr& ev) { + Send(TieredStorageFetcher, ev->Release()); + } + +public: + TActor(std::shared_ptr owner) + : Owner(owner) + , SecretsFetcher(std::make_shared()) + { + } + + void Bootstrap() { + AFL_INFO(NKikimrServices::TX_TIERING)("event", "start_subscribing_metadata"); + TieredStorageFetcher = Register(new TSchemeObjectWatcher(SelfId())); + Send(GetExternalDataActorId(), new NMetadata::NProvider::TEvSubscribeExternal(SecretsFetcher)); + Become(&TThis::StateMain); + } + + ~TActor() { + Owner->Stop(false); + } }; namespace NTiers { TManager& TManager::Restart(const TTierConfig& config, std::shared_ptr secrets) { - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Restarting tier '" << GetTierName() << "' at tablet " << TabletId; - if (Config.IsSame(config)) { - return *this; - } + ALS_DEBUG(NKikimrServices::TX_TIERING) << "Restarting tier '" << TierName << "' at tablet " << TabletId; Stop(); - Config = config; - Start(secrets); + Start(config, secrets); return *this; } bool TManager::Stop() { S3Settings.reset(); - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Tier '" << GetTierName() << "' stopped at tablet " << TabletId; + ALS_DEBUG(NKikimrServices::TX_TIERING) << "Tier '" << TierName << "' stopped at tablet " << TabletId; return true; } -bool TManager::Start(std::shared_ptr secrets) { - AFL_VERIFY(!S3Settings)("tier", GetTierName())("event", "already started"); - S3Settings = Config.GetPatchedConfig(secrets); - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Tier '" << GetTierName() << "' started at tablet " << TabletId; +bool TManager::Start(const TTierConfig& config, std::shared_ptr secrets) { + AFL_VERIFY(!S3Settings)("tier", TierName)("event", "already started"); + S3Settings = config.GetPatchedConfig(secrets); + ALS_DEBUG(NKikimrServices::TX_TIERING) << "Tier '" << TierName << "' started at tablet " << TabletId; return true; } -TManager::TManager(const ui64 tabletId, const NActors::TActorId& tabletActorId, const TTierConfig& config) +TManager::TManager(const ui64 tabletId, const NActors::TActorId& tabletActorId, const TString& tierName) : TabletId(tabletId) , TabletActorId(tabletActorId) - , Config(config) -{ + , TierName(tierName) { } NArrow::NSerialization::TSerializerContainer ConvertCompression(const NKikimrSchemeOp::TCompressionOptions& compressionProto) { @@ -119,40 +149,68 @@ NArrow::NSerialization::TSerializerContainer ConvertCompression(const NKikimrSch } } -void TTiersManager::TakeConfigs(NMetadata::NFetcher::ISnapshot::TPtr snapshotExt, std::shared_ptr secrets) { - ALS_INFO(NKikimrServices::TX_TIERING) << "Take configs:" - << (snapshotExt ? " snapshots" : "") << (secrets ? " secrets" : "") << " at tablet " << TabletId; +TTiersManager::TTierRefGuard::TTierRefGuard(const TString& tierName, TTiersManager& owner) + : TierName(tierName) + , Owner(&owner) { + if (!Owner->TierRefCount.contains(TierName)) { + Owner->RegisterTier(tierName); + } + ++Owner->TierRefCount[TierName]; +} - auto snapshotPtr = std::dynamic_pointer_cast(snapshotExt); - Y_ABORT_UNLESS(snapshotPtr); - Snapshot = snapshotExt; - Secrets = secrets; - auto& snapshot = *snapshotPtr; - for (auto itSelf = Managers.begin(); itSelf != Managers.end(); ) { - auto it = snapshot.GetTierConfigs().find(itSelf->first); - if (it == snapshot.GetTierConfigs().end()) { - itSelf->second.Stop(); - itSelf = Managers.erase(itSelf); - } else { - itSelf->second.Restart(it->second, Secrets); - ++itSelf; +TTiersManager::TTierRefGuard::~TTierRefGuard() { + if (Owner) { + auto findTier = Owner->TierRefCount.FindPtr(TierName); + AFL_VERIFY(findTier); + AFL_VERIFY(*findTier); + --*findTier; + if (!*findTier) { + AFL_VERIFY(Owner->TierRefCount.erase(TierName)); + Owner->UnregisterTier(TierName); } } - for (auto&& i : snapshot.GetTierConfigs()) { - auto tierName = i.second.GetTierName(); - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Take config for tier '" << tierName << "' at tablet " << TabletId; - if (Managers.contains(tierName)) { - ALS_DEBUG(NKikimrServices::TX_TIERING) << "Ignore tier '" << tierName << "' at tablet " << TabletId; - continue; +} + +void TTiersManager::OnConfigsUpdated(bool notifyShard) { + for (auto& [tierName, manager] : Managers) { + auto* findTierConfig = TierConfigs.FindPtr(tierName); + if (Secrets && findTierConfig) { + if (manager.IsReady()) { + manager.Restart(*findTierConfig, Secrets); + } else { + manager.Start(*findTierConfig, Secrets); + } + } else { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "skip_tier_manager_reloading")("tier", tierName)("has_secrets", !!Secrets)( + "found_tier_config", !!findTierConfig); } - NTiers::TManager localManager(TabletId, TabletActorId, i.second); - auto itManager = Managers.emplace(tierName, std::move(localManager)).first; - itManager->second.Start(Secrets); } - if (ShardCallback && TlsActivationContext) { + if (notifyShard && ShardCallback && TlsActivationContext) { ShardCallback(TActivationContext::AsActorContext()); } + + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "configs_updated")("configs", DebugString()); +} + +void TTiersManager::RegisterTier(const TString& name) { + auto emplaced = Managers.emplace(name, NTiers::TManager(TabletId, TabletActorId, name)); + AFL_VERIFY(emplaced.second); + + auto* findConfig = TierConfigs.FindPtr(name); + if (Secrets && findConfig) { + emplaced.first->second.Start(*findConfig, Secrets); + } else { + AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "skip_tier_manager_start")("tier", name)("has_secrets", !!Secrets)( + "found_tier_config", !!findConfig); + } +} + +void TTiersManager::UnregisterTier(const TString& name) { + auto findManager = Managers.find(name); + AFL_VERIFY(findManager != Managers.end()); + findManager->second.Stop(); + Managers.erase(findManager); } TTiersManager& TTiersManager::Start(std::shared_ptr ownerPtr) { @@ -185,11 +243,37 @@ const NTiers::TManager* TTiersManager::GetManagerOptional(const TString& tierId) } } -NMetadata::NFetcher::ISnapshotsFetcher::TPtr TTiersManager::GetExternalDataManipulation() const { - if (!ExternalDataManipulation) { - ExternalDataManipulation = std::make_shared(); +void TTiersManager::EnablePathId(const ui64 pathId, const THashSet& usedTiers) { + AFL_VERIFY(Actor)("error", "tiers_manager_is_not_started"); + auto& tierRefs = UsedTiers[pathId]; + tierRefs.clear(); + for (const TString& tierName : usedTiers) { + tierRefs.emplace_back(tierName, *this); + if (!TierConfigs.contains(tierName)) { + const auto& actorContext = NActors::TActivationContext::AsActorContext(); + AFL_VERIFY(&actorContext)("error", "no_actor_context"); + actorContext.Send(Actor->SelfId(), new NTiers::TEvWatchSchemeObject({ tierName })); + } } - return ExternalDataManipulation; + OnConfigsUpdated(false); +} + +void TTiersManager::DisablePathId(const ui64 pathId) { + UsedTiers.erase(pathId); + OnConfigsUpdated(false); +} + +void TTiersManager::UpdateSecretsSnapshot(std::shared_ptr secrets) { + AFL_INFO(NKikimrServices::TX_TIERING)("event", "update_secrets")("tablet", TabletId); + AFL_VERIFY(secrets); + Secrets = secrets; + OnConfigsUpdated(); +} + +void TTiersManager::UpdateTierConfig(const NTiers::TTierConfig& config, const TString& tierName, const bool notifyShard) { + AFL_INFO(NKikimrServices::TX_TIERING)("event", "update_tier_config")("name", tierName)("tablet", TabletId); + TierConfigs[tierName] = config; + OnConfigsUpdated(notifyShard); } TActorId TTiersManager::GetActorId() const { @@ -200,4 +284,36 @@ TActorId TTiersManager::GetActorId() const { } } +TString TTiersManager::DebugString() { + TStringBuilder sb; + sb << "TIERS="; + if (TierConfigs) { + sb << "{"; + for (const auto& [name, config] : TierConfigs) { + sb << name << ";"; + } + sb << "}"; + } + sb << ";USED_TIERS="; + { + sb << "{"; + for (const auto& [pathId, tiers] : UsedTiers) { + sb << pathId << ":{"; + for (const auto& tierRef : tiers) { + sb << tierRef.GetTierName() << ";"; + } + sb << "}"; + } + sb << "}"; + } + sb << ";SECRETS="; + if (Secrets) { + sb << "{"; + for (const auto& [name, config] : Secrets->GetSecrets()) { + sb << name.SerializeToString() << ";"; + } + sb << "}"; + } + return sb; +} } diff --git a/ydb/core/tx/tiering/manager.h b/ydb/core/tx/tiering/manager.h index d0a464e40e61..6644b95f5d6f 100644 --- a/ydb/core/tx/tiering/manager.h +++ b/ydb/core/tx/tiering/manager.h @@ -1,18 +1,16 @@ #pragma once -#include "external_data.h" +#include "common.h" #include "abstract/manager.h" -#include -#include +#include +#include #include #include #include #include -#include - #include namespace NKikimr::NColumnShard { @@ -24,9 +22,9 @@ NArrow::NSerialization::TSerializerContainer ConvertCompression(const NKikimrSch class TManager { private: ui64 TabletId = 0; - YDB_READONLY_DEF(NActors::TActorId, TabletActorId); - YDB_READONLY_DEF(TTierConfig, Config); - YDB_READONLY_DEF(NActors::TActorId, StorageActorId); + NActors::TActorId TabletActorId; + TString TierName; + NActors::TActorId StorageActorId; std::optional S3Settings; public: const NKikimrSchemeOp::TS3Settings& GetS3Settings() const { @@ -34,60 +32,88 @@ class TManager { return *S3Settings; } - TManager(const ui64 tabletId, const NActors::TActorId& tabletActorId, const TTierConfig& config); + TManager(const ui64 tabletId, const NActors::TActorId& tabletActorId, const TString& tierName); + bool IsReady() const { + return !!S3Settings; + } TManager& Restart(const TTierConfig& config, std::shared_ptr secrets); bool Stop(); - bool Start(std::shared_ptr secrets); - - TString GetTierName() const { - return GetConfig().GetTierName(); - } + bool Start(const TTierConfig& config, std::shared_ptr secrets); }; } class TTiersManager: public ITiersManager { +private: + friend class TTierRef; + class TTierRefGuard: public TMoveOnly { + private: + YDB_READONLY_DEF(TString, TierName); + TTiersManager* Owner; + + public: + TTierRefGuard(const TString& tierName, TTiersManager& owner); + ~TTierRefGuard(); + + TTierRefGuard(TTierRefGuard&& other) + : TierName(other.TierName) + , Owner(other.Owner) { + other.Owner = nullptr; + } + TTierRefGuard& operator=(TTierRefGuard&& other) { + std::swap(Owner, other.Owner); + std::swap(TierName, other.TierName); + return *this; + } + }; + private: class TActor; + friend class TActor; using TManagers = std::map; + ui64 TabletId = 0; const TActorId TabletActorId; std::function ShardCallback; - TActor* Actor = nullptr; + IActor* Actor = nullptr; TManagers Managers; - std::shared_ptr Secrets; - NMetadata::NFetcher::ISnapshot::TPtr Snapshot; - mutable NMetadata::NFetcher::ISnapshotsFetcher::TPtr ExternalDataManipulation; + using TTierRefCount = THashMap; + using TTierRefsByPathId = THashMap>; + YDB_READONLY_DEF(TTierRefCount, TierRefCount); + YDB_READONLY_DEF(TTierRefsByPathId, UsedTiers); + + using TTierByName = THashMap; + YDB_READONLY_DEF(TTierByName, TierConfigs); + YDB_READONLY_DEF(std::shared_ptr, Secrets); + +private: + void OnConfigsUpdated(bool notifyShard = true); + void RegisterTier(const TString& name); + void UnregisterTier(const TString& name); public: - TTiersManager(const ui64 tabletId, const TActorId& tabletActorId, - std::function shardCallback = {}) + TTiersManager(const ui64 tabletId, const TActorId& tabletActorId, std::function shardCallback = {}) : TabletId(tabletId) , TabletActorId(tabletActorId) , ShardCallback(shardCallback) - { + , Secrets(std::make_shared(TInstant::Zero())) { } TActorId GetActorId() const; - void TakeConfigs(NMetadata::NFetcher::ISnapshot::TPtr snapshot, std::shared_ptr secrets); - void EnablePathId(const ui64 /*pathId*/, const THashSet& /*usedTiers*/) { - } - void DisablePathId(const ui64 /*pathId*/) { - } + void EnablePathId(const ui64 pathId, const THashSet& usedTiers); + void DisablePathId(const ui64 pathId); - bool IsReady() const { - return !!Snapshot; - } + void UpdateSecretsSnapshot(std::shared_ptr secrets); + void UpdateTierConfig(const NTiers::TTierConfig& config, const TString& tierName, const bool notifyShard = true); + + TString DebugString(); TTiersManager& Start(std::shared_ptr ownerPtr); TTiersManager& Stop(const bool needStopActor); virtual const std::map& GetManagers() const override { - AFL_VERIFY(IsReady()); return Managers; } virtual const NTiers::TManager* GetManagerOptional(const TString& tierId) const override; - NMetadata::NFetcher::ISnapshotsFetcher::TPtr GetExternalDataManipulation() const; - }; } diff --git a/ydb/core/tx/tiering/snapshot.cpp b/ydb/core/tx/tiering/snapshot.cpp deleted file mode 100644 index d64987b5b62e..000000000000 --- a/ydb/core/tx/tiering/snapshot.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "snapshot.h" - -#include - -#include -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -bool TTiersSnapshot::DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawDataResult) { - Y_ABORT_UNLESS(rawDataResult.result_sets().size() == 1); - ParseSnapshotObjects(rawDataResult.result_sets()[0], [this](TTierConfig&& s) {TierConfigs.emplace(s.GetTierName(), s); }); - return true; -} - -std::optional TTiersSnapshot::GetTierById(const TString& tierName) const { - auto it = TierConfigs.find(tierName); - if (it == TierConfigs.end()) { - return {}; - } else { - return it->second; - } -} - -TString NTiers::TTiersSnapshot::DoSerializeToString() const { - NJson::TJsonValue result = NJson::JSON_MAP; - auto& jsonTiers = result.InsertValue("tiers", NJson::JSON_MAP); - for (auto&& i : TierConfigs) { - jsonTiers.InsertValue(i.first, i.second.GetDebugJson()); - } - return result.GetStringRobust(); -} - -} diff --git a/ydb/core/tx/tiering/snapshot.h b/ydb/core/tx/tiering/snapshot.h deleted file mode 100644 index 4eea9921c56a..000000000000 --- a/ydb/core/tx/tiering/snapshot.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include -#include -#include - -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTiersSnapshot: public NMetadata::NFetcher::ISnapshot { -private: - using TBase = NMetadata::NFetcher::ISnapshot; - using TConfigsMap = TMap; - YDB_ACCESSOR_DEF(TConfigsMap, TierConfigs); -protected: - virtual bool DoDeserializeFromResultSet(const Ydb::Table::ExecuteQueryResult& rawData) override; - virtual TString DoSerializeToString() const override; -public: - std::optional GetTierById(const TString& tierName) const; - using TBase::TBase; -}; - -} diff --git a/ydb/core/tx/tiering/tier/behaviour.cpp b/ydb/core/tx/tiering/tier/behaviour.cpp deleted file mode 100644 index 01b3d13290f1..000000000000 --- a/ydb/core/tx/tiering/tier/behaviour.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "behaviour.h" -#include "manager.h" -#include "initializer.h" - -#include - -#include -#include - -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -TTierConfigBehaviour::TFactory::TRegistrator TTierConfigBehaviour::Registrator(TTierConfig::GetTypeId()); - -TString TTierConfigBehaviour::GetInternalStorageTablePath() const { - return "tiering/tiers"; -} - -NMetadata::NModifications::IOperationsManager::TPtr TTierConfigBehaviour::ConstructOperationsManager() const { - return std::make_shared(); -} - -NMetadata::NInitializer::IInitializationBehaviour::TPtr TTierConfigBehaviour::ConstructInitializer() const { - return std::make_shared(); -} - -TString TTierConfigBehaviour::GetTypeId() const { - return TTierConfig::GetTypeId(); -} - -} diff --git a/ydb/core/tx/tiering/tier/behaviour.h b/ydb/core/tx/tiering/tier/behaviour.h deleted file mode 100644 index fd231708fffe..000000000000 --- a/ydb/core/tx/tiering/tier/behaviour.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "object.h" -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTierConfigBehaviour: public NMetadata::TClassBehaviour { -private: - static TFactory::TRegistrator Registrator; -protected: - virtual std::shared_ptr ConstructInitializer() const override; - virtual std::shared_ptr ConstructOperationsManager() const override; - - virtual TString GetInternalStorageTablePath() const override; - virtual TString GetTypeId() const override; - -}; - -} diff --git a/ydb/core/tx/tiering/tier/checker.cpp b/ydb/core/tx/tiering/tier/checker.cpp deleted file mode 100644 index 32ec8da0e7b4..000000000000 --- a/ydb/core/tx/tiering/tier/checker.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "checker.h" - -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -void TTierPreparationActor::StartChecker() { - if (!Secrets) { - return; - } - auto g = PassAwayGuard(); - for (auto&& tier : Objects) { - if (!Secrets->CheckSecretAccess(tier.GetAccessKey(), Context.GetExternalData().GetUserToken())) { - Controller->OnPreparationProblem("no access for secret: " + tier.GetAccessKey().DebugString()); - return; - } else if (!Secrets->CheckSecretAccess(tier.GetSecretKey(), Context.GetExternalData().GetUserToken())) { - Controller->OnPreparationProblem("no access for secret: " + tier.GetSecretKey().DebugString()); - return; - } - } - Controller->OnPreparationFinished(std::move(Objects)); -} - -void TTierPreparationActor::Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { - if (auto snapshot = ev->Get()->GetSnapshotPtrAs()) { - Secrets = snapshot; - } else { - Y_ABORT_UNLESS(false); - } - StartChecker(); -} - -void TTierPreparationActor::Bootstrap() { - Become(&TThis::StateMain); - Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), - new NMetadata::NProvider::TEvAskSnapshot(std::make_shared())); -} - -TTierPreparationActor::TTierPreparationActor(std::vector&& objects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const NMetadata::NModifications::IOperationsManager::TInternalModificationContext& context) - : Objects(std::move(objects)) - , Controller(controller) - , Context(context) -{ - -} - -} diff --git a/ydb/core/tx/tiering/tier/checker.h b/ydb/core/tx/tiering/tier/checker.h deleted file mode 100644 index 109c9de208aa..000000000000 --- a/ydb/core/tx/tiering/tier/checker.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once -#include "object.h" - -#include -#include - -#include -#include -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTierPreparationActor: public NActors::TActorBootstrapped { -private: - std::vector Objects; - NMetadata::NModifications::IAlterPreparationController::TPtr Controller; - NMetadata::NModifications::IOperationsManager::TInternalModificationContext Context; - std::shared_ptr Secrets; - void StartChecker(); -protected: - void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev); -public: - STATEFN(StateMain) { - switch (ev->GetTypeRewrite()) { - hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); - default: - break; - } - } - void Bootstrap(); - - TTierPreparationActor(std::vector&& objects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const NMetadata::NModifications::IOperationsManager::TInternalModificationContext& context); -}; - -} diff --git a/ydb/core/tx/tiering/tier/initializer.cpp b/ydb/core/tx/tiering/tier/initializer.cpp deleted file mode 100644 index 9bf517856a9a..000000000000 --- a/ydb/core/tx/tiering/tier/initializer.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "initializer.h" -#include "object.h" - -namespace NKikimr::NColumnShard::NTiers { - -TVector TTiersInitializer::BuildModifiers() const { - TVector result; - { - Ydb::Table::CreateTableRequest request; - request.set_session_id(""); - request.set_path(TTierConfig::GetBehaviour()->GetStorageTablePath()); - request.add_primary_key("tierName"); - { - auto& column = *request.add_columns(); - column.set_name("tierName"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); - } - { - auto& column = *request.add_columns(); - column.set_name("tierConfig"); - column.mutable_type()->mutable_optional_type()->mutable_item()->set_type_id(Ydb::Type::UTF8); - } - result.emplace_back(new NMetadata::NInitializer::TGenericTableModifier(request, "create")); - auto hRequest = TTierConfig::AddHistoryTableScheme(request); - result.emplace_back(new NMetadata::NInitializer::TGenericTableModifier(hRequest, "create_history")); - } - result.emplace_back(NMetadata::NInitializer::TACLModifierConstructor::GetReadOnlyModifier(TTierConfig::GetBehaviour()->GetStorageTablePath(), "acl")); - result.emplace_back(NMetadata::NInitializer::TACLModifierConstructor::GetReadOnlyModifier(TTierConfig::GetBehaviour()->GetStorageHistoryTablePath(), "acl_history")); - return result; -} - -void TTiersInitializer::DoPrepare(NMetadata::NInitializer::IInitializerInput::TPtr controller) const { - controller->OnPreparationFinished(BuildModifiers()); -} - -} diff --git a/ydb/core/tx/tiering/tier/initializer.h b/ydb/core/tx/tiering/tier/initializer.h deleted file mode 100644 index e3d99fffea0a..000000000000 --- a/ydb/core/tx/tiering/tier/initializer.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once -#include -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTiersInitializer: public NMetadata::NInitializer::IInitializationBehaviour { -protected: - TVector BuildModifiers() const; - virtual void DoPrepare(NMetadata::NInitializer::IInitializerInput::TPtr controller) const override; -public: -}; - -} diff --git a/ydb/core/tx/tiering/tier/manager.cpp b/ydb/core/tx/tiering/tier/manager.cpp deleted file mode 100644 index 8d60219624b4..000000000000 --- a/ydb/core/tx/tiering/tier/manager.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include "manager.h" -#include "initializer.h" -#include "checker.h" - -namespace NKikimr::NColumnShard::NTiers { - -NMetadata::NModifications::TOperationParsingResult TTiersManager::DoBuildPatchFromSettings( - const NYql::TObjectSettingsImpl& settings, - TInternalModificationContext& context) const -{ - if (HasAppData() && !AppDataVerified().FeatureFlags.GetEnableTieringInColumnShard()) { - return TConclusionStatus::Fail("Tiering functionality is disabled for OLAP tables."); - } - - NMetadata::NInternal::TTableRecord result; - result.SetColumn(TTierConfig::TDecoder::TierName, NMetadata::NInternal::TYDBValue::Utf8(settings.GetObjectId())); - if (settings.GetObjectId().StartsWith("$") || settings.GetObjectId().StartsWith("_")) { - return TConclusionStatus::Fail("tier name cannot start with '$', '_' characters"); - } - { - auto fConfig = settings.GetFeaturesExtractor().Extract(TTierConfig::TDecoder::TierConfig); - if (fConfig) { - NKikimrSchemeOp::TStorageTierConfig proto; - if (!::google::protobuf::TextFormat::ParseFromString(*fConfig, &proto)) { - return TConclusionStatus::Fail("incorrect proto format"); - } else if (proto.HasObjectStorage()) { - TString defaultUserId; - if (context.GetExternalData().GetUserToken()) { - defaultUserId = context.GetExternalData().GetUserToken()->GetUserSID(); - } - - if (proto.GetObjectStorage().HasSecretableAccessKey()) { - auto accessKey = NMetadata::NSecret::TSecretIdOrValue::DeserializeFromProto(proto.GetObjectStorage().GetSecretableAccessKey(), defaultUserId); - if (!accessKey) { - return TConclusionStatus::Fail("AccessKey description is incorrect"); - } - *proto.MutableObjectStorage()->MutableSecretableAccessKey() = accessKey->SerializeToProto(); - } else if (proto.GetObjectStorage().HasAccessKey()) { - auto accessKey = NMetadata::NSecret::TSecretIdOrValue::DeserializeFromString(proto.GetObjectStorage().GetAccessKey(), defaultUserId); - if (!accessKey) { - return TConclusionStatus::Fail("AccessKey is incorrect: " + proto.GetObjectStorage().GetAccessKey() + " for userId: " + defaultUserId); - } - *proto.MutableObjectStorage()->MutableAccessKey() = accessKey->SerializeToString(); - } else { - return TConclusionStatus::Fail("AccessKey not configured"); - } - - if (proto.GetObjectStorage().HasSecretableSecretKey()) { - auto secretKey = NMetadata::NSecret::TSecretIdOrValue::DeserializeFromProto(proto.GetObjectStorage().GetSecretableSecretKey(), defaultUserId); - if (!secretKey) { - return TConclusionStatus::Fail("SecretKey description is incorrect"); - } - *proto.MutableObjectStorage()->MutableSecretableSecretKey() = secretKey->SerializeToProto(); - } else if (proto.GetObjectStorage().HasSecretKey()) { - auto secretKey = NMetadata::NSecret::TSecretIdOrValue::DeserializeFromString(proto.GetObjectStorage().GetSecretKey(), defaultUserId); - if (!secretKey) { - return TConclusionStatus::Fail("SecretKey is incorrect"); - } - *proto.MutableObjectStorage()->MutableSecretKey() = secretKey->SerializeToString(); - } else { - return TConclusionStatus::Fail("SecretKey not configured"); - } - } - result.SetColumn(TTierConfig::TDecoder::TierConfig, NMetadata::NInternal::TYDBValue::Utf8(proto.DebugString())); - } - } - return result; -} - -void TTiersManager::DoPrepareObjectsBeforeModification(std::vector&& patchedObjects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const TInternalModificationContext& context, const NMetadata::NModifications::TAlterOperationContext& /*alterContext*/) const -{ - TActivationContext::Register(new TTierPreparationActor(std::move(patchedObjects), controller, context)); -} - -} diff --git a/ydb/core/tx/tiering/tier/manager.h b/ydb/core/tx/tiering/tier/manager.h deleted file mode 100644 index 7d8626c8c36c..000000000000 --- a/ydb/core/tx/tiering/tier/manager.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once -#include "object.h" - -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TTiersManager: public NMetadata::NModifications::TGenericOperationsManager { -protected: - virtual void DoPrepareObjectsBeforeModification(std::vector&& patchedObjects, - NMetadata::NModifications::IAlterPreparationController::TPtr controller, - const TInternalModificationContext& context, const NMetadata::NModifications::TAlterOperationContext& alterContext) const override; - - virtual NMetadata::NModifications::TOperationParsingResult DoBuildPatchFromSettings(const NYql::TObjectSettingsImpl& settings, - TInternalModificationContext& context) const override; -public: -}; - -} diff --git a/ydb/core/tx/tiering/tier/object.cpp b/ydb/core/tx/tiering/tier/object.cpp index e5045a3ee887..389ad0193c15 100644 --- a/ydb/core/tx/tiering/tier/object.cpp +++ b/ydb/core/tx/tiering/tier/object.cpp @@ -1,62 +1,86 @@ #include "object.h" -#include "behaviour.h" - -#include #include #include #include #include +#include namespace NKikimr::NColumnShard::NTiers { -NMetadata::IClassBehaviour::TPtr TTierConfig::GetBehaviour() { - static std::shared_ptr result = std::make_shared(); - return result; +NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig( + std::shared_ptr secrets) const { + auto config = ProtoConfig; + if (secrets) { + if (!secrets->GetSecretValue(GetAccessKey(), *config.MutableAccessKey())) { + AFL_ERROR(NKikimrServices::TX_TIERING)("error", "cannot_read_access_key")("secret", GetAccessKey().DebugString()); + } + if (!secrets->GetSecretValue(GetSecretKey(), *config.MutableSecretKey())) { + AFL_ERROR(NKikimrServices::TX_TIERING)("error", "cannot_read_secret_key")("secret", GetSecretKey().DebugString()); + } + } + return config; } -NJson::TJsonValue TTierConfig::GetDebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue(TDecoder::TierName, TierName); - NProtobufJson::Proto2Json(ProtoConfig, result.InsertValue(TDecoder::TierConfig, NJson::JSON_MAP)); - return result; -} +bool TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourceDescription& proto) { + if (!proto.GetAuth().HasAws()) { + return false; + } -bool TTierConfig::IsSame(const TTierConfig& item) const { - return TierName == item.TierName && ProtoConfig.SerializeAsString() == item.ProtoConfig.SerializeAsString(); -} + // TODO fix secret owner + { + auto makeSecretId = [](const TStringBuf& secret) -> TString { + return NMetadata::NSecret::TSecretId("a", secret).SerializeToString(); // ... and here + }; + ProtoConfig.SetSecretKey(makeSecretId(proto.GetAuth().GetAws().GetAwsAccessKeyIdSecretName())); + ProtoConfig.SetAccessKey(makeSecretId(proto.GetAuth().GetAws().GetAwsSecretAccessKeySecretName())); + } -bool TTierConfig::DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r) { - if (!decoder.Read(decoder.GetTierNameIdx(), TierName, r)) { + NUri::TUri url; + if (url.Parse(proto.GetLocation()) != NUri::TState::EParsed::ParsedOK) { return false; } - if (!decoder.ReadDebugProto(decoder.GetTierConfigIdx(), ProtoConfig, r)) { - return false; + + switch (url.GetScheme()) { + case NUri::TScheme::SchemeEmpty: + break; + case NUri::TScheme::SchemeHTTP: + ProtoConfig.SetScheme(::NKikimrSchemeOp::TS3Settings_EScheme_HTTP); + break; + case NUri::TScheme::SchemeHTTPS: + ProtoConfig.SetScheme(::NKikimrSchemeOp::TS3Settings_EScheme_HTTPS); + break; + default: + return false; } - return ProtoConfig.HasObjectStorage(); -} -NMetadata::NInternal::TTableRecord TTierConfig::SerializeToRecord() const { - NMetadata::NInternal::TTableRecord result; - result.SetColumn(TDecoder::TierName, NMetadata::NInternal::TYDBValue::Utf8(TierName)); - result.SetColumn(TDecoder::TierConfig, NMetadata::NInternal::TYDBValue::Utf8(ProtoConfig.DebugString())); - return result; -} + { + TStringBuf endpoint; + TStringBuf bucket; -NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig( - std::shared_ptr secrets) const -{ - auto config = ProtoConfig.GetObjectStorage(); - if (secrets) { - if (!secrets->GetSecretValue(GetAccessKey(), *config.MutableAccessKey())) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot read access key secret for " << GetAccessKey().DebugString(); - } - if (!secrets->GetSecretValue(GetSecretKey(), *config.MutableSecretKey())) { - ALS_ERROR(NKikimrServices::TX_TIERING) << "cannot read secret key secret for " << GetSecretKey().DebugString(); + TStringBuf host = url.GetHost(); + TStringBuf path = url.GetField(NUri::TField::FieldPath); + if (path.StartsWith("/") && path.Size() > 1) { + bucket = path.Skip(1); + if (bucket.EndsWith("/")) { + bucket = bucket.Chop(1); + } + if (bucket.Contains("/")) { + return false; + } + endpoint = host; + } else { + if (!path.TrySplit('.', endpoint, bucket)) { + return false; + } } + + ProtoConfig.SetEndpoint(TString(endpoint)); + ProtoConfig.SetBucket(TString(bucket)); } - return config; + + return true; } NJson::TJsonValue TTierConfig::SerializeConfigToJson() const { @@ -65,4 +89,8 @@ NJson::TJsonValue TTierConfig::SerializeConfigToJson() const { return result; } +bool TTierConfig::IsSame(const TTierConfig& item) const { + return ProtoConfig.SerializeAsString() == item.ProtoConfig.SerializeAsString(); +} + } diff --git a/ydb/core/tx/tiering/tier/object.h b/ydb/core/tx/tiering/tier/object.h index 96bdfc490ac2..f162eeb91403 100644 --- a/ydb/core/tx/tiering/tier/object.h +++ b/ydb/core/tx/tiering/tier/object.h @@ -1,10 +1,7 @@ #pragma once +#include #include -#include -#include -#include -#include -#include + #include #include @@ -15,32 +12,24 @@ class TSnapshot; namespace NKikimr::NColumnShard::NTiers { -class TTierConfig: public NMetadata::NModifications::TObject { +class TTierConfig { private: - using TTierProto = NKikimrSchemeOp::TStorageTierConfig; - YDB_ACCESSOR_DEF(TString, TierName); - TTierProto ProtoConfig; -public: + using TTierProto = NKikimrSchemeOp::TS3Settings; + YDB_READONLY_DEF(TTierProto, ProtoConfig); + YDB_READONLY_DEF(NKikimrSchemeOp::TCompressionOptions, Compression); +public: TTierConfig() = default; - TTierConfig(const TString& tierName) - : TierName(tierName) { - + TTierConfig(const TTierProto& config, const NKikimrSchemeOp::TCompressionOptions& compression) + : ProtoConfig(config) + , Compression(compression) { } - TTierConfig(const TString& tierName, const TTierProto& config) - : TierName(tierName) - , ProtoConfig(config) - { - - } - - const NKikimrSchemeOp::TCompressionOptions& GetCompression() const { - return ProtoConfig.GetCompression(); - } + bool DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourceDescription& proto); NMetadata::NSecret::TSecretIdOrValue GetAccessKey() const { - auto accessKey = NMetadata::NSecret::TSecretIdOrValue::DeserializeFromOptional(ProtoConfig.GetObjectStorage().GetSecretableAccessKey(), ProtoConfig.GetObjectStorage().GetAccessKey()); + auto accessKey = + NMetadata::NSecret::TSecretIdOrValue::DeserializeFromOptional(ProtoConfig.GetSecretableAccessKey(), ProtoConfig.GetAccessKey()); if (!accessKey) { return NMetadata::NSecret::TSecretIdOrValue::BuildEmpty(); } @@ -48,7 +37,8 @@ class TTierConfig: public NMetadata::NModifications::TObject { } NMetadata::NSecret::TSecretIdOrValue GetSecretKey() const { - auto secretKey = NMetadata::NSecret::TSecretIdOrValue::DeserializeFromOptional(ProtoConfig.GetObjectStorage().GetSecretableSecretKey(), ProtoConfig.GetObjectStorage().GetSecretKey()); + auto secretKey = + NMetadata::NSecret::TSecretIdOrValue::DeserializeFromOptional(ProtoConfig.GetSecretableSecretKey(), ProtoConfig.GetSecretKey()); if (!secretKey) { return NMetadata::NSecret::TSecretIdOrValue::BuildEmpty(); } @@ -57,30 +47,9 @@ class TTierConfig: public NMetadata::NModifications::TObject { NJson::TJsonValue SerializeConfigToJson() const; - - static NMetadata::IClassBehaviour::TPtr GetBehaviour(); NKikimrSchemeOp::TS3Settings GetPatchedConfig(std::shared_ptr secrets) const; - class TDecoder: public NMetadata::NInternal::TDecoderBase { - private: - YDB_READONLY(i32, TierNameIdx, -1); - YDB_READONLY(i32, TierConfigIdx, -1); - public: - static inline const TString TierName = "tierName"; - static inline const TString TierConfig = "tierConfig"; - TDecoder(const Ydb::ResultSet& rawData) { - TierNameIdx = GetFieldIndex(rawData, TierName); - TierConfigIdx = GetFieldIndex(rawData, TierConfig); - } - }; - bool DeserializeFromRecord(const TDecoder& decoder, const Ydb::Value& r); - NMetadata::NInternal::TTableRecord SerializeToRecord() const; - bool IsSame(const TTierConfig& item) const; NJson::TJsonValue GetDebugJson() const; - static TString GetTypeId() { - return "TIER"; - } }; - } diff --git a/ydb/core/tx/tiering/tier/ss_checker.cpp b/ydb/core/tx/tiering/tier/ss_checker.cpp deleted file mode 100644 index 7626a13a309e..000000000000 --- a/ydb/core/tx/tiering/tier/ss_checker.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "ss_checker.h" - -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -void TSSFetchingActor::Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev) { - auto g = PassAwayGuard(); - Controller->FetchingResult(ev->Get()->Record); -} - -TSSFetchingActor::TSSFetchingActor(NSchemeShard::ISSDataProcessor::TPtr processor, - ISSFetchingController::TPtr controller, const TDuration livetime) - : TBase(livetime) - , Processor(processor) - , Controller(controller) -{ - -} - -constexpr NKikimrServices::TActivity::EType TSSFetchingActor::ActorActivityType() { - return NKikimrServices::TActivity::SS_FETCHING_ACTOR; -} - -} diff --git a/ydb/core/tx/tiering/tier/ss_checker.h b/ydb/core/tx/tiering/tier/ss_checker.h deleted file mode 100644 index d1e9e777e5f5..000000000000 --- a/ydb/core/tx/tiering/tier/ss_checker.h +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once -#include "object.h" - -#include -#include -#include -#include -#include -#include - -namespace NKikimr::NColumnShard::NTiers { - -class ISSFetchingController { -public: - using TPtr = std::shared_ptr; - virtual ~ISSFetchingController() = default; - virtual void FetchingProblem(const TString& errorMessage) const = 0; - virtual void FetchingResult(const NKikimrScheme::TEvProcessingResponse& result) const = 0; -}; - -class TSSFetchingController: public ISSFetchingController { -private: - const TActorIdentity ActorId; -public: - TSSFetchingController(const TActorIdentity& actorId) - : ActorId(actorId) { - - } - - virtual void FetchingProblem(const TString& errorMessage) const override { - ActorId.Send(ActorId, new NSchemeShard::TEvSchemeShard::TEvProcessingResponse(errorMessage)); - } - virtual void FetchingResult(const NKikimrScheme::TEvProcessingResponse& result) const override { - ActorId.Send(ActorId, new NSchemeShard::TEvSchemeShard::TEvProcessingResponse(result)); - } -}; - -class TSSFetchingActor: public NMetadata::NInternal::TSSDialogActor { -private: - using TBase = NMetadata::NInternal::TSSDialogActor; - NSchemeShard::ISSDataProcessor::TPtr Processor; - ISSFetchingController::TPtr Controller; - void Handle(NSchemeShard::TEvSchemeShard::TEvProcessingResponse::TPtr& ev); -protected: - virtual void OnBootstrap() override { - UnsafeBecome(&TSSFetchingActor::StateMain); - TBase::OnBootstrap(); - } - virtual void OnFail(const TString& errorMessage) override { - Controller->FetchingProblem(errorMessage); - } - virtual void Execute() override { - auto req = std::make_unique(*Processor); - Send(SchemeShardPipe, new TEvPipeCache::TEvForward(req.release(), SchemeShardId, false)); - } -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType(); - STFUNC(StateMain) { - switch (ev->GetTypeRewrite()) { - hFunc(NSchemeShard::TEvSchemeShard::TEvProcessingResponse, Handle); - default: - TBase::StateMain(ev); - } - } - TSSFetchingActor(NSchemeShard::ISSDataProcessor::TPtr processor, ISSFetchingController::TPtr controller, const TDuration livetime); -}; - -} diff --git a/ydb/core/tx/tiering/tier/ss_fetcher.cpp b/ydb/core/tx/tiering/tier/ss_fetcher.cpp deleted file mode 100644 index e822ace4c5a8..000000000000 --- a/ydb/core/tx/tiering/tier/ss_fetcher.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include "ss_fetcher.h" -#include - -namespace NKikimr::NColumnShard::NTiers { - -TFetcherCheckUserTieringPermissions::TFactory::TRegistrator - TFetcherCheckUserTieringPermissions::Registrator(TFetcherCheckUserTieringPermissions::GetTypeIdStatic()); - -void TFetcherCheckUserTieringPermissions::DoProcess(NSchemeShard::TSchemeShard& schemeShard, NKikimrScheme::TEvProcessingResponse& result) const { - TResult content; - content.MutableContent().SetOperationAllow(true); - ui32 access = 0; - access |= NACLib::EAccessRights::AlterSchema; - - if (ActivityType == NMetadata::NModifications::IOperationsManager::EActivityType::Undefined) { - content.Deny("undefined activity type"); - } else { - bool denied = false; - for (auto&& i : TieringRuleIds) { - const auto& pathIds = schemeShard.ColumnTables.GetTablesWithTiering(i); - for (auto&& pathId : pathIds) { - auto path = NSchemeShard::TPath::Init(pathId, &schemeShard); - if (!path.IsResolved() || path.IsUnderDeleting() || path.IsDeleted()) { - continue; - } - if (ActivityType == NMetadata::NModifications::IOperationsManager::EActivityType::Drop) { - denied = true; - content.Deny("tiering in using by table"); - break; - } else if (ActivityType == NMetadata::NModifications::IOperationsManager::EActivityType::Alter) { - if (!UserToken) { - continue; - } - TSecurityObject sObject(path->Owner, path->ACL, path->IsContainer()); - if (!sObject.CheckAccess(access, *UserToken)) { - denied = true; - content.Deny("no alter permissions for affected table"); - break; - } - } - } - if (denied) { - break; - } - } - } - result.MutableContent()->SetData(content.SerializeToString()); -} - -bool TFetcherCheckUserTieringPermissions::DoDeserializeFromProto(const TProtoClass& protoData) { - if (!TryFromString(protoData.GetActivityType(), ActivityType)) { - ALS_ERROR(0) << "Cannot parse activity type: undefined value = " << protoData.GetActivityType(); - return false; - } - if (protoData.GetUserToken()) { - NACLib::TUserToken uToken(protoData.GetUserToken()); - UserToken = uToken; - } - for (auto&& i : protoData.GetTieringRuleIds()) { - TieringRuleIds.emplace(i); - } - return true; -} - -NKikimr::NColumnShard::NTiers::TFetcherCheckUserTieringPermissions::TProtoClass TFetcherCheckUserTieringPermissions::DoSerializeToProto() const { - TProtoClass result; - result.SetActivityType(::ToString(ActivityType)); - if (UserToken) { - result.SetUserToken(UserToken->SerializeAsString()); - } - for (auto&& i : TieringRuleIds) { - *result.AddTieringRuleIds() = i; - } - return result; -} - -} diff --git a/ydb/core/tx/tiering/tier/ss_fetcher.h b/ydb/core/tx/tiering/tier/ss_fetcher.h deleted file mode 100644 index 8d0891d7362d..000000000000 --- a/ydb/core/tx/tiering/tier/ss_fetcher.h +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once -#include "object.h" - -#include -#include -#include - -#include - -namespace NKikimr::NColumnShard::NTiers { - -class TFetcherCheckUserTieringPermissionsResult: public NBackgroundTasks::IProtoStringSerializable< - NKikimrScheme::TFetcherCheckUserTieringPermissionsResult, NBackgroundTasks::IStringSerializable> { -private: - using TProtoClass = NKikimrScheme::TFetcherCheckUserTieringPermissionsResult; - YDB_ACCESSOR_DEF(TProtoClass, Content); -protected: - virtual TProtoClass DoSerializeToProto() const override { - return Content; - } - virtual bool DoDeserializeFromProto(const TProtoClass& protoData) override { - Content = protoData; - return true; - } -public: - void Deny(const TString& reason) { - Content.SetOperationAllow(false); - Content.SetDenyReason(reason); - } -}; - -class TFetcherCheckUserTieringPermissions: public NBackgroundTasks::IProtoStringSerializable< - NKikimrScheme::TFetcherCheckUserTieringPermissions, NSchemeShard::ISSDataProcessor> { -private: - using TBase = NBackgroundTasks::IProtoStringSerializable< - NKikimrScheme::TFetcherCheckUserTieringPermissions, NSchemeShard::ISSDataProcessor>; - using TBase::TFactory; - using TProtoClass = NKikimrScheme::TFetcherCheckUserTieringPermissions; - static TFactory::TRegistrator Registrator; - YDB_ACCESSOR_DEF(std::set, TieringRuleIds); - YDB_ACCESSOR_DEF(std::optional, UserToken); - YDB_ACCESSOR(NMetadata::NModifications::IOperationsManager::EActivityType, ActivityType, - NMetadata::NModifications::IOperationsManager::EActivityType::Undefined); -protected: - virtual TProtoClass DoSerializeToProto() const override; - virtual bool DoDeserializeFromProto(const TProtoClass& protoData) override; - virtual void DoProcess(NSchemeShard::TSchemeShard& schemeShard, NKikimrScheme::TEvProcessingResponse& result) const override; -public: - using TResult = TFetcherCheckUserTieringPermissionsResult; - std::optional UnpackResult(const TString& content) const { - TFetcherCheckUserTieringPermissionsResult result; - if (!result.DeserializeFromString(content)) { - return {}; - } else { - return result; - } - } - - TFetcherCheckUserTieringPermissions() = default; - - virtual TString DebugString() const override { - TStringBuilder sb; - sb << "USID=" << (UserToken ? UserToken->GetUserSID() : "nobody") << ";"; - sb << "tierings="; - for (auto&& i : TieringRuleIds) { - sb << i << ","; - } - sb << ";"; - return sb; - } - virtual TString GetClassName() const override { - return GetTypeIdStatic(); - } - static TString GetTypeIdStatic() { - return "ss_fetcher_tiering_permissions"; - } -}; - -} diff --git a/ydb/core/tx/tiering/tier/ya.make b/ydb/core/tx/tiering/tier/ya.make index 822435a2ec4c..ca6842eab416 100644 --- a/ydb/core/tx/tiering/tier/ya.make +++ b/ydb/core/tx/tiering/tier/ya.make @@ -1,20 +1,11 @@ LIBRARY() SRCS( - manager.cpp object.cpp - initializer.cpp - checker.cpp - GLOBAL behaviour.cpp - ss_checker.cpp ) PEERDIR( - ydb/services/bg_tasks/abstract - ydb/services/metadata/initializer - ydb/services/metadata/abstract ydb/services/metadata/secret - ydb/core/tx/schemeshard ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/tiering/ya.make b/ydb/core/tx/tiering/ya.make index 4090ce51fb6d..587457ed5a0d 100644 --- a/ydb/core/tx/tiering/ya.make +++ b/ydb/core/tx/tiering/ya.make @@ -3,8 +3,6 @@ LIBRARY() SRCS( common.cpp manager.cpp - GLOBAL external_data.cpp - snapshot.cpp ) IF (OS_WINDOWS) @@ -27,6 +25,8 @@ PEERDIR( ydb/services/metadata ) +YQL_LAST_ABI_VERSION() + END() RECURSE_FOR_TESTS( From c0affe8d0bda0afe0e566a502e909b228b5e3e86 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Wed, 13 Nov 2024 18:29:03 +0000 Subject: [PATCH 11/12] wip --- ydb/core/kqp/ut/common/columnshard.cpp | 26 +- ydb/core/kqp/ut/common/kqp_ut_common.h | 6 +- ydb/core/kqp/ut/olap/tiering_ut.cpp | 4 +- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 16 +- .../blobs_action/tier/gc_actor.cpp | 21 +- .../columnshard/blobs_action/tier/gc_actor.h | 1 + .../tx/columnshard/blobs_action/tier/read.cpp | 11 + .../columnshard/blobs_action/tier/storage.cpp | 20 +- .../columnshard/blobs_action/tier/write.cpp | 7 + .../test_helper/columnshard_ut_common.h | 2 +- ydb/core/tx/columnshard/ut_rw/ut_backup.cpp | 2 +- .../ut_schema/ut_columnshard_schema.cpp | 4 +- .../alter/in_store/schema/update.cpp | 2 +- .../operations/alter/standalone/update.cpp | 2 +- .../olap/operations/alter_store.cpp | 2 +- .../olap/operations/create_table.cpp | 6 +- .../tx/schemeshard/olap/schema/schema.cpp | 4 +- ydb/core/tx/schemeshard/olap/schema/schema.h | 2 +- .../tx/schemeshard/olap/ttl/validator.cpp | 17 +- ydb/core/tx/schemeshard/olap/ttl/validator.h | 2 +- ...d__operation_drop_external_data_source.cpp | 4 + ydb/core/tx/tiering/manager.cpp | 10 +- ydb/core/tx/tiering/tier/object.cpp | 32 +- ydb/core/tx/tiering/tier/object.h | 2 +- ydb/core/tx/tiering/ut/ut_tiers.cpp | 315 +++++------------- ydb/core/wrappers/abstract.cpp | 2 +- ydb/core/wrappers/fake_storage.h | 2 +- 27 files changed, 210 insertions(+), 314 deletions(-) diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 2383a075aa6e..32591f447469 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -13,27 +13,6 @@ extern "C" { namespace NKikimr { namespace NKqp { - - TString GetConfigProtoWithName(const TString & tierName) { - return TStringBuilder() << "Name : \"" << tierName << "\"\n" << - R"( - ObjectStorage : { - Endpoint: "fake" - Bucket: "fake" - SecretableAccessKey: { - Value: { - Data: "secretAccessKey" - } - } - SecretableSecretKey: { - Value: { - Data: "fakeSecret" - } - } - } - )"; - } - using namespace NYdb; TTestHelper::TTestHelper(const TKikimrSettings& settings) { @@ -69,12 +48,13 @@ namespace NKqp { } void TTestHelper::CreateTier(const TString& tierName) { - auto result = GetSession().ExecuteSchemeQuery(R"( + // auto result = GetSession().ExecuteSchemeQuery(R"( + auto result = Kikimr->GetTableClient(NYdb::NTable::TClientSettings().AuthToken("root@builtin")).GetSession().GetValueSync().GetSession().ExecuteSchemeQuery(R"( UPSERT OBJECT `accessKey` (TYPE SECRET) WITH (value = `secretAccessKey`); UPSERT OBJECT `secretKey` (TYPE SECRET) WITH (value = `fakeSecret`); CREATE EXTERNAL DATA SOURCE `)" + tierName + R"(` WITH ( SOURCE_TYPE="ObjectStorage", - LOCATION="fake/fake", + LOCATION="http://fake.fake/fake", AUTH_METHOD="AWS", AWS_ACCESS_KEY_ID_SECRET_NAME="accessKey", AWS_SECRET_ACCESS_KEY_SECRET_NAME="secretKey", diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index ed9bf33ff214..86b32e0da63f 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -167,9 +167,9 @@ class TKikimrRunner { NYdb::TDriverConfig GetDriverConfig() const { return DriverConfig; } - NYdb::NTable::TTableClient GetTableClient() const { - return NYdb::NTable::TTableClient(*Driver, NYdb::NTable::TClientSettings() - .UseQueryCache(false)); + NYdb::NTable::TTableClient GetTableClient( + NYdb::NTable::TClientSettings settings = NYdb::NTable::TClientSettings()) const { + return NYdb::NTable::TTableClient(*Driver, settings.UseQueryCache(false)); } NYdb::NQuery::TQueryClient GetQueryClient( diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index f14fa26a325b..0fe7bb4a5690 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -68,7 +68,7 @@ class TTestEvictionBase { UNIT_ASSERT_GT(columnRawBytes, 0); } - TestHelper->SetTiering("/Root/olapStore/olapTable", "tier1", "timestamp"); + TestHelper->SetTiering("/Root/olapStore/olapTable", "/Root/tier1", "timestamp"); csController->WaitActualization(TDuration::Seconds(5)); { @@ -82,7 +82,7 @@ class TTestEvictionBase { auto rows = ExecuteScanQuery(tableClient, selectQuery); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "/Root/tier1"); UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes << " after=" << GetUint64(rows[0].at("RawBytes"))); diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index acb9892f1ba8..596586b2f2b0 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5441,7 +5441,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { WITH ( STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 10, - TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE tier1 ON Key + TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE `/Root/tier1` ON Key );)"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -5457,7 +5457,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { } auto query2 = TStringBuilder() << R"( --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` SET (TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE tier2 ON Key);)"; + ALTER TABLE `)" << tableName << R"(` SET (TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE `/Root/tier2` ON Key);)"; result = session.ExecuteSchemeQuery(query2).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -5468,7 +5468,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); auto ttl = desc.GetTableDescription().GetTtlSettings(); UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "tier2"); + UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "/Root/tier"); UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10)); } @@ -5488,7 +5488,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto query4 = TStringBuilder() << R"( --!syntax_v1 - ALTER TABLE `)" << tableName << R"(` SET (TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE tier1 ON Key);)"; + ALTER TABLE `)" << tableName << R"(` SET (TTL = Interval("PT10S") TO EXTERNAL DATA SOURCE `/Root/tier1` ON Key);)"; result = session.ExecuteSchemeQuery(query4).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); @@ -5499,7 +5499,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT(desc.GetTableDescription().GetTtlSettings()); auto ttl = desc.GetTableDescription().GetTtlSettings(); UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "tier1"); + UNIT_ASSERT_VALUES_EQUAL(std::get(ttl->GetTiers()[0].GetAction()).StorageName, "/Root/tier1"); UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers()[0].GetApplyAfter(), TDuration::Seconds(10)); } @@ -8118,7 +8118,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.BulkUpsert(testTable, tableInserter); } - testHelper.SetTiering(tableName, "tier1", "created_at"); + testHelper.SetTiering(tableName, "/root/tier1", "created_at"); while (csController->GetTieringUpdates().Val() == 0) { Cout << "Wait tiering..." << Endl; @@ -8186,7 +8186,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { UNIT_ASSERT_VALUES_EQUAL(description.GetTtlSettings()->GetDateTypeColumn().GetExpireAfter(), TDuration::Hours(1)); } { - auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`SET (TTL = Interval(\"PT10S\") TO EXTERNAL DATA SOURCE tier1, Interval(\"PT1H\") DELETE ON created_at);"; + auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`SET (TTL = Interval(\"PT10S\") TO EXTERNAL DATA SOURCE `/Root/tier1`, Interval(\"PT1H\") DELETE ON created_at);"; auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); } @@ -8201,7 +8201,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { UNIT_ASSERT_VALUES_EQUAL(ttl->GetTiers().size(), 2); auto evictTier = ttl->GetTiers()[0]; UNIT_ASSERT(std::holds_alternative(evictTier.GetAction())); - UNIT_ASSERT_VALUES_EQUAL(std::get(evictTier.GetAction()).StorageName, "tier1"); + UNIT_ASSERT_VALUES_EQUAL(std::get(evictTier.GetAction()).StorageName, "/Root/tier1"); UNIT_ASSERT_VALUES_EQUAL(evictTier.GetApplyAfter(), TDuration::Seconds(10)); auto deleteTier = ttl->GetTiers()[1]; UNIT_ASSERT(std::holds_alternative(deleteTier.GetAction())); diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.cpp b/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.cpp index e3dc0b3ca24d..2c1c03ea0385 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.cpp @@ -8,8 +8,7 @@ void TGarbageCollectionActor::Handle(NWrappers::NExternalStorage::TEvDeleteObjec TString errorMessage; Y_ABORT_UNLESS(ev->Get()->Key); AFL_VERIFY(TLogoBlobID::Parse(logoBlobId, *ev->Get()->Key, errorMessage))("error", errorMessage); - BlobIdsToRemove.erase(logoBlobId); - CheckFinished(); + OnDeleteBlobFinished(logoBlobId, ev->Get()->IsSuccess()); } void TGarbageCollectionActor::Bootstrap(const TActorContext& ctx) { @@ -20,6 +19,14 @@ void TGarbageCollectionActor::Bootstrap(const TActorContext& ctx) { for (auto&& i : GCTask->GetDraftBlobIds()) { BlobIdsToRemove.emplace(i.GetLogoBlobId()); } + TBase::Bootstrap(ctx); + Become(&TGarbageCollectionActor::StateWork); + if (!GCTask->GetExternalStorageOperator()) { + for (auto&& i : BlobIdsToRemove) { + OnDeleteBlobFinished(i, false, "storage operator is uninitialized for tier: " + GCTask->GetStorageId()); + } + return; + } for (auto&& i : BlobIdsToRemove) { auto awsRequest = Aws::S3::Model::DeleteObjectRequest().WithKey(i.ToString()); auto request = std::make_unique(awsRequest); @@ -27,8 +34,6 @@ void TGarbageCollectionActor::Bootstrap(const TActorContext& ctx) { TAutoPtr> evPtr((TEventHandle*)hRequest.release()); GCTask->GetExternalStorageOperator()->Execute(evPtr); } - TBase::Bootstrap(ctx); - Become(&TGarbageCollectionActor::StateWork); } void TGarbageCollectionActor::CheckFinished() { @@ -39,4 +44,12 @@ void TGarbageCollectionActor::CheckFinished() { } } +void TGarbageCollectionActor::OnDeleteBlobFinished(const TLogoBlobID& blobId, bool success, const TString& errorMessage) { + if (success) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_BLOBS_TIER)("actor", "TGarbageCollectionActor")("event", "delete_object_failed")( + "reason", errorMessage); + } + BlobIdsToRemove.erase(blobId); + CheckFinished(); +} } diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h b/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h index 4834b0e3911d..0ec15a91e4f0 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h @@ -17,6 +17,7 @@ class TGarbageCollectionActor: public TSharedBlobsCollectionActor BlobIdsToRemove; void Handle(NWrappers::NExternalStorage::TEvDeleteObjectResponse::TPtr& ev); void CheckFinished(); + void OnDeleteBlobFinished(const TLogoBlobID& blobId, bool success, const TString& errorMessage = ""); virtual void DoOnSharedRemovingFinished() override { CheckFinished(); diff --git a/ydb/core/tx/columnshard/blobs_action/tier/read.cpp b/ydb/core/tx/columnshard/blobs_action/tier/read.cpp index 7af0c28583af..dd16b96991b7 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/read.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/read.cpp @@ -1,8 +1,19 @@ #include "read.h" +#include + namespace NKikimr::NOlap::NBlobOperations::NTier { void TReadingAction::DoStartReading(THashSet&& ranges) { + if (!ExternalStorageOperator) { + for (const auto& range : ranges) { + auto response = std::make_unique(range, NKikimrProto::EReplyStatus::ERROR, + "storage operator is uninitialized for tier: " + GetStorageId(), false, GetStorageId()); + TActorContext::AsActorContext().Send(TActorContext::AsActorContext().SelfID, response.release()); + } + return; + } + for (auto&& r : ranges) { auto awsRequest = Aws::S3::Model::GetObjectRequest() .WithKey(r.BlobId.GetLogoBlobId().ToString()) diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp index 9188630e10a3..e2b5cd691945 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp @@ -54,18 +54,21 @@ void TOperator::DoStartGCAction(const std::shared_ptr& action) c } void TOperator::InitNewExternalOperator(const NColumnShard::NTiers::TManager* tierManager) { - NKikimrSchemeOp::TS3Settings settings; - if (tierManager && tierManager->IsReady()) { - settings = tierManager->GetS3Settings(); - } else { - settings.SetEndpoint("nowhere"); + if (!tierManager || !tierManager->IsReady()) { + TGuard changeLock(ChangeOperatorLock); + CurrentS3Settings.reset(); + ExternalStorageOperator = nullptr; + return; } + + NKikimrSchemeOp::TS3Settings settings = tierManager->GetS3Settings(); { TGuard changeLock(ChangeOperatorLock); if (CurrentS3Settings && CurrentS3Settings->SerializeAsString() == settings.SerializeAsString()) { return; } } + auto extStorageConfig = NWrappers::NExternalStorage::IExternalStorageConfig::Construct(settings); AFL_VERIFY(extStorageConfig); auto extStorageOperator = extStorageConfig->ConstructStorageOperator(false); @@ -103,12 +106,7 @@ TOperator::TOperator(const TString& storageId, const TActorId& shardActorId, con void TOperator::DoOnTieringModified(const std::shared_ptr& tiers) { auto* tierManager = tiers->GetManagerOptional(TBase::GetStorageId()); - if (tierManager) { - InitNewExternalOperator(tierManager); - } else { - TGuard changeLock(ChangeOperatorLock); - ExternalStorageOperator = nullptr; - } + InitNewExternalOperator(tierManager); } bool TOperator::DoLoad(IBlobManagerDb& dbBlobs) { diff --git a/ydb/core/tx/columnshard/blobs_action/tier/write.cpp b/ydb/core/tx/columnshard/blobs_action/tier/write.cpp index 552513304506..86158f817c3e 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/write.cpp @@ -6,6 +6,13 @@ namespace NKikimr::NOlap::NBlobOperations::NTier { void TWriteAction::DoSendWriteBlobRequest(const TString& data, const TUnifiedBlobId& blobId) { + if (!ExternalStorageOperator) { + auto response = std::make_unique(NKikimrProto::EReplyStatus::ERROR, blobId.GetLogoBlobId(), 0, TGroupId::FromValue(Max()), 0, GetStorageId()); + response->ErrorReason = "storage operator is uninitialized for tier: " + GetStorageId(); + TActorContext::AsActorContext().Send(TActorContext::AsActorContext().SelfID, response.release()); + return; + } + auto awsRequest = Aws::S3::Model::PutObjectRequest().WithKey(blobId.GetLogoBlobId().ToString()); TString moveData = data; diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 4cc86fa9d642..f20fabfe3a5f 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -108,7 +108,7 @@ struct TTestSchema { s3Config.SetProxyPort(8080); s3Config.SetProxyScheme(NKikimrSchemeOp::TS3Settings::HTTP); #else - s3Config.SetEndpoint("fake"); + s3Config.SetEndpoint("fake.fake"); s3Config.SetSecretKey("fakeSecret"); #endif s3Config.SetRequestTimeoutMs(10000); diff --git a/ydb/core/tx/columnshard/ut_rw/ut_backup.cpp b/ydb/core/tx/columnshard/ut_rw/ut_backup.cpp index 887780e4da73..8ca1b5a87245 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_backup.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_backup.cpp @@ -98,7 +98,7 @@ Y_UNIT_TEST_SUITE(Backup) { txBody.MutableBackupTask()->SetTableId(tableId); txBody.MutableBackupTask()->SetSnapshotStep(backupSnapshot.GetPlanStep()); txBody.MutableBackupTask()->SetSnapshotTxId(backupSnapshot.GetTxId()); - txBody.MutableBackupTask()->MutableS3Settings()->SetEndpoint("fake"); + txBody.MutableBackupTask()->MutableS3Settings()->SetEndpoint("fake.fake"); txBody.MutableBackupTask()->MutableS3Settings()->SetSecretKey("fakeSecret"); AFL_VERIFY(csControllerGuard->GetFinishedExportsCount() == 0); UNIT_ASSERT(ProposeTx(runtime, sender, NKikimrTxColumnShard::TX_KIND_BACKUP, txBody.SerializeAsString(), ++txId)); diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 1220870814bf..7e365cf9fb8d 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -608,14 +608,14 @@ std::vector> TestTiers(bool reboots, const std::vector OK, misconfig after export => ERROR if (i > 1) { expectedReadResult = EExpectedResult::ERROR; } originalEndpoint = spec.S3.GetEndpoint(); - spec.S3.SetEndpoint("fake"); + spec.S3.SetEndpoint("fake.fake"); tIdxCorrect = tIdx++; } break; diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp index bbf1845ac1bb..fbca60ce80bb 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp @@ -41,7 +41,7 @@ NKikimr::TConclusionStatus TInStoreSchemaUpdate::DoInitializeImpl(const TUpdateI return patch; } TSimpleErrorCollector collector; - if (!originalSchema.ValidateTtlSettings(ttl.GetData(), collector)) { + if (!originalSchema.ValidateTtlSettings(ttl.GetData(), context.GetSSOperationContext()->SS, collector)) { return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE"); } *description.MutableTtlSettings() = ttl.SerializeToProto(); diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp index 2902534fbb24..352192c6f29c 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp @@ -65,7 +65,7 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitializeImpl(const TUpda } *description.MutableTtlSettings() = ttl.SerializeToProto(); } - if (!targetSchema.ValidateTtlSettings(ttl.GetData(), collector)) { + if (!targetSchema.ValidateTtlSettings(ttl.GetData(), context.GetSSOperationContext()->SS, collector)) { return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE"); } auto saSharding = originalTable.GetTableInfoVerified().GetStandaloneShardingVerified(); diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp index 1f4c83a40101..1e1b5b9c7716 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp @@ -533,7 +533,7 @@ class TAlterOlapStore: public TSubOperation { } auto it = alterData->SchemaPresets.find(table->Description.GetSchemaPresetId()); AFL_VERIFY(it != alterData->SchemaPresets.end())("preset_info", table->Description.DebugString()); - if (!it->second.ValidateTtlSettings(table->Description.GetTtlSettings(), errors)) { + if (!it->second.ValidateTtlSettings(table->Description.GetTtlSettings(), context.SS, errors)) { return result; } } diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp index b4d6df8f483f..3641c4ae0d71 100644 --- a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp @@ -24,7 +24,7 @@ class TTableConstructorBase { protected: ui32 ShardsCount = 0; public: - bool Deserialize(const NKikimrSchemeOp::TColumnTableDescription& description, IErrorCollector& errors) { + bool Deserialize(const NKikimrSchemeOp::TColumnTableDescription& description, TSchemeShard* ctx, IErrorCollector& errors) { Name = description.GetName(); ShardsCount = std::max(description.GetColumnShardCount(), 1); @@ -34,7 +34,7 @@ class TTableConstructorBase { if (description.HasTtlSettings()) { TtlSettings = description.GetTtlSettings(); - if (!GetSchema().ValidateTtlSettings(description.GetTtlSettings(), errors)) { + if (!GetSchema().ValidateTtlSettings(description.GetTtlSettings(), ctx, errors)) { return false; } } @@ -50,7 +50,7 @@ class TTableConstructorBase { } FillDefaultSharding(*tableInfo->Description.MutableSharding()); - if (!Deserialize(description, errors)) { + if (!Deserialize(description, context.SS, errors)) { return nullptr; } if (tableInfo->Description.GetSharding().HasHashSharding()) { diff --git a/ydb/core/tx/schemeshard/olap/schema/schema.cpp b/ydb/core/tx/schemeshard/olap/schema/schema.cpp index 97cc54eeaaf0..d193c9740fed 100644 --- a/ydb/core/tx/schemeshard/olap/schema/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/schema/schema.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NSchemeShard { -bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycle& ttl, IErrorCollector& errors) const { +bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycle& ttl, TSchemeShard* ctx, IErrorCollector& errors) const { using TTtlProto = NKikimrSchemeOp::TColumnDataLifeCycle; switch (ttl.GetStatusCase()) { case TTtlProto::kEnabled: @@ -15,7 +15,7 @@ bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycl errors.AddError("Incorrect ttl column - not found in scheme"); return false; } - return TTTLValidator::ValidateColumnTableTtl(ttl.GetEnabled(), Indexes, {}, Columns.GetColumns(), Columns.GetColumnsByName(), errors); + return TTTLValidator::ValidateColumnTableTtl(ttl.GetEnabled(), Indexes, {}, Columns.GetColumns(), Columns.GetColumnsByName(), ctx, errors); } case TTtlProto::kDisabled: default: diff --git a/ydb/core/tx/schemeshard/olap/schema/schema.h b/ydb/core/tx/schemeshard/olap/schema/schema.h index 309ce3ab69f6..2b74631e72c9 100644 --- a/ydb/core/tx/schemeshard/olap/schema/schema.h +++ b/ydb/core/tx/schemeshard/olap/schema/schema.h @@ -27,7 +27,7 @@ namespace NKikimr::NSchemeShard { void ParseFromLocalDB(const NKikimrSchemeOp::TColumnTableSchema& tableSchema); void Serialize(NKikimrSchemeOp::TColumnTableSchema& tableSchema) const; bool Validate(const NKikimrSchemeOp::TColumnTableSchema& opSchema, IErrorCollector& errors) const; - bool ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycle& ttlSettings, IErrorCollector& errors) const; + bool ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycle& ttlSettings, TSchemeShard* ctx, IErrorCollector& errors) const; }; class TOlapStoreSchemaPreset: public TOlapSchema { diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp index bb9b0230aada..275f75ab4610 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -1,5 +1,7 @@ #include "validator.h" + #include +#include namespace NKikimr::NSchemeShard { @@ -15,7 +17,7 @@ static inline NScheme::TTypeInfo GetType(const TOlapColumnsDescription::TColumn& } -bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, const THashMap& sourceColumns, const THashMap& alterColumns, const THashMap& colName2Id, IErrorCollector& errors) { +bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, const THashMap& sourceColumns, const THashMap& alterColumns, const THashMap& colName2Id, TSchemeShard* ctx, IErrorCollector& errors) { const TString colName = ttl.GetColumnName(); auto it = colName2Id.find(colName); @@ -97,6 +99,19 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif } } + for (const auto& tier : ttl.GetTiers()) { + TPath tierPath = TPath::Resolve(tier.GetStorageName(), ctx); + if (!tierPath.IsResolved() || tierPath.IsDeleted() || tierPath.IsUnderDeleting()) { + errors.AddError("Object not found: " + tier.GetStorageName()); + return false; + } + if (!tierPath->IsExternalDataSource()) { + errors.AddError("Not an external data source: " + tier.GetStorageName()); + return false; + } + // TODO: check that TTierConfig is deserializable from external data source + } + return true; } diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.h b/ydb/core/tx/schemeshard/olap/ttl/validator.h index 72b9a975e835..34d12532ef79 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.h +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.h @@ -8,7 +8,7 @@ namespace NKikimr::NSchemeShard { static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, const THashMap& sourceColumns, const THashMap& alterColumns, - const THashMap& colName2Id, + const THashMap& colName2Id, TSchemeShard* ctx, IErrorCollector& errors); }; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_data_source.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_data_source.cpp index 6eee14346f20..26f6e1eba143 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_data_source.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_external_data_source.cpp @@ -152,6 +152,10 @@ class TDropExternalDataSource: public TSubOperation { result->SetError(NKikimrScheme::StatusSchemeError, "Other entities depend on this data source, please remove them at the beginning: " + externalDataSource->ExternalTableReferences.GetReferences(0).GetPath()); return result; } + if (auto tables = context.SS->ColumnTables.GetTablesWithTier(path.PathString()); !tables.empty()) { + result->SetError(NKikimrScheme::StatusSchemeError, "Column tables depend on this data source: " + TPath::Init(*tables.begin(), context.SS).PathString()); + return result; + } TString errStr; if (!context.SS->CheckApplyIf(Transaction, errStr)) { diff --git a/ydb/core/tx/tiering/manager.cpp b/ydb/core/tx/tiering/manager.cpp index 1fc5948b5d8e..880756a1fe84 100644 --- a/ydb/core/tx/tiering/manager.cpp +++ b/ydb/core/tx/tiering/manager.cpp @@ -57,12 +57,14 @@ class TTiersManager::TActor: public TActorBootstrapped { } void Handle(NTiers::TEvNotifySchemeObjectUpdated::TPtr& ev) { - AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiering_manager")("event", "object_updated")("path", ev->Get()->GetObjectId())("type", "TIERED_STORAGE"); + AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiering_manager")("event", "object_updated")("path", ev->Get()->GetObjectId()); const TString& objectId = ev->Get()->GetObjectId(); const auto& description = ev->Get()->GetDescription(); if (description.GetSelf().GetPathType() == NKikimrSchemeOp::EPathTypeExternalDataSource) { NTiers::TTierConfig tier; - AFL_VERIFY(tier.DeserializeFromProto(description.GetExternalDataSourceDescription())); + if (const auto status = tier.DeserializeFromProto(description.GetExternalDataSourceDescription()); status.IsFail()) { + AFL_VERIFY(false)("error", status.GetErrorMessage()); + } Owner->UpdateTierConfig(tier, objectId); } else { AFL_WARN(NKikimrServices::TX_TIERING)("error", "invalid_object_type")("type", static_cast(description.GetSelf().GetPathType()))("path", objectId); @@ -70,14 +72,14 @@ class TTiersManager::TActor: public TActorBootstrapped { } void Handle(NTiers::TEvNotifySchemeObjectDeleted::TPtr& ev) { - AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiering_manager")("event", "object_deleted")("name", ev->Get()->GetObjectId())("type", "TIERED_STORAGE"); + AFL_DEBUG(NKikimrServices::TX_TIERING)("component", "tiering_manager")("event", "object_deleted")("name", ev->Get()->GetObjectId()); } void Handle(NTiers::TEvSchemeObjectResolutionFailed::TPtr& ev) { const TString objectId = ev->Get()->GetObjectId(); switch (ev->Get()->GetReason()) { case NTiers::TEvSchemeObjectResolutionFailed::NOT_FOUND: - AFL_WARN(NKikimrServices::TX_TIERING)("event", "object_not_found")("name", objectId)("type", "TIERING_RULE"); + AFL_WARN(NKikimrServices::TX_TIERING)("event", "object_not_found")("name", objectId); break; case NTiers::TEvSchemeObjectResolutionFailed::LOOKUP_ERROR: ScheduleRetryWatchObjects(std::make_unique(std::vector({ objectId }))); diff --git a/ydb/core/tx/tiering/tier/object.cpp b/ydb/core/tx/tiering/tier/object.cpp index 389ad0193c15..4aac45cd511a 100644 --- a/ydb/core/tx/tiering/tier/object.cpp +++ b/ydb/core/tx/tiering/tier/object.cpp @@ -23,23 +23,23 @@ NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig( return config; } -bool TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourceDescription& proto) { +TConclusionStatus TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourceDescription& proto) { if (!proto.GetAuth().HasAws()) { - return false; + return TConclusionStatus::Fail("AWS auth is not defined for storage tier"); } // TODO fix secret owner { auto makeSecretId = [](const TStringBuf& secret) -> TString { - return NMetadata::NSecret::TSecretId("a", secret).SerializeToString(); // ... and here + return NMetadata::NSecret::TSecretId("root@builtin", secret).SerializeToString(); // ... here }; - ProtoConfig.SetSecretKey(makeSecretId(proto.GetAuth().GetAws().GetAwsAccessKeyIdSecretName())); - ProtoConfig.SetAccessKey(makeSecretId(proto.GetAuth().GetAws().GetAwsSecretAccessKeySecretName())); + ProtoConfig.SetAccessKey(makeSecretId(proto.GetAuth().GetAws().GetAwsAccessKeyIdSecretName())); + ProtoConfig.SetSecretKey(makeSecretId(proto.GetAuth().GetAws().GetAwsSecretAccessKeySecretName())); } NUri::TUri url; - if (url.Parse(proto.GetLocation()) != NUri::TState::EParsed::ParsedOK) { - return false; + if (url.Parse(proto.GetLocation(), NUri::TFeature::FeaturesAll) != NUri::TState::EParsed::ParsedOK) { + return TConclusionStatus::Fail("Cannot parse url: " + proto.GetLocation()); } switch (url.GetScheme()) { @@ -52,7 +52,7 @@ bool TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourc ProtoConfig.SetScheme(::NKikimrSchemeOp::TS3Settings_EScheme_HTTPS); break; default: - return false; + return TConclusionStatus::Fail("Unknown schema in url"); } { @@ -61,18 +61,16 @@ bool TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourc TStringBuf host = url.GetHost(); TStringBuf path = url.GetField(NUri::TField::FieldPath); - if (path.StartsWith("/") && path.Size() > 1) { - bucket = path.Skip(1); - if (bucket.EndsWith("/")) { - bucket = bucket.Chop(1); - } + if (!path.Empty()) { + endpoint = host; + bucket = path; + bucket.SkipPrefix("/"); if (bucket.Contains("/")) { - return false; + return TConclusionStatus::Fail(TStringBuilder() << "Not a bucket (contains directories): " << bucket); } - endpoint = host; } else { if (!path.TrySplit('.', endpoint, bucket)) { - return false; + return TConclusionStatus::Fail(TStringBuilder() << "Bucket is not specified in URL: " << path); } } @@ -80,7 +78,7 @@ bool TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourc ProtoConfig.SetBucket(TString(bucket)); } - return true; + return TConclusionStatus::Success(); } NJson::TJsonValue TTierConfig::SerializeConfigToJson() const { diff --git a/ydb/core/tx/tiering/tier/object.h b/ydb/core/tx/tiering/tier/object.h index f162eeb91403..4a6a097ae42c 100644 --- a/ydb/core/tx/tiering/tier/object.h +++ b/ydb/core/tx/tiering/tier/object.h @@ -25,7 +25,7 @@ class TTierConfig { , Compression(compression) { } - bool DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourceDescription& proto); + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TExternalDataSourceDescription& proto); NMetadata::NSecret::TSecretIdOrValue GetAccessKey() const { auto accessKey = diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index 0afdde164162..bef6636a9c38 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -1,15 +1,17 @@ #include +#include #include -#include #include +#include #include +#include #include -#include -#include -#include #include -#include +#include +#include + #include +#include #include #include #include @@ -17,10 +19,8 @@ #include #include -#include #include #include - #include namespace NKikimr { @@ -122,178 +122,78 @@ class TLocalHelper: public Tests::NCS::THelper { } )", tableName.c_str(), tableShardsCount, shardingFunction.c_str(), shardingColumns.c_str())); } -}; - -Y_UNIT_TEST_SUITE(ColumnShardTiers) { - - TString GetConfigProtoWithName(const TString & tierName) { - return TStringBuilder() << "Name : \"" << tierName << "\"\n" << - R"( - ObjectStorage : { - Endpoint: "fake" - Bucket: "fake" - SecretableAccessKey: { - Value: { - Data: "secretAccessKey" - } - } - SecretableSecretKey: { - Value: { - Data: "secretSecretKey" - } - } - } - )"; + void CreateSecrets() const { + StartSchemaRequest(R"( + UPSERT OBJECT `accessKey` (TYPE SECRET) WITH (value = `secretAccessKey`); + UPSERT OBJECT `secretKey` (TYPE SECRET) WITH (value = `fakeSecret`); + )"); } - const TString ConfigTiering1Str = R"({ - "rules" : [ - { - "tierName" : "tier1", - "durationForEvict" : "10d" - }, - { - "tierName" : "tier2", - "durationForEvict" : "20d" - } - ] - })"; - - const TString ConfigTiering2Str = R"({ - "rules" : [ - { - "tierName" : "tier1", - "durationForEvict" : "10d" - } - ] - })"; - - const TString ConfigTieringNothingStr = R"({ - "rules" : [ - { - "tierName" : "tier1", - "durationForEvict" : "10000d" - }, - { - "tierName" : "tier2", - "durationForEvict" : "20000d" - } - ] - })"; + void CreateExternalDataSource(const TString& name, const TString& location = "http://fake.fake/fake") const { + StartSchemaRequest(R"( + CREATE EXTERNAL DATA SOURCE `)" + + name + R"(` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION=")" + + location + R"(", + AUTH_METHOD="AWS", + AWS_ACCESS_KEY_ID_SECRET_NAME="accessKey", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="secretKey", + AWS_REGION="ru-central1" + ); + )"); + } +}; - class TJsonChecker { - private: - YDB_ACCESSOR_DEF(TString, Path); - YDB_ACCESSOR_DEF(TString, Expectation); - public: - TJsonChecker(const TString& path, const TString& expectation) - : Path(path) - , Expectation(expectation) - { - } - bool Check(const NJson::TJsonValue& jsonInfo) const { - auto* jsonPathValue = jsonInfo.GetValueByPath(Path); - if (!jsonPathValue) { - return Expectation == "__NULL"; - } - return jsonPathValue->GetStringRobust() == Expectation; - } - - TString GetDebugString() const { - TStringBuilder sb; - sb << "path=" << Path << ";" - << "expectation=" << Expectation << ";"; - return sb; - } - }; +Y_UNIT_TEST_SUITE(ColumnShardTiers) { class TTestCSEmulator: public NActors::TActorBootstrapped { private: using TBase = NActors::TActorBootstrapped; - std::shared_ptr ExternalDataManipulation; - TActorId ProviderId; + THashSet ExpectedTiers; TInstant Start; - YDB_READONLY_FLAG(Found, false); - YDB_ACCESSOR(ui32, ExpectedTiersCount, 1); + std::shared_ptr Manager; - using TKeyCheckers = TMap; - YDB_ACCESSOR_DEF(TKeyCheckers, Checkers); public: - void ResetConditions() { - FoundFlag = false; - Checkers.clear(); - } - STATEFN(StateInit) { switch (ev->GetTypeRewrite()) { - hFunc(NMetadata::NProvider::TEvRefreshSubscriberData, Handle); default: Y_ABORT_UNLESS(false); } } void CheckRuntime(TTestActorRuntime& runtime) { - const auto pred = [this](TAutoPtr& event)->TTestActorRuntimeBase::EEventAction { - if (event->HasBuffer() && !event->HasEvent()) { - } else if (!event->HasEvent()) { - } else { - auto ptr = event->CastAsLocal(); - if (ptr) { - CheckFound(ptr); - } - } - return TTestActorRuntimeBase::EEventAction::PROCESS; - }; - - runtime.SetObserverFunc(pred); - for (const TInstant start = Now(); !IsFound() && Now() - start < TDuration::Seconds(30); ) { runtime.SimulateSleep(TDuration::Seconds(1)); } - runtime.SetObserverFunc(TTestActorRuntime::DefaultObserverFunc); Y_ABORT_UNLESS(IsFound()); } - void CheckFound(NMetadata::NProvider::TEvRefreshSubscriberData* event) { - auto snapshot = event->GetSnapshotAs(); - if (!snapshot) { - Cerr << "incorrect snapshot" << Endl; - return; - } - Cerr << "SNAPSHOT: " << snapshot->SerializeToString() << Endl; - if (ExpectedTiersCount != snapshot->GetTierConfigs().size()) { - Cerr << "TiersCount incorrect: " << snapshot->SerializeToString() << ";expectation=" << ExpectedTiersCount << Endl; - return; - } - for (auto&& i : Checkers) { - NJson::TJsonValue jsonData; - if (i.first.StartsWith("TIER.")) { - auto value = snapshot->GetTierById(i.first.substr(5)); - jsonData = value->SerializeConfigToJson(); - } else { - Y_ABORT_UNLESS(false); - } - if (!i.second.Check(jsonData)) { - Cerr << "config value incorrect:" << snapshot->SerializeToString() << ";snapshot_check_path=" << i.first << Endl; - Cerr << "json path incorrect:" << jsonData << ";" << i.second.GetDebugString() << Endl; - return; - } + bool IsFound() const { + THashSet notFoundTiers = ExpectedTiers; + for (const auto& [id, config] : Manager->GetTierConfigs()) { + notFoundTiers.erase(id); } - FoundFlag = true; + return notFoundTiers.empty(); } - void Handle(NMetadata::NProvider::TEvRefreshSubscriberData::TPtr& ev) { - CheckFound(ev->Get()); + const THashMap& GetTierConfigs() { + return Manager->GetTierConfigs(); } void Bootstrap() { - ProviderId = NMetadata::NProvider::MakeServiceId(SelfId().NodeId()); - ExternalDataManipulation = std::make_shared(); Become(&TThis::StateInit); - Sender(ExternalDataManipulation).SendTo(ProviderId); Start = Now(); + Manager = std::make_shared(0, SelfId(), [](const TActorContext&) { + }); + Manager->Start(Manager); + Manager->EnablePathId(0, ExpectedTiers); + } + + TTestCSEmulator(THashSet expectedTiers) + : ExpectedTiers(std::move(expectedTiers)) { } }; @@ -323,6 +223,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { .SetUseRealThreads(false) .SetEnableMetadataProvider(true) .SetEnableTieringInColumnShard(true) + .SetEnableExternalDataSources(true) ; Tests::TServer::TPtr server = new Tests::TServer(serverSettings); @@ -339,55 +240,32 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { TLocalHelper lHelper(*server); { lHelper.CreateTestOlapTable(); - lHelper.StartSchemaRequest("CREATE OBJECT tier1 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc") + "`"); - lHelper.StartSchemaRequest("CREATE OBJECT tier2 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc") + "`"); - lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1, Interval("P20D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); + lHelper.CreateSecrets(); + lHelper.CreateExternalDataSource("/Root/tier1", "http://fake.fake/abc"); + lHelper.CreateExternalDataSource("/Root/tier2", "http://fake.fake/abc"); + lHelper.StartSchemaRequest( + R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1`, Interval("P20D") TO EXTERNAL DATA SOURCE `/Root/tier2` ON timestamp)"); - TTestCSEmulator* emulator = new TTestCSEmulator(); - emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc")); - emulator->SetExpectedTiersCount(2); - runtime.Register(emulator); - runtime.SimulateSleep(TDuration::Seconds(10)); - Cerr << "Initialization finished" << Endl; { - const TInstant start = Now(); - while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) { - runtime.SimulateSleep(TDuration::Seconds(1)); - } - Y_ABORT_UNLESS(emulator->IsFound()); + TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); + runtime.Register(emulator); + emulator->CheckRuntime(runtime); + UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc"); } + Cerr << "Initialization finished" << Endl; { - emulator->ResetConditions(); - emulator->SetExpectedTiersCount(2); - emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); + lHelper.StartSchemaRequest("ALTER EXTERNAL DATA SOURCE `/Root/tier1` SET ENDPOINT = \"fake.fake/abc1\""); - lHelper.StartSchemaRequest("ALTER OBJECT tier1 (TYPE TIER) SET tierConfig = `" + GetConfigProtoWithName("abc1") + "`"); - - { - const TInstant start = Now(); - while (!emulator->IsFound() && Now() - start < TDuration::Seconds(2000)) { - runtime.SimulateSleep(TDuration::Seconds(1)); - } - Y_ABORT_UNLESS(emulator->IsFound()); - } + TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); + runtime.Register(emulator); + emulator->CheckRuntime(runtime); + UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc1"); } { - emulator->ResetConditions(); - emulator->SetExpectedTiersCount(0); - - // TODO: add validation - // lHelper.StartSchemaRequest("DROP OBJECT tier1(TYPE TIER)", false); + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", false); lHelper.StartSchemaRequest("DROP TABLE `/Root/olapStore/olapTable`"); - lHelper.StartSchemaRequest("DROP OBJECT tier1(TYPE TIER)"); - lHelper.StartSchemaRequest("DROP OBJECT tier2(TYPE TIER)"); - - { - const TInstant start = Now(); - while (!emulator->IsFound() && Now() - start < TDuration::Seconds(20)) { - runtime.SimulateSleep(TDuration::Seconds(1)); - } - Y_ABORT_UNLESS(emulator->IsFound()); - } + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`"); + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier2`"); } } } @@ -409,6 +287,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { .SetUseRealThreads(false) .SetEnableMetadataProvider(true) .SetEnableTieringInColumnShard(true) + .SetEnableExternalDataSources(true) .SetAppConfig(appConfig); Tests::TServer::TPtr server = new Tests::TServer(serverSettings); @@ -429,46 +308,38 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { runtime.SimulateSleep(TDuration::Seconds(10)); Cerr << "Initialization finished" << Endl; - lHelper.StartSchemaRequest("CREATE OBJECT tier1 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc1") + "`", true, false); + lHelper.CreateSecrets(); + lHelper.CreateExternalDataSource("/Root/tier1", "http://fake.fake/abc1"); { - TTestCSEmulator* emulator = new TTestCSEmulator; + TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1" }); runtime.Register(emulator); - emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); - emulator->SetExpectedTiersCount(1); emulator->CheckRuntime(runtime); + UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc1"); } - lHelper.StartSchemaRequest("CREATE OBJECT tier2 (TYPE TIER) WITH tierConfig = `" + GetConfigProtoWithName("abc2") + "`"); + lHelper.CreateExternalDataSource("/Root/tier2", "http://fake.fake/abc2"); { - TTestCSEmulator* emulator = new TTestCSEmulator(); + TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); runtime.Register(emulator); - emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "abc1")); - emulator->MutableCheckers().emplace("TIER.tier2", TJsonChecker("Name", "abc2")); - emulator->SetExpectedTiersCount(2); emulator->CheckRuntime(runtime); + UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc1"); + UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier2").GetProtoConfig().GetBucket(), "abc2"); } lHelper.CreateTestOlapTable("olapTable"); - lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1, Interval("P20D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); + lHelper.StartSchemaRequest( + R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1`, Interval("P20D") TO EXTERNAL DATA SOURCE `/Root/tier2` ON timestamp)"); - // TODO: add validation - // lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)", false); - // lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)", false); + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier2`", false); + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", false); lHelper.StartSchemaRequest("DROP TABLE `/Root/olapStore/olapTable`"); { - TTestCSEmulator* emulator = new TTestCSEmulator; - runtime.Register(emulator); - emulator->SetExpectedTiersCount(2); - emulator->CheckRuntime(runtime); - } - lHelper.StartSchemaRequest("DROP OBJECT tier2 (TYPE TIER)"); - lHelper.StartSchemaRequest("DROP OBJECT tier1 (TYPE TIER)", true, false); - { - TTestCSEmulator* emulator = new TTestCSEmulator; + TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); runtime.Register(emulator); - emulator->SetExpectedTiersCount(0); emulator->CheckRuntime(runtime); } + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier2`"); + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", true, false); //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE); //runtime.SetLogPriority(NKikimrServices::KQP_YQL, NLog::PRI_TRACE); @@ -516,7 +387,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { SecretKey: "SId:secretSecretKey" } )"; - const TString TierEndpoint = "fake"; + const TString TierEndpoint = "fake.fake"; #endif Y_UNIT_TEST(TieringUsage) { @@ -536,6 +407,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { .SetUseRealThreads(false) .SetEnableMetadataProvider(true) .SetEnableTieringInColumnShard(true) + .SetEnableExternalDataSources(true) ; Tests::TServer::TPtr server = new Tests::TServer(serverSettings); @@ -558,27 +430,21 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { TLocalHelper lHelper(*server); lHelper.SetOptionalStorageId("__DEFAULT"); - lHelper.StartSchemaRequest("CREATE OBJECT secretAccessKey ( " - "TYPE SECRET) WITH (value = ak)"); - lHelper.StartSchemaRequest("CREATE OBJECT secretSecretKey ( " - "TYPE SECRET) WITH (value = fakeSecret)"); + lHelper.CreateSecrets(); Singleton()->SetSecretKey("fakeSecret"); - lHelper.StartSchemaRequest("CREATE OBJECT tier1 ( " - "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)"); - lHelper.StartSchemaRequest("CREATE OBJECT tier2 ( " - "TYPE TIER) WITH (tierConfig = `" + TierConfigProtoStr + "`)"); + lHelper.CreateExternalDataSource("/Root/tier1", TierEndpoint + "/fake"); + lHelper.CreateExternalDataSource("/Root/tier2", TierEndpoint + "/fake"); { - TTestCSEmulator* emulator = new TTestCSEmulator(); + TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); runtime.Register(emulator); - emulator->MutableCheckers().emplace("TIER.tier1", TJsonChecker("Name", "fakeTier")); - emulator->MutableCheckers().emplace("TIER.tier2", TJsonChecker("ObjectStorage.Endpoint", TierEndpoint)); - emulator->SetExpectedTiersCount(2); emulator->CheckRuntime(runtime); + UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetEndpoint(), TierEndpoint); } lHelper.CreateTestOlapTable("olapTable", 2); - lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE tier1, Interval("P20D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); + lHelper.StartSchemaRequest( + R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1`, Interval("P20D") TO EXTERNAL DATA SOURCE `/Root/tier2` ON timestamp)"); Cerr << "Wait tables" << Endl; runtime.SimulateSleep(TDuration::Seconds(20)); Cerr << "Initialization tables" << Endl; @@ -624,7 +490,8 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { lHelper.DropTable("/Root/olapStore/olapTable"); lHelper.StartDataRequest("DELETE FROM `/Root/olapStore/olapTable`"); */ - lHelper.StartSchemaRequest(R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10000D") TO EXTERNAL DATA SOURCE tier1, Interval("P20000D") TO EXTERNAL DATA SOURCE tier2 ON timestamp)"); + lHelper.StartSchemaRequest( + R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10000D") TO EXTERNAL DATA SOURCE `/Root/tier1`, Interval("P20000D") TO EXTERNAL DATA SOURCE `/Root/tier2` ON timestamp)"); { const TInstant start = Now(); bool check = false; diff --git a/ydb/core/wrappers/abstract.cpp b/ydb/core/wrappers/abstract.cpp index c13cd9bc6e0a..89f799b4ac90 100644 --- a/ydb/core/wrappers/abstract.cpp +++ b/ydb/core/wrappers/abstract.cpp @@ -9,7 +9,7 @@ IExternalStorageOperator::TPtr IExternalStorageConfig::ConstructStorageOperator( } IExternalStorageConfig::TPtr IExternalStorageConfig::Construct(const NKikimrSchemeOp::TS3Settings& settings) { - if (settings.GetEndpoint() == "fake") { + if (settings.GetEndpoint() == "fake.fake") { return std::make_shared(settings.GetBucket(), settings.GetSecretKey()); } else { return std::make_shared(settings); diff --git a/ydb/core/wrappers/fake_storage.h b/ydb/core/wrappers/fake_storage.h index fff953f8203f..472197a4eda6 100644 --- a/ydb/core/wrappers/fake_storage.h +++ b/ydb/core/wrappers/fake_storage.h @@ -138,7 +138,7 @@ class TFakeExternalStorageOperator: public IExternalStorageOperator { template void ExecuteImpl(TEvent& ev) const { ev->Get()->MutableRequest().WithBucket(Bucket); - Y_ABORT_UNLESS(SecretKey == Singleton()->GetSecretKey()); + Y_ABORT_UNLESS(SecretKey == Singleton()->GetSecretKey(), "%s != %s", SecretKey.data(), Singleton()->GetSecretKey().data()); if (OwnedStorage) { OwnedStorage->Execute(ev, ReplyAdapter); } else { From 41d2c69507e5bbd98a474b1f783e88c44d967ed1 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Thu, 14 Nov 2024 16:18:21 +0000 Subject: [PATCH 12/12] fix tests --- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 2 +- .../tx/schemeshard/olap/ttl/validator.cpp | 19 +- ydb/core/tx/schemeshard/olap/ttl/ya.make | 1 + ...__operation_alter_external_data_source.cpp | 8 + .../tx/schemeshard/schemeshard_info_types.h | 9 + .../schemeshard_path_describer.cpp | 7 +- ydb/core/tx/schemeshard/ya.make | 1 + ydb/core/tx/tiering/tier/object.cpp | 5 +- ydb/core/tx/tiering/tier/object.h | 8 +- ydb/core/tx/tiering/tier/ya.make | 3 +- ydb/core/tx/tiering/ut/ut_tiers.cpp | 25 +-- .../metadata/secret/accessor/secret_id.cpp | 24 +++ .../metadata/secret/accessor/secret_id.h | 169 ++++++++++++++++++ .../metadata/secret/accessor/snapshot.h | 17 ++ ydb/services/metadata/secret/accessor/ya.make | 17 ++ ydb/services/metadata/secret/secret.cpp | 16 -- ydb/services/metadata/secret/secret.h | 161 +---------------- ydb/services/metadata/secret/snapshot.h | 11 +- ydb/services/metadata/secret/ya.make | 1 + 19 files changed, 288 insertions(+), 216 deletions(-) create mode 100644 ydb/services/metadata/secret/accessor/secret_id.cpp create mode 100644 ydb/services/metadata/secret/accessor/secret_id.h create mode 100644 ydb/services/metadata/secret/accessor/snapshot.h create mode 100644 ydb/services/metadata/secret/accessor/ya.make diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 596586b2f2b0..efd86c42221d 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -8118,7 +8118,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.BulkUpsert(testTable, tableInserter); } - testHelper.SetTiering(tableName, "/root/tier1", "created_at"); + testHelper.SetTiering(tableName, "/Root/tier1", "created_at"); while (csController->GetTieringUpdates().Val() == 0) { Cout << "Wait tiering..." << Endl; diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp index 275f75ab4610..65a380398358 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace NKikimr::NSchemeShard { @@ -100,16 +101,26 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif } for (const auto& tier : ttl.GetTiers()) { - TPath tierPath = TPath::Resolve(tier.GetStorageName(), ctx); + const TString& tierPathString = tier.GetStorageName(); + TPath tierPath = TPath::Resolve(tierPathString, ctx); if (!tierPath.IsResolved() || tierPath.IsDeleted() || tierPath.IsUnderDeleting()) { - errors.AddError("Object not found: " + tier.GetStorageName()); + errors.AddError("Object not found: " + tierPathString); return false; } if (!tierPath->IsExternalDataSource()) { - errors.AddError("Not an external data source: " + tier.GetStorageName()); + errors.AddError("Not an external data source: " + tierPathString); return false; } - // TODO: check that TTierConfig is deserializable from external data source + { + auto* findExternalDataSource = ctx->ExternalDataSources.FindPtr(tierPath->PathId); + AFL_VERIFY(findExternalDataSource); + NKikimrSchemeOp::TExternalDataSourceDescription proto; + (*findExternalDataSource)->FillProto(proto); + if (auto status = NColumnShard::NTiers::TTierConfig().DeserializeFromProto(proto); status.IsFail()) { + errors.AddError("Cannot use external data source \"" + tierPathString + "\" for tiering: " + status.GetErrorMessage()); + return false; + } + } } return true; diff --git a/ydb/core/tx/schemeshard/olap/ttl/ya.make b/ydb/core/tx/schemeshard/olap/ttl/ya.make index f6c57de62a9f..3d84cb700fb2 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/ya.make +++ b/ydb/core/tx/schemeshard/olap/ttl/ya.make @@ -9,6 +9,7 @@ SRCS( PEERDIR( ydb/core/base ydb/core/protos + ydb/core/tx/tiering/tier ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_external_data_source.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_external_data_source.cpp index 1009c5dff470..52bf30f56463 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_external_data_source.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_external_data_source.cpp @@ -3,6 +3,8 @@ #include "schemeshard__operation_common.h" #include "schemeshard_impl.h" +#include + #include namespace { @@ -232,6 +234,12 @@ class TAlterExternalDataSource : public TSubOperation { RETURN_RESULT_UNLESS(IsDescriptionValid(result, externalDataSourceDescription, context.SS->ExternalSourceFactory)); + if (!context.SS->ColumnTables.GetTablesWithTier(dstPath.PathString()).empty()) { + if (auto status = NColumnShard::NTiers::TTierConfig().DeserializeFromProto(externalDataSourceDescription); status.IsFail()) { + result->SetError(NKikimrScheme::StatusInvalidParameter, "Cannot make this change while external data source is used as an OLAP tiered storage: " + status.GetErrorMessage()); + return result; + } + } const auto oldExternalDataSourceInfo = context.SS->ExternalDataSources.Value(dstPath->PathId, nullptr); diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index e4f571d4be39..12f0333b7c69 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3581,6 +3581,15 @@ struct TExternalDataSourceInfo: TSimpleRefCount { NKikimrSchemeOp::TAuth Auth; NKikimrSchemeOp::TExternalTableReferences ExternalTableReferences; NKikimrSchemeOp::TExternalDataSourceProperties Properties; + + void FillProto(NKikimrSchemeOp::TExternalDataSourceDescription& proto) const { + proto.SetVersion(AlterVersion); + proto.SetSourceType(SourceType); + proto.SetLocation(Location); + proto.SetInstallation(Installation); + proto.MutableAuth()->CopyFrom(Auth); + proto.MutableProperties()->CopyFrom(Properties); + } }; struct TViewInfo : TSimpleRefCount { diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 537812895079..cd2f71d552e9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -1061,12 +1061,7 @@ void TPathDescriber::DescribeExternalDataSource(const TActorContext&, TPathId pa auto entry = Result->Record.MutablePathDescription()->MutableExternalDataSourceDescription(); entry->SetName(pathEl->Name); PathIdFromPathId(pathId, entry->MutablePathId()); - entry->SetVersion(externalDataSourceInfo->AlterVersion); - entry->SetSourceType(externalDataSourceInfo->SourceType); - entry->SetLocation(externalDataSourceInfo->Location); - entry->SetInstallation(externalDataSourceInfo->Installation); - entry->MutableAuth()->CopyFrom(externalDataSourceInfo->Auth); - entry->MutableProperties()->CopyFrom(externalDataSourceInfo->Properties); + externalDataSourceInfo->FillProto(*entry); } void TPathDescriber::DescribeView(const TActorContext&, TPathId pathId, TPathElement::TPtr pathEl) { diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index bf252a10a9b8..58c5c8949b57 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -307,6 +307,7 @@ PEERDIR( yql/essentials/providers/common/proto ydb/services/bg_tasks ydb/core/tx/columnshard/bg_tasks/manager + ydb/core/tx/tiering/tier ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/tiering/tier/object.cpp b/ydb/core/tx/tiering/tier/object.cpp index 4aac45cd511a..c6e6f263a478 100644 --- a/ydb/core/tx/tiering/tier/object.cpp +++ b/ydb/core/tx/tiering/tier/object.cpp @@ -1,8 +1,5 @@ #include "object.h" -#include -#include - #include #include #include @@ -10,7 +7,7 @@ namespace NKikimr::NColumnShard::NTiers { NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig( - std::shared_ptr secrets) const { + const std::shared_ptr& secrets) const { auto config = ProtoConfig; if (secrets) { if (!secrets->GetSecretValue(GetAccessKey(), *config.MutableAccessKey())) { diff --git a/ydb/core/tx/tiering/tier/object.h b/ydb/core/tx/tiering/tier/object.h index 4a6a097ae42c..31c7bc9fead6 100644 --- a/ydb/core/tx/tiering/tier/object.h +++ b/ydb/core/tx/tiering/tier/object.h @@ -1,8 +1,10 @@ #pragma once -#include #include -#include +#include +#include +#include +#include #include @@ -47,7 +49,7 @@ class TTierConfig { NJson::TJsonValue SerializeConfigToJson() const; - NKikimrSchemeOp::TS3Settings GetPatchedConfig(std::shared_ptr secrets) const; + NKikimrSchemeOp::TS3Settings GetPatchedConfig(const std::shared_ptr& secrets) const; bool IsSame(const TTierConfig& item) const; NJson::TJsonValue GetDebugJson() const; diff --git a/ydb/core/tx/tiering/tier/ya.make b/ydb/core/tx/tiering/tier/ya.make index ca6842eab416..32f06144cbd8 100644 --- a/ydb/core/tx/tiering/tier/ya.make +++ b/ydb/core/tx/tiering/tier/ya.make @@ -5,7 +5,8 @@ SRCS( ) PEERDIR( - ydb/services/metadata/secret + ydb/library/conclusion + ydb/services/metadata/secret/accessor ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index bef6636a9c38..59f3ddf8a177 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -132,11 +132,9 @@ class TLocalHelper: public Tests::NCS::THelper { void CreateExternalDataSource(const TString& name, const TString& location = "http://fake.fake/fake") const { StartSchemaRequest(R"( - CREATE EXTERNAL DATA SOURCE `)" + - name + R"(` WITH ( + CREATE EXTERNAL DATA SOURCE `)" + name + R"(` WITH ( SOURCE_TYPE="ObjectStorage", - LOCATION=")" + - location + R"(", + LOCATION=")" + location + R"(", AUTH_METHOD="AWS", AWS_ACCESS_KEY_ID_SECRET_NAME="accessKey", AWS_SECRET_ACCESS_KEY_SECRET_NAME="secretKey", @@ -172,6 +170,9 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { } bool IsFound() const { + if (!Manager) { + return false; + } THashSet notFoundTiers = ExpectedTiers; for (const auto& [id, config] : Manager->GetTierConfigs()) { notFoundTiers.erase(id); @@ -253,14 +254,6 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc"); } Cerr << "Initialization finished" << Endl; - { - lHelper.StartSchemaRequest("ALTER EXTERNAL DATA SOURCE `/Root/tier1` SET ENDPOINT = \"fake.fake/abc1\""); - - TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); - runtime.Register(emulator); - emulator->CheckRuntime(runtime); - UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc1"); - } { lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", false); lHelper.StartSchemaRequest("DROP TABLE `/Root/olapStore/olapTable`"); @@ -339,7 +332,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { emulator->CheckRuntime(runtime); } lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier2`"); - lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", true, false); + lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`"); //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE); //runtime.SetLogPriority(NKikimrServices::KQP_YQL, NLog::PRI_TRACE); @@ -433,13 +426,13 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) { lHelper.CreateSecrets(); Singleton()->SetSecretKey("fakeSecret"); - lHelper.CreateExternalDataSource("/Root/tier1", TierEndpoint + "/fake"); - lHelper.CreateExternalDataSource("/Root/tier2", TierEndpoint + "/fake"); + lHelper.CreateExternalDataSource("/Root/tier1", "http://" + TierEndpoint + "/fake"); + lHelper.CreateExternalDataSource("/Root/tier2", "http://" + TierEndpoint + "/fake"); { TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" }); runtime.Register(emulator); emulator->CheckRuntime(runtime); - UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetEndpoint(), TierEndpoint); + UNIT_ASSERT_VALUES_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetEndpoint(), TierEndpoint); } lHelper.CreateTestOlapTable("olapTable", 2); diff --git a/ydb/services/metadata/secret/accessor/secret_id.cpp b/ydb/services/metadata/secret/accessor/secret_id.cpp new file mode 100644 index 000000000000..621dfef5dff4 --- /dev/null +++ b/ydb/services/metadata/secret/accessor/secret_id.cpp @@ -0,0 +1,24 @@ +#include "secret_id.h" + +#include +#include + +namespace NKikimr::NMetadata::NSecret { + +TString TSecretId::SerializeToString() const { + TStringBuilder sb; + sb << "USId:" << OwnerUserId << ":" << SecretId; + return sb; +} + + +TString TSecretIdOrValue::DebugString() const { + if (SecretId) { + return SecretId->SerializeToString(); + } else if (Value) { + return MD5::Calc(*Value); + } + return ""; +} + +} diff --git a/ydb/services/metadata/secret/accessor/secret_id.h b/ydb/services/metadata/secret/accessor/secret_id.h new file mode 100644 index 000000000000..010f6a59a78b --- /dev/null +++ b/ydb/services/metadata/secret/accessor/secret_id.h @@ -0,0 +1,169 @@ +#pragma once +#include + +#include + +namespace NKikimr::NMetadata::NSecret { + +class TSecretId { +private: + YDB_READONLY_PROTECT_DEF(TString, OwnerUserId); + YDB_READONLY_PROTECT_DEF(TString, SecretId); + +public: + TSecretId() = default; + TSecretId(const TString& ownerUserId, const TString& secretId) + : OwnerUserId(ownerUserId) + , SecretId(secretId) { + } + + TSecretId(const TStringBuf ownerUserId, const TStringBuf secretId) + : OwnerUserId(ownerUserId) + , SecretId(secretId) { + } + + TString SerializeToString() const; + + template + TString BuildSecretAccessString(const TProto& proto, const TString& defaultOwnerId) { + if (proto.HasValue()) { + return proto.GetValue(); + } else { + return TStringBuilder() << "USId:" << (proto.GetSecretOwnerId() ? proto.GetSecretOwnerId() : defaultOwnerId) << ":" << SecretId; + } + } + + bool operator<(const TSecretId& item) const { + return std::tie(OwnerUserId, SecretId) < std::tie(item.OwnerUserId, item.SecretId); + } + bool operator==(const TSecretId& item) const { + return std::tie(OwnerUserId, SecretId) == std::tie(item.OwnerUserId, item.SecretId); + } +}; + +class TSecretIdOrValue { +private: + YDB_READONLY_DEF(std::optional, SecretId); + YDB_READONLY_DEF(std::optional, Value); + TSecretIdOrValue() = default; + + bool DeserializeFromStringImpl(const TString& info, const TString& defaultUserId) { + static const TString prefixWithUser = "USId:"; + static const TString prefixNoUser = "SId:"; + if (info.StartsWith(prefixWithUser)) { + TStringBuf sb(info.data(), info.size()); + sb.Skip(prefixWithUser.size()); + TStringBuf uId; + TStringBuf sId; + if (!sb.TrySplit(':', uId, sId)) { + return false; + } + if (!uId || !sId) { + return false; + } + SecretId = TSecretId(uId, sId); + } else if (info.StartsWith(prefixNoUser)) { + TStringBuf sb(info.data(), info.size()); + sb.Skip(prefixNoUser.size()); + SecretId = TSecretId(defaultUserId, TString(sb)); + if (!sb || !defaultUserId) { + return false; + } + } else { + Value = info; + } + return true; + } + explicit TSecretIdOrValue(const TSecretId& id) + : SecretId(id) { + } + + explicit TSecretIdOrValue(const TString& value) + : Value(value) { + } + +public: + bool operator!() const { + return !Value && !SecretId; + } + + static TSecretIdOrValue BuildAsValue(const TString& value) { + return TSecretIdOrValue(value); + } + + static TSecretIdOrValue BuildEmpty() { + return TSecretIdOrValue(); + } + + static TSecretIdOrValue BuildAsId(const TSecretId& id) { + return TSecretIdOrValue(id); + } + + static std::optional DeserializeFromOptional( + const NKikimrSchemeOp::TSecretableVariable& proto, const TString& secretInfo, const TString& defaultOwnerId = Default()) { + if (proto.HasSecretId()) { + return DeserializeFromProto(proto, defaultOwnerId); + } else if (proto.HasValue()) { + return DeserializeFromString(proto.GetValue().GetData()); + } + if (secretInfo) { + return DeserializeFromString(secretInfo, defaultOwnerId); + } else { + return {}; + } + } + + NKikimrSchemeOp::TSecretableVariable SerializeToProto() const { + NKikimrSchemeOp::TSecretableVariable result; + if (SecretId) { + result.MutableSecretId()->SetId(SecretId->GetSecretId()); + result.MutableSecretId()->SetOwnerId(SecretId->GetOwnerUserId()); + } else if (Value) { + result.MutableValue()->SetData(*Value); + } + return result; + } + + static std::optional DeserializeFromProto( + const NKikimrSchemeOp::TSecretableVariable& proto, const TString& defaultOwnerId = Default()) { + if (proto.HasSecretId()) { + TString ownerId; + TString secretId; + if (!proto.GetSecretId().HasOwnerId() || !proto.GetSecretId().GetOwnerId()) { + ownerId = defaultOwnerId; + } else { + ownerId = proto.GetSecretId().GetOwnerId(); + } + secretId = proto.GetSecretId().GetId(); + if (!ownerId || !secretId) { + return {}; + } + return TSecretIdOrValue::BuildAsId(TSecretId(ownerId, secretId)); + } else if (proto.HasValue()) { + return TSecretIdOrValue::BuildAsValue(proto.GetValue().GetData()); + } else { + return {}; + } + } + + static std::optional DeserializeFromString(const TString& info, const TString& defaultOwnerId = Default()) { + TSecretIdOrValue result; + if (!result.DeserializeFromStringImpl(info, defaultOwnerId)) { + return {}; + } else { + return result; + } + } + + TString SerializeToString() const { + if (SecretId) { + return SecretId->SerializeToString(); + } else if (Value) { + return *Value; + } + return ""; + } + + TString DebugString() const; +}; +} // namespace NKikimr::NMetadata::NSecret diff --git a/ydb/services/metadata/secret/accessor/snapshot.h b/ydb/services/metadata/secret/accessor/snapshot.h new file mode 100644 index 000000000000..e7eff13fe96d --- /dev/null +++ b/ydb/services/metadata/secret/accessor/snapshot.h @@ -0,0 +1,17 @@ +#pragma once + +#include "secret_id.h" + +#include + +namespace NKikimr::NMetadata::NSecret { + +class ISecretAccessor { +public: + virtual bool CheckSecretAccess(const TSecretIdOrValue& sIdOrValue, const std::optional& userToken) const = 0; + virtual bool PatchString(TString& stringForPath) const = 0; + virtual bool GetSecretValue(const TSecretIdOrValue& secretId, TString& result) const = 0; + virtual std::vector GetSecretIds(const std::optional& userToken, const TString& secretId) const = 0; +}; + +} // namespace NKikimr::NMetadata::NSecret diff --git a/ydb/services/metadata/secret/accessor/ya.make b/ydb/services/metadata/secret/accessor/ya.make new file mode 100644 index 000000000000..ec96b677e70b --- /dev/null +++ b/ydb/services/metadata/secret/accessor/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +SRCS( + secret_id.cpp +) + +PEERDIR( + ydb/core/base + ydb/library/actors/core + ydb/library/aclib +) + +END() + +RECURSE_FOR_TESTS( + ut +) \ No newline at end of file diff --git a/ydb/services/metadata/secret/secret.cpp b/ydb/services/metadata/secret/secret.cpp index ec447c1d0ed6..ddab3fdaf3cd 100644 --- a/ydb/services/metadata/secret/secret.cpp +++ b/ydb/services/metadata/secret/secret.cpp @@ -32,20 +32,4 @@ IClassBehaviour::TPtr TSecret::GetBehaviour() { return TSecretBehaviour::GetInstance(); } -TString TSecretId::SerializeToString() const { - TStringBuilder sb; - sb << "USId:" << OwnerUserId << ":" << SecretId; - return sb; -} - - -TString TSecretIdOrValue::DebugString() const { - if (SecretId) { - return SecretId->SerializeToString(); - } else if (Value) { - return MD5::Calc(*Value); - } - return ""; -} - } diff --git a/ydb/services/metadata/secret/secret.h b/ydb/services/metadata/secret/secret.h index 70920091bf31..8cc936766c2d 100644 --- a/ydb/services/metadata/secret/secret.h +++ b/ydb/services/metadata/secret/secret.h @@ -5,169 +5,10 @@ #include #include #include +#include namespace NKikimr::NMetadata::NSecret { -class TSecretId { -private: - YDB_READONLY_PROTECT_DEF(TString, OwnerUserId); - YDB_READONLY_PROTECT_DEF(TString, SecretId); -public: - TSecretId() = default; - TSecretId(const TString& ownerUserId, const TString& secretId) - : OwnerUserId(ownerUserId) - , SecretId(secretId) { - } - - TSecretId(const TStringBuf ownerUserId, const TStringBuf secretId) - : OwnerUserId(ownerUserId) - , SecretId(secretId) { - } - - TString SerializeToString() const; - - template - TString BuildSecretAccessString(const TProto& proto, const TString& defaultOwnerId) { - if (proto.HasValue()) { - return proto.GetValue(); - } else { - return TStringBuilder() << "USId:" << (proto.GetSecretOwnerId() ? proto.GetSecretOwnerId() : defaultOwnerId) << ":" << SecretId; - } - } - - bool operator<(const TSecretId& item) const { - return std::tie(OwnerUserId, SecretId) < std::tie(item.OwnerUserId, item.SecretId); - } - bool operator==(const TSecretId& item) const { - return std::tie(OwnerUserId, SecretId) == std::tie(item.OwnerUserId, item.SecretId); - } -}; - -class TSecretIdOrValue { -private: - YDB_READONLY_DEF(std::optional, SecretId); - YDB_READONLY_DEF(std::optional, Value); - TSecretIdOrValue() = default; - - bool DeserializeFromStringImpl(const TString& info, const TString& defaultUserId) { - static const TString prefixWithUser = "USId:"; - static const TString prefixNoUser = "SId:"; - if (info.StartsWith(prefixWithUser)) { - TStringBuf sb(info.data(), info.size()); - sb.Skip(prefixWithUser.size()); - TStringBuf uId; - TStringBuf sId; - if (!sb.TrySplit(':', uId, sId)) { - return false; - } - if (!uId || !sId) { - return false; - } - SecretId = TSecretId(uId, sId); - } else if (info.StartsWith(prefixNoUser)) { - TStringBuf sb(info.data(), info.size()); - sb.Skip(prefixNoUser.size()); - SecretId = TSecretId(defaultUserId, TString(sb)); - if (!sb || !defaultUserId) { - return false; - } - } else { - Value = info; - } - return true; - } - explicit TSecretIdOrValue(const TSecretId& id) - : SecretId(id) { - - } - - explicit TSecretIdOrValue(const TString& value) - : Value(value) { - - } - -public: - bool operator!() const { - return !Value && !SecretId; - } - - static TSecretIdOrValue BuildAsValue(const TString& value) { - return TSecretIdOrValue(value); - } - - static TSecretIdOrValue BuildEmpty() { - return TSecretIdOrValue(); - } - - static TSecretIdOrValue BuildAsId(const TSecretId& id) { - return TSecretIdOrValue(id); - } - - static std::optional DeserializeFromOptional(const NKikimrSchemeOp::TSecretableVariable& proto, const TString& secretInfo, const TString& defaultOwnerId = Default()) { - if (proto.HasSecretId()) { - return DeserializeFromProto(proto, defaultOwnerId); - } else if (proto.HasValue()) { - return DeserializeFromString(proto.GetValue().GetData()); - } if (secretInfo) { - return DeserializeFromString(secretInfo, defaultOwnerId); - } else { - return {}; - } - } - - NKikimrSchemeOp::TSecretableVariable SerializeToProto() const { - NKikimrSchemeOp::TSecretableVariable result; - if (SecretId) { - result.MutableSecretId()->SetId(SecretId->GetSecretId()); - result.MutableSecretId()->SetOwnerId(SecretId->GetOwnerUserId()); - } else if (Value) { - result.MutableValue()->SetData(*Value); - } - return result; - } - - static std::optional DeserializeFromProto(const NKikimrSchemeOp::TSecretableVariable& proto, const TString& defaultOwnerId = Default()) { - if (proto.HasSecretId()) { - TString ownerId; - TString secretId; - if (!proto.GetSecretId().HasOwnerId() || !proto.GetSecretId().GetOwnerId()) { - ownerId = defaultOwnerId; - } else { - ownerId = proto.GetSecretId().GetOwnerId(); - } - secretId = proto.GetSecretId().GetId(); - if (!ownerId || !secretId) { - return {}; - } - return TSecretIdOrValue::BuildAsId(TSecretId(ownerId, secretId)); - } else if (proto.HasValue()) { - return TSecretIdOrValue::BuildAsValue(proto.GetValue().GetData()); - } else { - return {}; - } - } - - static std::optional DeserializeFromString(const TString& info, const TString& defaultOwnerId = Default()) { - TSecretIdOrValue result; - if (!result.DeserializeFromStringImpl(info, defaultOwnerId)) { - return {}; - } else { - return result; - } - } - - TString SerializeToString() const { - if (SecretId) { - return SecretId->SerializeToString(); - } else if (Value) { - return *Value; - } - return ""; - } - - TString DebugString() const; -}; - class TSecret: public TSecretId, public NModifications::TObject { private: using TBase = TSecretId; diff --git a/ydb/services/metadata/secret/snapshot.h b/ydb/services/metadata/secret/snapshot.h index c216aa73dee2..1cd892efdf74 100644 --- a/ydb/services/metadata/secret/snapshot.h +++ b/ydb/services/metadata/secret/snapshot.h @@ -3,11 +3,12 @@ #include "access.h" #include +#include #include namespace NKikimr::NMetadata::NSecret { -class TSnapshot: public NFetcher::ISnapshot { +class TSnapshot: public NFetcher::ISnapshot, public ISecretAccessor { private: using TBase = NFetcher::ISnapshot; using TSecrets = std::map; @@ -22,10 +23,10 @@ class TSnapshot: public NFetcher::ISnapshot { virtual TString DoSerializeToString() const override; public: using TBase::TBase; - bool CheckSecretAccess(const TSecretIdOrValue& sIdOrValue, const std::optional& userToken) const; - bool PatchString(TString& stringForPath) const; - bool GetSecretValue(const TSecretIdOrValue& secretId, TString& result) const; - std::vector GetSecretIds(const std::optional& userToken, const TString& secretId) const; + bool CheckSecretAccess(const TSecretIdOrValue& sIdOrValue, const std::optional& userToken) const override; + bool PatchString(TString& stringForPath) const override; + bool GetSecretValue(const TSecretIdOrValue& secretId, TString& result) const override; + std::vector GetSecretIds(const std::optional& userToken, const TString& secretId) const override; }; } diff --git a/ydb/services/metadata/secret/ya.make b/ydb/services/metadata/secret/ya.make index e44e3e3152dc..6cfd391a7365 100644 --- a/ydb/services/metadata/secret/ya.make +++ b/ydb/services/metadata/secret/ya.make @@ -21,6 +21,7 @@ PEERDIR( ydb/core/grpc_services/base ydb/core/grpc_services ydb/services/metadata/request + ydb/services/metadata/secret/accessor ) END()