fix tests

ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp

@@ -8118,7 +8118,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) {
             testHelper.BulkUpsert(testTable, tableInserter);
         }
 
-        testHelper.SetTiering(tableName, "/root/tier1", "created_at");
+        testHelper.SetTiering(tableName, "/Root/tier1", "created_at");
 
         while (csController->GetTieringUpdates().Val() == 0) {
             Cout << "Wait tiering..." << Endl;

ydb/core/tx/schemeshard/olap/ttl/validator.cpp

@@ -2,6 +2,7 @@
 
 #include <ydb/core/tx/schemeshard/common/validation.h>
 #include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+#include <ydb/core/tx/tiering/tier/object.h>
 
 namespace NKikimr::NSchemeShard {
 
@@ -100,16 +101,26 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif
     }
 
     for (const auto& tier : ttl.GetTiers()) {
-        TPath tierPath = TPath::Resolve(tier.GetStorageName(), ctx);
+        const TString& tierPathString = tier.GetStorageName();
+        TPath tierPath = TPath::Resolve(tierPathString, ctx);
         if (!tierPath.IsResolved() || tierPath.IsDeleted() || tierPath.IsUnderDeleting()) {
-            errors.AddError("Object not found: " + tier.GetStorageName());
+            errors.AddError("Object not found: " + tierPathString);
             return false;
         }
         if (!tierPath->IsExternalDataSource()) {
-            errors.AddError("Not an external data source: " + tier.GetStorageName());
+            errors.AddError("Not an external data source: " + tierPathString);
             return false;
         }
-        // TODO: check that TTierConfig is deserializable from external data source
+        {
+            auto* findExternalDataSource = ctx->ExternalDataSources.FindPtr(tierPath->PathId);
+            AFL_VERIFY(findExternalDataSource);
+            NKikimrSchemeOp::TExternalDataSourceDescription proto;
+            (*findExternalDataSource)->FillProto(proto);
+            if (auto status = NColumnShard::NTiers::TTierConfig().DeserializeFromProto(proto); status.IsFail()) {
+                errors.AddError("Cannot use external data source \"" + tierPathString + "\" for tiering: " + status.GetErrorMessage());
+                return false;
+            }
+        }
     }
 
     return true;

ydb/core/tx/schemeshard/olap/ttl/ya.make

@@ -9,6 +9,7 @@ SRCS(
 PEERDIR(
     ydb/core/base
     ydb/core/protos
+    ydb/core/tx/tiering/tier
 )
 
 YQL_LAST_ABI_VERSION()

ydb/core/tx/schemeshard/schemeshard__operation_alter_external_data_source.cpp

@@ -3,6 +3,8 @@
 #include "schemeshard__operation_common.h"
 #include "schemeshard_impl.h"
 
+#include <ydb/core/tx/tiering/tier/object.h>
+
 #include <utility>
 
 namespace {
@@ -232,6 +234,12 @@ class TAlterExternalDataSource : public TSubOperation {
         RETURN_RESULT_UNLESS(IsDescriptionValid(result,
                                 externalDataSourceDescription,
                                 context.SS->ExternalSourceFactory));
+        if (!context.SS->ColumnTables.GetTablesWithTier(dstPath.PathString()).empty()) {
+            if (auto status = NColumnShard::NTiers::TTierConfig().DeserializeFromProto(externalDataSourceDescription); status.IsFail()) {
+                result->SetError(NKikimrScheme::StatusInvalidParameter, "Cannot make this change while external data source is used as an OLAP tiered storage: " + status.GetErrorMessage());
+                return result;
+            }
+        }
 
         const auto oldExternalDataSourceInfo =
         context.SS->ExternalDataSources.Value(dstPath->PathId, nullptr);

ydb/core/tx/schemeshard/schemeshard_info_types.h

@@ -3581,6 +3581,15 @@ struct TExternalDataSourceInfo: TSimpleRefCount<TExternalDataSourceInfo> {
     NKikimrSchemeOp::TAuth Auth;
     NKikimrSchemeOp::TExternalTableReferences ExternalTableReferences;
     NKikimrSchemeOp::TExternalDataSourceProperties Properties;
+
+    void FillProto(NKikimrSchemeOp::TExternalDataSourceDescription& proto) const {
+        proto.SetVersion(AlterVersion);
+        proto.SetSourceType(SourceType);
+        proto.SetLocation(Location);
+        proto.SetInstallation(Installation);
+        proto.MutableAuth()->CopyFrom(Auth);
+        proto.MutableProperties()->CopyFrom(Properties);
+    }
 };
 
 struct TViewInfo : TSimpleRefCount<TViewInfo> {

ydb/core/tx/schemeshard/schemeshard_path_describer.cpp

@@ -1061,12 +1061,7 @@ void TPathDescriber::DescribeExternalDataSource(const TActorContext&, TPathId pa
     auto entry = Result->Record.MutablePathDescription()->MutableExternalDataSourceDescription();
     entry->SetName(pathEl->Name);
     PathIdFromPathId(pathId, entry->MutablePathId());
-    entry->SetVersion(externalDataSourceInfo->AlterVersion);
-    entry->SetSourceType(externalDataSourceInfo->SourceType);
-    entry->SetLocation(externalDataSourceInfo->Location);
-    entry->SetInstallation(externalDataSourceInfo->Installation);
-    entry->MutableAuth()->CopyFrom(externalDataSourceInfo->Auth);
-    entry->MutableProperties()->CopyFrom(externalDataSourceInfo->Properties);
+    externalDataSourceInfo->FillProto(*entry);
 }
 
 void TPathDescriber::DescribeView(const TActorContext&, TPathId pathId, TPathElement::TPtr pathEl) {

ydb/core/tx/schemeshard/ya.make

@@ -307,6 +307,7 @@ PEERDIR(
     yql/essentials/providers/common/proto
     ydb/services/bg_tasks
     ydb/core/tx/columnshard/bg_tasks/manager
+    ydb/core/tx/tiering/tier
 )
 
 YQL_LAST_ABI_VERSION()

ydb/core/tx/tiering/tier/object.cpp

@@ -1,16 +1,13 @@
 #include "object.h"
 
-#include <ydb/services/metadata/manager/ydb_value_operator.h>
-#include <ydb/services/metadata/secret/fetcher.h>
-
 #include <library/cpp/json/writer/json_value.h>
 #include <library/cpp/protobuf/json/proto2json.h>
 #include <library/cpp/uri/uri.h>
 
 namespace NKikimr::NColumnShard::NTiers {
 
 NKikimrSchemeOp::TS3Settings TTierConfig::GetPatchedConfig(
-    std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) const {
+    const std::shared_ptr<NMetadata::NSecret::ISecretAccessor>& secrets) const {
     auto config = ProtoConfig;
     if (secrets) {
         if (!secrets->GetSecretValue(GetAccessKey(), *config.MutableAccessKey())) {

ydb/core/tx/tiering/tier/object.h

@@ -1,8 +1,10 @@
 #pragma once
-#include <ydb/library/accessor/accessor.h>
 #include <ydb/core/protos/flat_scheme_op.pb.h>
 
-#include <ydb/services/metadata/secret/secret.h>
+#include <ydb/library/accessor/accessor.h>
+#include <ydb/library/conclusion/status.h>
+#include <ydb/services/metadata/secret/accessor/secret_id.h>
+#include <ydb/services/metadata/secret/accessor/snapshot.h>
 
 #include <library/cpp/json/writer/json_value.h>
 
@@ -47,7 +49,7 @@ class TTierConfig {
 
     NJson::TJsonValue SerializeConfigToJson() const;
 
-    NKikimrSchemeOp::TS3Settings GetPatchedConfig(std::shared_ptr<NMetadata::NSecret::TSnapshot> secrets) const;
+    NKikimrSchemeOp::TS3Settings GetPatchedConfig(const std::shared_ptr<NMetadata::NSecret::ISecretAccessor>& secrets) const;
 
     bool IsSame(const TTierConfig& item) const;
     NJson::TJsonValue GetDebugJson() const;

ydb/core/tx/tiering/tier/ya.make

@@ -5,7 +5,8 @@ SRCS(
 )
 
 PEERDIR(
-    ydb/services/metadata/secret
+    ydb/library/conclusion
+    ydb/services/metadata/secret/accessor
 )
 
 YQL_LAST_ABI_VERSION()

ydb/core/tx/tiering/ut/ut_tiers.cpp

@@ -132,11 +132,9 @@ class TLocalHelper: public Tests::NCS::THelper {
 
     void CreateExternalDataSource(const TString& name, const TString& location = "http://fake.fake/fake") const {
         StartSchemaRequest(R"(
-            CREATE EXTERNAL DATA SOURCE `)" +
-                           name + R"(` WITH (
+            CREATE EXTERNAL DATA SOURCE `)" + name + R"(` WITH (
                 SOURCE_TYPE="ObjectStorage",
-                LOCATION=")" +
-                           location + R"(",
+                LOCATION=")" + location + R"(",
                 AUTH_METHOD="AWS",
                 AWS_ACCESS_KEY_ID_SECRET_NAME="accessKey",
                 AWS_SECRET_ACCESS_KEY_SECRET_NAME="secretKey",
@@ -172,6 +170,9 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
         }
 
         bool IsFound() const {
+            if (!Manager) {
+                return false;
+            }
             THashSet notFoundTiers = ExpectedTiers;
             for (const auto& [id, config] : Manager->GetTierConfigs()) {
                 notFoundTiers.erase(id);
@@ -253,14 +254,6 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
                 UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc");
             }
             Cerr << "Initialization finished" << Endl;
-            {
-                lHelper.StartSchemaRequest("ALTER EXTERNAL DATA SOURCE `/Root/tier1` SET ENDPOINT = \"fake.fake/abc1\"");
-
-                TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" });
-                runtime.Register(emulator);
-                emulator->CheckRuntime(runtime);
-                UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetBucket(), "abc1");
-            }
             {
                 lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", false);
                 lHelper.StartSchemaRequest("DROP TABLE `/Root/olapStore/olapTable`");
@@ -339,7 +332,7 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
             emulator->CheckRuntime(runtime);
         }
         lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier2`");
-        lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`", true, false);
+        lHelper.StartSchemaRequest("DROP EXTERNAL DATA SOURCE `/Root/tier1`");
 
         //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_TRACE);
         //runtime.SetLogPriority(NKikimrServices::KQP_YQL, NLog::PRI_TRACE);
@@ -433,13 +426,13 @@ Y_UNIT_TEST_SUITE(ColumnShardTiers) {
         lHelper.CreateSecrets();
         Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->SetSecretKey("fakeSecret");
 
-        lHelper.CreateExternalDataSource("/Root/tier1", TierEndpoint + "/fake");
-        lHelper.CreateExternalDataSource("/Root/tier2", TierEndpoint + "/fake");
+        lHelper.CreateExternalDataSource("/Root/tier1", "http://" + TierEndpoint + "/fake");
+        lHelper.CreateExternalDataSource("/Root/tier2", "http://" + TierEndpoint + "/fake");
         {
             TTestCSEmulator* emulator = new TTestCSEmulator({ "/Root/tier1", "/Root/tier2" });
             runtime.Register(emulator);
             emulator->CheckRuntime(runtime);
-            UNIT_ASSERT_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetEndpoint(), TierEndpoint);
+            UNIT_ASSERT_VALUES_EQUAL(emulator->GetTierConfigs().at("/Root/tier1").GetProtoConfig().GetEndpoint(), TierEndpoint);
         }
 
         lHelper.CreateTestOlapTable("olapTable", 2);

ydb/services/metadata/secret/accessor/secret_id.cpp

@@ -0,0 +1,24 @@
+#include "secret_id.h"
+
+#include <ydb/services/metadata/manager/ydb_value_operator.h>
+#include <library/cpp/digest/md5/md5.h>
+
+namespace NKikimr::NMetadata::NSecret {
+
+TString TSecretId::SerializeToString() const {
+    TStringBuilder sb;
+    sb << "USId:" << OwnerUserId << ":" << SecretId;
+    return sb;
+}
+
+
+TString TSecretIdOrValue::DebugString() const {
+    if (SecretId) {
+        return SecretId->SerializeToString();
+    } else if (Value) {
+        return MD5::Calc(*Value);
+    }
+    return "";
+}
+
+}