fix parsing of s3 settings on CS (ydb-platform#12856)

zverevgeny · Jan 21, 2025 · d0eec48 · d0eec48
1 parent 3e06721
commit d0eec48
Show file tree

Hide file tree

Showing 3 changed files with 88 additions and 4 deletions.
diff --git a/ydb/core/tx/tiering/tier/object.cpp b/ydb/core/tx/tiering/tier/object.cpp
@@ -37,9 +37,14 @@ TConclusionStatus TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExte
         return TConclusionStatus::Fail("AWS auth is not defined for storage tier");
     }
 
-    ProtoConfig.SetAccessKey(NMetadata::NSecret::TSecretName(proto.GetAuth().GetAws().GetAwsAccessKeyIdSecretName()).SerializeToString());
-    ProtoConfig.SetSecretKey(
-        NMetadata::NSecret::TSecretName(proto.GetAuth().GetAws().GetAwsSecretAccessKeySecretName()).SerializeToString());
+    {
+        const auto aws = proto.GetAuth().GetAws();
+        ProtoConfig.SetAccessKey(NMetadata::NSecret::TSecretName(aws.GetAwsAccessKeyIdSecretName()).SerializeToString());
+        ProtoConfig.SetSecretKey(NMetadata::NSecret::TSecretName(aws.GetAwsSecretAccessKeySecretName()).SerializeToString());
+        if (aws.HasAwsRegion()) {
+            ProtoConfig.SetRegion(aws.GetAwsRegion());
+        }
+    }
 
     NUri::TUri url;
     if (url.Parse(proto.GetLocation(), NUri::TFeature::FeaturesAll) != NUri::TState::EParsed::ParsedOK) {
@@ -78,7 +83,11 @@ TConclusionStatus TTierConfig::DeserializeFromProto(const NKikimrSchemeOp::TExte
             }
         }
 
-        ProtoConfig.SetEndpoint(TString(endpoint));
+        if (url.GetField(NUri::TField::FieldPort)) {
+            ProtoConfig.SetEndpoint(TStringBuilder() << endpoint << ":" << url.GetPort());
+        } else {
+            ProtoConfig.SetEndpoint(TString(endpoint));
+        }
         ProtoConfig.SetBucket(TString(bucket));
     }
 

diff --git a/ydb/core/tx/tiering/ut/ut_object.cpp b/ydb/core/tx/tiering/ut/ut_object.cpp
@@ -0,0 +1,74 @@
+#include <ydb/core/tx/tiering/tier/object.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+
+using namespace NColumnShard;
+
+Y_UNIT_TEST_SUITE(S3SettingsConvertion) {
+    void ValidateConversion(
+        const NKikimrSchemeOp::TExternalDataSourceDescription& input, TConclusion<const NKikimrSchemeOp::TS3Settings> expectedResult) {
+        NTiers::TTierConfig config;
+        TConclusionStatus status = config.DeserializeFromProto(input);
+        if (expectedResult.IsFail()) {
+            UNIT_ASSERT(status.IsFail());
+            UNIT_ASSERT_VALUES_EQUAL(expectedResult.GetErrorMessage(), status.GetErrorMessage());
+        } else {
+            UNIT_ASSERT(status.IsSuccess());
+            UNIT_ASSERT_VALUES_EQUAL(expectedResult->DebugString(), config.GetProtoConfig().DebugString());
+        }
+    }
+
+    Y_UNIT_TEST(Basic) {
+        NKikimrSchemeOp::TExternalDataSourceDescription input;
+        UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"(
+            SourceType: "ObjectStorage"
+            Location: "http://storage.yandexcloud.net/my-bucket"
+            Auth {
+                Aws {
+                    AwsAccessKeyIdSecretName: "access-key"
+                    AwsSecretAccessKeySecretName: "secret-key"
+                    AwsRegion: "ru-central1"
+                }
+            }
+        )", &input));
+        NKikimrSchemeOp::TS3Settings output;
+        UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"(
+            Scheme: HTTP
+            Endpoint: "storage.yandexcloud.net"
+            Bucket: "my-bucket"
+            SecretKey: "SId:secret-key"
+            AccessKey: "SId:access-key"
+            Region: "ru-central1"
+        )", &output));
+        ValidateConversion(input, output);
+    }
+
+    Y_UNIT_TEST(Port) {
+        NKikimrSchemeOp::TExternalDataSourceDescription input;
+        UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"(
+            SourceType: "ObjectStorage"
+            Location: "http://storage.yandexcloud.net:12345/my-bucket"
+            Auth {
+                Aws {
+                    AwsAccessKeyIdSecretName: "access-key"
+                    AwsSecretAccessKeySecretName: "secret-key"
+                    AwsRegion: "ru-central1"
+                }
+            }
+        )", &input));
+        NKikimrSchemeOp::TS3Settings output;
+        UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"(
+            Scheme: HTTP
+            Endpoint: "storage.yandexcloud.net:12345"
+            Bucket: "my-bucket"
+            SecretKey: "SId:secret-key"
+            AccessKey: "SId:access-key"
+            Region: "ru-central1"
+        )", &output));
+        ValidateConversion(input, output);
+    }
+}
+
+}   // namespace NKikimr
diff --git a/ydb/core/tx/tiering/ut/ya.make b/ydb/core/tx/tiering/ut/ya.make
@@ -30,6 +30,7 @@ YQL_LAST_ABI_VERSION()
 
 SRCS(
     ut_tiers.cpp
+    ut_object.cpp
 )
 
 END()
-Original file line number
+Diff line change
@@ Expand Up / @@ -30,6 +30,7 @@ YQL_LAST_ABI_VERSION() @@
     SRCS(
         ut_tiers.cpp
+        ut_object.cpp
     )
     END()