Merge 00816fc into 52d9c30

ydb-platform · Aug 15, 2024 · 998e11e · 998e11e
2 parents 52d9c30 + 00816fc
commit 998e11e
Show file tree

Hide file tree

Showing 7 changed files with 14 additions and 96 deletions.
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
@@ -2072,7 +2072,7 @@ std::pair<NYql::NDq::IDqComputeActorAsyncInput*, IActor*> CreateS3ReadActor(
     const IFunctionRegistry& functionRegistry = *holderFactory.GetFunctionRegistry();
 
     TPathList paths;
-    ReadPathsList(params, taskParams, readRanges, paths);
+    ReadPathsList(taskParams, readRanges, paths);
 
     const auto token = secureParams.Value(params.GetToken(), TString{});
     const TS3Credentials credentials(credentialsFactory, token);

diff --git a/ydb/library/yql/providers/s3/proto/source.proto b/ydb/library/yql/providers/s3/proto/source.proto
@@ -13,7 +13,7 @@ message TPath {
 message TSource {
     string Url = 1;
     string Token = 2;
-    repeated TPath DeprecatedPath = 3; // deprecated
+    reserved 3;
     optional string RowType = 4;
     optional string Format = 5;
     map<string, string> Settings = 6;

diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp
@@ -479,7 +479,7 @@ class TS3DqIntegration: public TDqIntegrationBase {
                 range.Save(&out);
 
                 paths.clear();
-                ReadPathsList(srcDesc, {}, serialized, paths);
+                ReadPathsList({}, serialized, paths);
 
                 const NDq::TS3ReadActorFactoryConfig& readActorConfig = State_->Configuration->S3ReadActorFactoryConfig;
                 ui64 fileSizeLimit = readActorConfig.FileSizeLimit;

diff --git a/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp b/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp
@@ -107,7 +107,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
         b.Save(&range);
 
         TPathList paths;
-        ReadPathsList({}, MakeParams(range), {}, paths);
+        ReadPathsList(MakeParams(range), {}, paths);
 
         UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1);
         UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "name");
@@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
         b.Save(&range);
 
         TPathList paths;
-        ReadPathsList({}, MakeParams(range), {}, paths);
+        ReadPathsList(MakeParams(range), {}, paths);
 
         UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1);
         UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "a///b");
@@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
         b.Save(&range);
 
         TPathList paths;
-        ReadPathsList({}, MakeParams(range), {}, paths);
+        ReadPathsList(MakeParams(range), {}, paths);
 
         UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2);
         UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "root/name/");
@@ -165,7 +165,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
         b.Save(&range);
 
         TPathList paths;
-        ReadPathsList({}, MakeParams(range), {}, paths);
+        ReadPathsList(MakeParams(range), {}, paths);
 
         UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2);
 
@@ -190,7 +190,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
         b.Save(&range);
 
         TPathList paths;
-        ReadPathsList({}, MakeParams(range), {}, paths);
+        ReadPathsList(MakeParams(range), {}, paths);
 
         UNIT_ASSERT_VALUES_EQUAL(paths.size(), 3);
 

diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp
@@ -38,7 +38,7 @@ static void BuildPathsFromTree(const google::protobuf::RepeatedPtrField<NYql::NS
     BuildPathsFromTree(children, paths, currentPath, currentDepth, nextPathIndex);
 }
 
-void DecodeS3Range(const NS3::TSource& sourceDesc, const TString& data, TPathList& paths) {
+void DecodeS3Range(const TString& data, TPathList& paths) {
     NS3::TRange range;
     TStringInput input(data);
     range.Load(&input);
@@ -49,35 +49,15 @@ void DecodeS3Range(const NS3::TSource& sourceDesc, const TString& data, TPathLis
         TString buf;
         return BuildPathsFromTree(range.GetPaths(), paths, buf, 0, startPathIndex);
     }
-
-    std::unordered_map<TString, size_t> map(sourceDesc.GetDeprecatedPath().size());
-    for (auto i = 0; i < sourceDesc.GetDeprecatedPath().size(); ++i) {
-        map.emplace(sourceDesc.GetDeprecatedPath().Get(i).GetPath(), sourceDesc.GetDeprecatedPath().Get(i).GetSize());
-    }
-
-    for (auto i = 0; i < range.GetDeprecatedPath().size(); ++i) {
-        const auto& path = range.GetDeprecatedPath().Get(i);
-        auto it = map.find(path);
-        YQL_ENSURE(it != map.end());
-        paths.emplace_back(TPath{path, it->second, false, i + startPathIndex});
-    }
 }
 
-void ReadPathsList(const NS3::TSource& sourceDesc, const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths) {
+void ReadPathsList(const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths) {
     if (!readRanges.empty()) {
         for (auto readRange : readRanges) {
-            DecodeS3Range(sourceDesc, readRange, paths);
+            DecodeS3Range(readRange, paths);
         }
     } else if (const auto taskParamsIt = taskParams.find(S3ProviderName); taskParamsIt != taskParams.cend()) {
-        DecodeS3Range(sourceDesc, taskParamsIt->second, paths);
-    } else {
-        for (auto i = 0; i < sourceDesc.GetDeprecatedPath().size(); ++i) {
-            paths.emplace_back(TPath{
-                sourceDesc.GetDeprecatedPath().Get(i).GetPath(),
-                sourceDesc.GetDeprecatedPath().Get(i).GetSize(),
-                false,
-                static_cast<ui64>(i)});
-        }
+        DecodeS3Range(taskParamsIt->second, paths);
     }
 }
 

diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.h b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.h
@@ -23,7 +23,7 @@ struct TPath {
 };
 using TPathList = std::vector<TPath>;
 
-void ReadPathsList(const NS3::TSource& sourceDesc, const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths);
+void ReadPathsList(const THashMap<TString, TString>& taskParams, const TVector<TString>& readRanges, TPathList& paths);
 
 void PackPathsList(const TPathList& paths, TString& packed, bool& isTextEncoded);
 void UnpackPathsList(TStringBuf packed, bool isTextEncoded, TPathList& paths);

diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp b/ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp
@@ -18,63 +18,6 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) {
         return map;
     }
 
-    Y_UNIT_TEST(ReadsFilesListFromSourceSettings) {
-        NS3::TSource src;
-        {
-            auto* p = src.AddDeprecatedPath();
-            p->SetPath("my/path");
-            p->SetSize(100500);
-        }
-        {
-            auto* p = src.AddDeprecatedPath();
-            p->SetPath("other/path");
-            p->SetSize(1);
-        }
-
-        TPathList paths;
-        ReadPathsList(src, {}, {}, paths);
-
-        UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2);
-
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "my/path");
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 100500);
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, false);
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 0);
-
-        UNIT_ASSERT_VALUES_EQUAL(paths[1].Path, "other/path");
-        UNIT_ASSERT_VALUES_EQUAL(paths[1].Size, 1);
-        UNIT_ASSERT_VALUES_EQUAL(paths[1].IsDirectory, false);
-        UNIT_ASSERT_VALUES_EQUAL(paths[1].PathIndex, 1);
-    }
-
-    Y_UNIT_TEST(ReadsFilesListFromParamsAndSourceSettings) {
-        NS3::TSource src;
-        {
-            auto* p = src.AddDeprecatedPath();
-            p->SetPath("my/path");
-            p->SetSize(100500);
-        }
-        {
-            auto* p = src.AddDeprecatedPath();
-            p->SetPath("other/path");
-            p->SetSize(1);
-        }
-
-        NS3::TRange range;
-        range.SetStartPathIndex(42);
-        range.AddDeprecatedPath("my/path");
-
-        TPathList paths;
-        ReadPathsList(src, MakeParams(range), {}, paths);
-
-        UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1);
-
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "my/path");
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 100500);
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, false);
-        UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 42);
-    }
-
     NYql::NS3::TRange::TPath* SetPath(NYql::NS3::TRange::TPath* path, const TString& name = {}, ui64 size = 0, bool read = false) {
         path->SetName(name);
         path->SetSize(size);
@@ -84,11 +27,6 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) {
 
     Y_UNIT_TEST(ReadsFilesListFromTreeParams) {
         NS3::TSource src;
-        {
-            auto* p = src.AddDeprecatedPath();
-            p->SetPath("my/path");
-            p->SetSize(100500);
-        }
 
         NS3::TRange range;
         range.SetStartPathIndex(42);
@@ -110,7 +48,7 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) {
         }
 
         TPathList paths;
-        ReadPathsList(src, MakeParams(range), {}, paths);
+        ReadPathsList(MakeParams(range), {}, paths);
 
         UNIT_ASSERT_VALUES_EQUAL(paths.size(), 5);