From 2e701ce80a56d8c5b456beda4207afcf017f4c3e Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Fri, 16 Aug 2024 16:33:33 +0300 Subject: [PATCH] Removed deprecated proto field (#7842) --- .../providers/s3/actors/yql_s3_read_actor.cpp | 2 +- .../yql/providers/s3/proto/source.proto | 2 +- .../s3/provider/yql_s3_dq_integration.cpp | 2 +- .../s3/range_helpers/file_tree_builder_ut.cpp | 10 +-- .../s3/range_helpers/path_list_reader.cpp | 28 ++------ .../s3/range_helpers/path_list_reader.h | 2 +- .../s3/range_helpers/path_list_reader_ut.cpp | 64 +------------------ 7 files changed, 14 insertions(+), 96 deletions(-) diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp index 6c3f66b16bed..25bf1a02ff21 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp @@ -2073,7 +2073,7 @@ std::pair CreateS3ReadActor( const IFunctionRegistry& functionRegistry = *holderFactory.GetFunctionRegistry(); TPathList paths; - ReadPathsList(params, taskParams, readRanges, paths); + ReadPathsList(taskParams, readRanges, paths); const auto token = secureParams.Value(params.GetToken(), TString{}); const auto authInfo = GetAuthInfo(credentialsFactory, token); diff --git a/ydb/library/yql/providers/s3/proto/source.proto b/ydb/library/yql/providers/s3/proto/source.proto index 225a0720d762..e21ead1cab0b 100644 --- a/ydb/library/yql/providers/s3/proto/source.proto +++ b/ydb/library/yql/providers/s3/proto/source.proto @@ -13,7 +13,7 @@ message TPath { message TSource { string Url = 1; string Token = 2; - repeated TPath DeprecatedPath = 3; // deprecated + reserved 3; optional string RowType = 4; optional string Format = 5; map Settings = 6; diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp index 330029a578d0..58228a557d1a 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp @@ -479,7 +479,7 @@ class TS3DqIntegration: public TDqIntegrationBase { range.Save(&out); paths.clear(); - ReadPathsList(srcDesc, {}, serialized, paths); + ReadPathsList({}, serialized, paths); NDq::TS3ReadActorFactoryConfig readActorConfig; ui64 fileSizeLimit = readActorConfig.FileSizeLimit; diff --git a/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp b/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp index 07ca574e18d4..9098936e78dd 100644 --- a/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp +++ b/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp @@ -107,7 +107,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) { b.Save(&range); TPathList paths; - ReadPathsList({}, MakeParams(range), {}, paths); + ReadPathsList(MakeParams(range), {}, paths); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1); UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "name"); @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) { b.Save(&range); TPathList paths; - ReadPathsList({}, MakeParams(range), {}, paths); + ReadPathsList(MakeParams(range), {}, paths); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1); UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "a///b"); @@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) { b.Save(&range); TPathList paths; - ReadPathsList({}, MakeParams(range), {}, paths); + ReadPathsList(MakeParams(range), {}, paths); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2); UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "root/name/"); @@ -165,7 +165,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) { b.Save(&range); TPathList paths; - ReadPathsList({}, MakeParams(range), {}, paths); + ReadPathsList(MakeParams(range), {}, paths); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2); @@ -190,7 +190,7 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) { b.Save(&range); TPathList paths; - ReadPathsList({}, MakeParams(range), {}, paths); + ReadPathsList(MakeParams(range), {}, paths); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 3); diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp index 54cf3b3cd963..69b219311b73 100644 --- a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp +++ b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp @@ -38,7 +38,7 @@ static void BuildPathsFromTree(const google::protobuf::RepeatedPtrField map(sourceDesc.GetDeprecatedPath().size()); - for (auto i = 0; i < sourceDesc.GetDeprecatedPath().size(); ++i) { - map.emplace(sourceDesc.GetDeprecatedPath().Get(i).GetPath(), sourceDesc.GetDeprecatedPath().Get(i).GetSize()); - } - - for (auto i = 0; i < range.GetDeprecatedPath().size(); ++i) { - const auto& path = range.GetDeprecatedPath().Get(i); - auto it = map.find(path); - YQL_ENSURE(it != map.end()); - paths.emplace_back(TPath{path, it->second, false, i + startPathIndex}); - } } -void ReadPathsList(const NS3::TSource& sourceDesc, const THashMap& taskParams, const TVector& readRanges, TPathList& paths) { +void ReadPathsList(const THashMap& taskParams, const TVector& readRanges, TPathList& paths) { if (!readRanges.empty()) { for (auto readRange : readRanges) { - DecodeS3Range(sourceDesc, readRange, paths); + DecodeS3Range(readRange, paths); } } else if (const auto taskParamsIt = taskParams.find(S3ProviderName); taskParamsIt != taskParams.cend()) { - DecodeS3Range(sourceDesc, taskParamsIt->second, paths); - } else { - for (auto i = 0; i < sourceDesc.GetDeprecatedPath().size(); ++i) { - paths.emplace_back(TPath{ - sourceDesc.GetDeprecatedPath().Get(i).GetPath(), - sourceDesc.GetDeprecatedPath().Get(i).GetSize(), - false, - static_cast(i)}); - } + DecodeS3Range(taskParamsIt->second, paths); } } diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.h b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.h index ab490d0cae52..5bcecceafb1d 100644 --- a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.h +++ b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.h @@ -23,7 +23,7 @@ struct TPath { }; using TPathList = std::vector; -void ReadPathsList(const NS3::TSource& sourceDesc, const THashMap& taskParams, const TVector& readRanges, TPathList& paths); +void ReadPathsList(const THashMap& taskParams, const TVector& readRanges, TPathList& paths); void PackPathsList(const TPathList& paths, TString& packed, bool& isTextEncoded); void UnpackPathsList(TStringBuf packed, bool isTextEncoded, TPathList& paths); diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp b/ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp index 8e28d06e0aaa..61ea7bf1dcf1 100644 --- a/ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp +++ b/ydb/library/yql/providers/s3/range_helpers/path_list_reader_ut.cpp @@ -18,63 +18,6 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) { return map; } - Y_UNIT_TEST(ReadsFilesListFromSourceSettings) { - NS3::TSource src; - { - auto* p = src.AddDeprecatedPath(); - p->SetPath("my/path"); - p->SetSize(100500); - } - { - auto* p = src.AddDeprecatedPath(); - p->SetPath("other/path"); - p->SetSize(1); - } - - TPathList paths; - ReadPathsList(src, {}, {}, paths); - - UNIT_ASSERT_VALUES_EQUAL(paths.size(), 2); - - UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "my/path"); - UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 100500); - UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, false); - UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 0); - - UNIT_ASSERT_VALUES_EQUAL(paths[1].Path, "other/path"); - UNIT_ASSERT_VALUES_EQUAL(paths[1].Size, 1); - UNIT_ASSERT_VALUES_EQUAL(paths[1].IsDirectory, false); - UNIT_ASSERT_VALUES_EQUAL(paths[1].PathIndex, 1); - } - - Y_UNIT_TEST(ReadsFilesListFromParamsAndSourceSettings) { - NS3::TSource src; - { - auto* p = src.AddDeprecatedPath(); - p->SetPath("my/path"); - p->SetSize(100500); - } - { - auto* p = src.AddDeprecatedPath(); - p->SetPath("other/path"); - p->SetSize(1); - } - - NS3::TRange range; - range.SetStartPathIndex(42); - range.AddDeprecatedPath("my/path"); - - TPathList paths; - ReadPathsList(src, MakeParams(range), {}, paths); - - UNIT_ASSERT_VALUES_EQUAL(paths.size(), 1); - - UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "my/path"); - UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 100500); - UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, false); - UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 42); - } - NYql::NS3::TRange::TPath* SetPath(NYql::NS3::TRange::TPath* path, const TString& name = {}, ui64 size = 0, bool read = false) { path->SetName(name); path->SetSize(size); @@ -84,11 +27,6 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) { Y_UNIT_TEST(ReadsFilesListFromTreeParams) { NS3::TSource src; - { - auto* p = src.AddDeprecatedPath(); - p->SetPath("my/path"); - p->SetSize(100500); - } NS3::TRange range; range.SetStartPathIndex(42); @@ -110,7 +48,7 @@ Y_UNIT_TEST_SUITE(PathListReaderTest) { } TPathList paths; - ReadPathsList(src, MakeParams(range), {}, paths); + ReadPathsList(MakeParams(range), {}, paths); UNIT_ASSERT_VALUES_EQUAL(paths.size(), 5);