diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 4a58f7f35b68..15715891e853 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -244,25 +244,6 @@ #include -#include - -namespace { - -struct TAwsApiGuard { - TAwsApiGuard() { - Aws::InitAPI(Options); - } - - ~TAwsApiGuard() { - Aws::ShutdownAPI(Options); - } - -private: - Aws::SDKOptions Options; -}; - -} - namespace NKikimr { namespace NKikimrServicesInitializers { @@ -2782,16 +2763,5 @@ void TGraphServiceInitializer::InitializeServices(NActors::TActorSystemSetup* se TActorSetupCmd(NGraph::CreateGraphService(appData->TenantName), TMailboxType::HTSwap, appData->UserPoolId)); } -TAwsApiInitializer::TAwsApiInitializer(IGlobalObjectStorage& globalObjects) - : GlobalObjects(globalObjects) -{ -} - -void TAwsApiInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - Y_UNUSED(setup); - Y_UNUSED(appData); - GlobalObjects.AddGlobalObject(std::make_shared()); -} - } // namespace NKikimrServicesInitializers } // namespace NKikimr diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 95a5cfc18ae1..891d9a848be3 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -625,14 +625,5 @@ class TGraphServiceInitializer : public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TAwsApiInitializer : public IServiceInitializer { - IGlobalObjectStorage& GlobalObjects; - -public: - TAwsApiInitializer(IGlobalObjectStorage& globalObjects); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - } // namespace NKikimrServicesInitializers } // namespace NKikimr diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index a9b8e7713a46..c5a4461c7ee8 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1676,8 +1676,6 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TGraphServiceInitializer(runConfig)); } - sil->AddServiceInitializer(new TAwsApiInitializer(*this)); - return sil; } diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 30a23d8f6528..a285c17fa5cb 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -21,7 +21,6 @@ SRCS( ) PEERDIR( - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core contrib/libs/protobuf ydb/library/actors/core ydb/library/actors/dnsresolver diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index deb7be3d89e9..39673d15b071 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -17,9 +17,6 @@ #include #include -#include - -#include namespace NKikimr { @@ -35,16 +32,6 @@ enum class EInitialEviction { namespace { -Aws::SDKOptions Options; - -Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); -} - -Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); -} - static const std::vector testYdbSchema = TTestSchema::YdbSchema(); static const std::vector testYdbPk = TTestSchema::YdbPkSchema(); diff --git a/ydb/core/tx/columnshard/ut_schema/ya.make b/ydb/core/tx/columnshard/ut_schema/ya.make index d67c0d2ad5b8..35d906ee2055 100644 --- a/ydb/core/tx/columnshard/ut_schema/ya.make +++ b/ydb/core/tx/columnshard/ut_schema/ya.make @@ -18,7 +18,6 @@ PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/hooks/testing diff --git a/ydb/core/tx/datashard/import_s3.cpp b/ydb/core/tx/datashard/import_s3.cpp index 655ee80172ab..ba7227a7a74c 100644 --- a/ydb/core/tx/datashard/import_s3.cpp +++ b/ydb/core/tx/datashard/import_s3.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include diff --git a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp index b23d0dfffd7e..fd73669032ad 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp +++ b/ydb/core/tx/schemeshard/ut_backup/ut_backup.cpp @@ -7,27 +7,9 @@ #include #include -#include - -#include - using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; -namespace { - -Aws::SDKOptions Options; - -Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); -} - -Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); -} - -} - Y_UNIT_TEST_SUITE(TBackupTests) { using TFillFn = std::function; diff --git a/ydb/core/tx/schemeshard/ut_backup/ya.make b/ydb/core/tx/schemeshard/ut_backup/ya.make index aac9bc5f9334..d9ee6dd81405 100644 --- a/ydb/core/tx/schemeshard/ut_backup/ya.make +++ b/ydb/core/tx/schemeshard/ut_backup/ya.make @@ -20,7 +20,6 @@ IF (NOT OS_WINDOWS) library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp index cb8e586fb01a..dc446151da00 100644 --- a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp @@ -13,10 +13,6 @@ #include #include -#include - -#include - using namespace NSchemeShardUT_Private; using namespace NKikimr::NWrappers::NTestHelpers; @@ -24,16 +20,6 @@ using TTablesWithAttrs = TVector>>; namespace { - Aws::SDKOptions Options; - - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); - } - - Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); - } - void Run(TTestBasicRuntime& runtime, TTestEnv& env, const std::variant, TTablesWithAttrs>& tablesVar, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "", const TString& peerName = "") { @@ -1860,7 +1846,7 @@ partitioning_settings { return ev->Get()->Record .GetTransaction(0).GetOperationType() == NKikimrSchemeOp::ESchemeOpBackup; }; - + THolder delayed; auto prevObserver = runtime.SetObserverFunc([&](TAutoPtr& ev) { if (delayFunc(ev)) { @@ -2256,7 +2242,7 @@ partitioning_settings { min_partitions_count: 10 )")); } - + Y_UNIT_TEST(UserSID) { TTestBasicRuntime runtime; TTestEnv env(runtime); diff --git a/ydb/core/tx/schemeshard/ut_export/ya.make b/ydb/core/tx/schemeshard/ut_export/ya.make index c62dc9ea8ebc..4d5bf91e2698 100644 --- a/ydb/core/tx/schemeshard/ut_export/ya.make +++ b/ydb/core/tx/schemeshard/ut_export/ya.make @@ -20,7 +20,6 @@ IF (NOT OS_WINDOWS) library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp index 97e34a6d3fa4..5e1e42d17ce9 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp @@ -4,28 +4,10 @@ #include -#include - -#include - using namespace NSchemeShardUT_Private; using namespace NSchemeShardUT_Private::NExportReboots; using namespace NKikimr::NWrappers::NTestHelpers; -namespace { - -Aws::SDKOptions Options; - -Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); -} - -Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); -} - -} - Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) { using TUnderlying = std::function&, const TString&, TTestWithReboots&)>; diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make index caf4fb7de362..bc7ca966e0dc 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make @@ -19,7 +19,6 @@ PEERDIR( library/cpp/getopt library/cpp/regex/pcre library/cpp/svnversion - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/testlib/default ydb/core/tx ydb/core/tx/schemeshard/ut_helpers diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 2b558341b0d5..5e84f26ec583 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -21,10 +21,8 @@ #include -#include #include #include -#include #include #include @@ -40,16 +38,6 @@ using namespace NKikimr::NWrappers::NTestHelpers; namespace { - Aws::SDKOptions Options; - - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); - } - - Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); - } - const TString EmptyYsonStr = R"([[[[];%false]]])"; TString GenerateScheme(const NKikimrSchemeOp::TPathDescription& pathDesc) { @@ -329,6 +317,7 @@ namespace { runtime.SetObserverFunc(prevObserver); } + } // anonymous Y_UNIT_TEST_SUITE(TRestoreTests) { diff --git a/ydb/core/tx/schemeshard/ut_restore/ya.make b/ydb/core/tx/schemeshard/ut_restore/ya.make index d514b36b49ee..7044d4283b5e 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ya.make +++ b/ydb/core/tx/schemeshard/ut_restore/ya.make @@ -14,7 +14,6 @@ ELSE() ENDIF() PEERDIR( - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core contrib/libs/double-conversion library/cpp/string_utils/quote ydb/core/kqp/ut/common diff --git a/ydb/core/wrappers/s3_storage.h b/ydb/core/wrappers/s3_storage.h index 5af392c4f63c..d678f237616e 100644 --- a/ydb/core/wrappers/s3_storage.h +++ b/ydb/core/wrappers/s3_storage.h @@ -3,6 +3,7 @@ #ifndef KIKIMR_DISABLE_S3_OPS #include "abstract.h" +#include "s3_storage_config.h" #include #include @@ -31,7 +32,7 @@ namespace NKikimr::NWrappers::NExternalStorage { -class TS3ExternalStorage: public IExternalStorageOperator { +class TS3ExternalStorage: public IExternalStorageOperator, TS3User { private: THolder Client; const Aws::Client::ClientConfiguration Config; diff --git a/ydb/core/wrappers/s3_storage_config.cpp b/ydb/core/wrappers/s3_storage_config.cpp index 9785eb7ed5ef..f0bc70e15fd0 100644 --- a/ydb/core/wrappers/s3_storage_config.cpp +++ b/ydb/core/wrappers/s3_storage_config.cpp @@ -1,15 +1,81 @@ #include "s3_storage.h" #include "s3_storage_config.h" +#include +#include +#include +#include #include +#include +#include +#include #include #ifndef KIKIMR_DISABLE_S3_OPS namespace NKikimr::NWrappers::NExternalStorage { +using namespace Aws; +using namespace Aws::Auth; +using namespace Aws::Client; +using namespace Aws::S3; +using namespace Aws::S3::Model; +using namespace Aws::Utils::Stream; + namespace { +struct TCurlInitializer { + TCurlInitializer() { + curl_global_init(CURL_GLOBAL_ALL); + } + + ~TCurlInitializer() { + curl_global_cleanup(); + } +}; + +struct TApiInitializer { + TApiInitializer() { + Options.httpOptions.initAndCleanupCurl = false; + InitAPI(Options); + + Internal::CleanupEC2MetadataClient(); // speeds up config construction + } + + ~TApiInitializer() { + ShutdownAPI(Options); + } + +private: + SDKOptions Options; +}; + +class TApiOwner { +public: + void Ref() { + auto guard = Guard(Mutex); + if (!RefCount++) { + if (!CurlInitializer) { + CurlInitializer.Reset(new TCurlInitializer); + } + ApiInitializer.Reset(new TApiInitializer); + } + } + + void UnRef() { + auto guard = Guard(Mutex); + if (!--RefCount) { + ApiInitializer.Destroy(); + } + } + +private: + ui64 RefCount = 0; + TMutex Mutex; + THolder CurlInitializer; + THolder ApiInitializer; +}; + namespace NPrivate { template @@ -27,10 +93,10 @@ Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { switch (settings.scheme()) { case TSettings::HTTP: - config.scheme = Aws::Http::Scheme::HTTP; + config.scheme = Http::Scheme::HTTP; break; case TSettings::HTTPS: - config.scheme = Aws::Http::Scheme::HTTPS; + config.scheme = Http::Scheme::HTTPS; break; default: Y_ABORT("Unknown scheme"); @@ -48,6 +114,22 @@ Aws::Auth::AWSCredentials CredentialsFromSettings(const TSettings& settings) { } // anonymous +TS3User::TS3User(const TS3User& /*baseObject*/) { + Singleton()->Ref(); +} + +TS3User::TS3User(TS3User& /*baseObject*/) { + Singleton()->Ref(); +} + +TS3User::TS3User() { + Singleton()->Ref(); +} + +TS3User::~TS3User() { + Singleton()->UnRef(); +} + class TS3ThreadsPoolByEndpoint { private: diff --git a/ydb/core/wrappers/s3_storage_config.h b/ydb/core/wrappers/s3_storage_config.h index b276bd79b363..544ccc74af82 100644 --- a/ydb/core/wrappers/s3_storage_config.h +++ b/ydb/core/wrappers/s3_storage_config.h @@ -17,7 +17,14 @@ namespace NKikimr::NWrappers::NExternalStorage { -class TS3ExternalStorageConfig: public IExternalStorageConfig { +struct TS3User { + TS3User(); + TS3User(const TS3User& baseObject); + TS3User(TS3User& baseObject); + ~TS3User(); +}; + +class TS3ExternalStorageConfig: public IExternalStorageConfig, TS3User { private: YDB_READONLY_DEF(TString, Bucket); Aws::Client::ClientConfiguration Config; diff --git a/ydb/core/wrappers/s3_wrapper_ut.cpp b/ydb/core/wrappers/s3_wrapper_ut.cpp index b8b0e185cde4..4e310add8777 100644 --- a/ydb/core/wrappers/s3_wrapper_ut.cpp +++ b/ydb/core/wrappers/s3_wrapper_ut.cpp @@ -8,33 +8,16 @@ #include #include -#include #include #include -#include - using namespace NActors; using namespace NKikimr; using namespace NKikimr::NWrappers; using namespace Aws::S3::Model; -namespace { - -Aws::SDKOptions Options; - -Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); -} - -Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); -} - -} - -class TS3MockTest: public NUnitTest::TTestBase { +class TS3MockTest: public NUnitTest::TTestBase, private NExternalStorage::TS3User { using TS3Mock = NWrappers::NTestHelpers::TS3Mock; static auto MakeClientConfig(ui16 port) { @@ -65,8 +48,8 @@ class TS3MockTest: public NUnitTest::TTestBase { } void TearDown() override { - Runtime.Reset(); S3Mock.Reset(); + Runtime.Reset(); } ui16 GetPort() const { diff --git a/ydb/core/wrappers/ut/ya.make b/ydb/core/wrappers/ut/ya.make index 2ba33af92409..b10493198ed7 100644 --- a/ydb/core/wrappers/ut/ya.make +++ b/ydb/core/wrappers/ut/ya.make @@ -9,7 +9,6 @@ IF (NOT OS_WINDOWS) ydb/library/actors/core library/cpp/digest/md5 library/cpp/testing/unittest - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core ydb/core/protos ydb/core/testlib/basics/default ydb/library/yql/minikql/comp_nodes/llvm14 diff --git a/ydb/services/ydb/backup_ut/ya.make b/ydb/services/ydb/backup_ut/ya.make deleted file mode 100644 index 39732a69bd9c..000000000000 --- a/ydb/services/ydb/backup_ut/ya.make +++ /dev/null @@ -1,32 +0,0 @@ -UNITTEST_FOR(ydb/services/ydb) - -FORK_SUBTESTS() - -IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) - SIZE(LARGE) - TAG(ya:fat) -ELSE() - SIZE(MEDIUM) -ENDIF() - -SRCS( - ydb_backup_ut.cpp -) - -PEERDIR( - ydb/core/testlib/default - ydb/core/wrappers/ut_helpers - ydb/public/lib/ydb_cli/dump - ydb/public/sdk/cpp/client/ydb_export - ydb/public/sdk/cpp/client/ydb_import - ydb/public/sdk/cpp/client/ydb_operation - ydb/public/sdk/cpp/client/ydb_result - ydb/public/sdk/cpp/client/ydb_table - ydb/public/sdk/cpp/client/ydb_value - ydb/library/backup - contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp deleted file mode 100644 index f7e9371cf9c3..000000000000 --- a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp +++ /dev/null @@ -1,626 +0,0 @@ -#include "ydb_common_ut.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -using namespace NYdb; -using namespace NYdb::NTable; - -namespace NYdb::NTable { - -bool operator==(const TValue& lhs, const TValue& rhs) { - return google::protobuf::util::MessageDifferencer::Equals(lhs.GetProto(), rhs.GetProto()); -} - -bool operator==(const TKeyBound& lhs, const TKeyBound& rhs) { - return lhs.GetValue() == rhs.GetValue() && lhs.IsInclusive() == rhs.IsInclusive(); -} - -bool operator==(const TKeyRange& lhs, const TKeyRange& rhs) { - return lhs.From() == lhs.From() && lhs.To() == rhs.To(); -} - -} - -namespace { - -#define DEBUG_HINT (TStringBuilder() << "at line " << __LINE__) - -void ExecuteDataDefinitionQuery(TSession& session, const TString& script) { - const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); -} - -TDataQueryResult ExecuteDataModificationQuery(TSession& session, - const TString& script, - const TExecDataQuerySettings& settings = {} -) { - const auto result = session.ExecuteDataQuery( - script, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), - settings - ).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); - - return result; -} - -TDataQueryResult GetTableContent(TSession& session, const char* table) { - return ExecuteDataModificationQuery(session, Sprintf(R"( - SELECT * FROM `%s` ORDER BY Key; - )", table - )); -} - -void CompareResults(const TDataQueryResult& first, const TDataQueryResult& second) { - const auto& firstResults = first.GetResultSets(); - const auto& secondResults = second.GetResultSets(); - - UNIT_ASSERT_VALUES_EQUAL(firstResults.size(), secondResults.size()); - for (size_t i = 0; i < firstResults.size(); ++i) { - UNIT_ASSERT_STRINGS_EQUAL( - FormatResultSetYson(firstResults[i]), - FormatResultSetYson(secondResults[i]) - ); - } -} - -TTableDescription GetTableDescription(TSession& session, const TString& path, - const TDescribeTableSettings& settings = {} -) { - auto describeResult = session.DescribeTable(path, settings).ExtractValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); - return describeResult.GetTableDescription(); -} - -auto CreateMinPartitionsChecker(ui32 expectedMinPartitions, const TString& debugHint = "") { - return [=](const TTableDescription& tableDescription) { - UNIT_ASSERT_VALUES_EQUAL_C( - tableDescription.GetPartitioningSettings().GetMinPartitionsCount(), - expectedMinPartitions, - debugHint - ); - }; -} - -void CheckTableDescription(TSession& session, const TString& path, auto&& checker, - const TDescribeTableSettings& settings = {} -) { - checker(GetTableDescription(session, path, settings)); -} - -using TBackupFunction = std::function; -using TRestoreFunction = std::function; - -void TestTableContentIsPreserved( - const char* table, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore -) { - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ); - )", - table - )); - ExecuteDataModificationQuery(session, Sprintf(R"( - UPSERT INTO `%s` ( - Key, - Value - ) - VALUES - (1, "one"), - (2, "two"), - (3, "three"), - (4, "four"), - (5, "five"); - )", - table - )); - const auto originalContent = GetTableContent(session, table); - - backup(table); - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - - restore(table); - CompareResults(GetTableContent(session, table), originalContent); -} - -void TestTablePartitioningSettingsArePreserved( - const char* table, ui32 minPartitions, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore -) { - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ) - WITH ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %u - ); - )", - table, minPartitions - )); - CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT)); - - backup(table); - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - - restore(table); - CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions, DEBUG_HINT)); -} - -void TestIndexTablePartitioningSettingsArePreserved( - const char* table, const char* index, ui32 minIndexPartitions, TSession& session, - TBackupFunction&& backup, TRestoreFunction&& restore -) { - const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Uint32, - PRIMARY KEY (Key), - INDEX %s GLOBAL ON (Value) - ); - )", - table, index - )); - ExecuteDataDefinitionQuery(session, Sprintf(R"( - ALTER TABLE `%s` ALTER INDEX %s SET ( - AUTO_PARTITIONING_BY_LOAD = ENABLED, - AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %u - ); - )", table, index, minIndexPartitions - )); - CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT)); - - backup(table); - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - - restore(table); - CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT)); -} - -void TestTableSplitBoundariesArePreserved( - const char* table, ui64 partitions, TSession& session, TBackupFunction&& backup, TRestoreFunction&& restore -) { - ExecuteDataDefinitionQuery(session, Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint32, - Value Utf8, - PRIMARY KEY (Key) - ) - WITH ( - PARTITION_AT_KEYS = (1, 2, 4, 8, 16, 32, 64, 128, 256) - ); - )", - table - )); - const auto describeSettings = TDescribeTableSettings() - .WithTableStatistics(true) - .WithKeyShardBoundary(true); - const auto originalTableDescription = GetTableDescription(session, table, describeSettings); - UNIT_ASSERT_VALUES_EQUAL(originalTableDescription.GetPartitionsCount(), partitions); - const auto& originalKeyRanges = originalTableDescription.GetKeyRanges(); - UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), partitions); - - backup(table); - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - - restore(table); - const auto restoredTableDescription = GetTableDescription(session, table, describeSettings); - UNIT_ASSERT_VALUES_EQUAL(restoredTableDescription.GetPartitionsCount(), partitions); - const auto& restoredKeyRanges = restoredTableDescription.GetKeyRanges(); - UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), partitions); - UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges); -} - -void TestIndexTableSplitBoundariesArePreserved( - const char* table, const char* index, ui64 indexPartitions, TSession& session, - TBackupFunction&& backup, TRestoreFunction&& restore -) { - const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); - - { - TExplicitPartitions indexPartitionBoundaries; - for (ui32 boundary : {1, 2, 4, 8, 16, 32, 64, 128, 256}) { - indexPartitionBoundaries.AppendSplitPoints( - // split boundary is technically always a tuple - TValueBuilder().BeginTuple().AddElement().OptionalUint32(boundary).EndTuple().Build() - ); - } - // By default indexImplTable has auto partitioning by size enabled, - // so you must set min partition count for partitions to not merge immediately after indexImplTable is built. - TPartitioningSettingsBuilder partitioningSettingsBuilder; - partitioningSettingsBuilder - .SetMinPartitionsCount(indexPartitions) - .SetMaxPartitionsCount(indexPartitions); - - const auto indexSettings = TGlobalIndexSettings{ - .PartitioningSettings = partitioningSettingsBuilder.Build(), - .Partitions = std::move(indexPartitionBoundaries) - }; - - auto tableBuilder = TTableBuilder() - .AddNullableColumn("Key", EPrimitiveType::Uint32) - .AddNullableColumn("Value", EPrimitiveType::Uint32) - .SetPrimaryKeyColumn("Key") - .AddSecondaryIndex(TIndexDescription("byValue", EIndexType::GlobalSync, { "Value" }, {}, { indexSettings })); - - const auto result = session.CreateTable(table, tableBuilder.Build()).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - const auto describeSettings = TDescribeTableSettings() - .WithTableStatistics(true) - .WithKeyShardBoundary(true); - const auto originalIndexTableDescription = GetTableDescription( - session, indexTablePath, describeSettings - ); - UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions); - const auto& originalKeyRanges = originalIndexTableDescription.GetKeyRanges(); - UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), indexPartitions); - - backup(table); - - ExecuteDataDefinitionQuery(session, Sprintf(R"( - DROP TABLE `%s`; - )", table - )); - - restore(table); - const auto restoredIndexTableDescription = GetTableDescription( - session, indexTablePath, describeSettings - ); - UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions); - const auto& restoredKeyRanges = restoredIndexTableDescription.GetKeyRanges(); - UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), indexPartitions); - UNIT_ASSERT_EQUAL(restoredKeyRanges, originalKeyRanges); -} - -} - -Y_UNIT_TEST_SUITE(BackupRestore) { - - void Restore(NDump::TClient& client, const TFsPath& sourceFile, const TString& dbPath) { - auto result = client.Restore(sourceFile, dbPath); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - auto CreateBackupLambda(const TDriver& driver, const TFsPath& pathToBackup, bool schemaOnly = false) { - return [&driver, &pathToBackup, schemaOnly](const char* table) { - Y_UNUSED(table); - // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder - NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, schemaOnly, false); - }; - } - - auto CreateRestoreLambda(const TDriver& driver, const TFsPath& pathToBackup) { - return [&driver, &pathToBackup](const char* table) { - Y_UNUSED(table); - NDump::TClient backupClient(driver); - Restore(backupClient, pathToBackup, "/Root"); - }; - } - - Y_UNIT_TEST(RestoreTableContent) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - - constexpr const char* table = "/Root/table"; - - TestTableContentIsPreserved( - table, - session, - CreateBackupLambda(driver, pathToBackup), - CreateRestoreLambda(driver, pathToBackup) - ); - } - - Y_UNIT_TEST(RestoreTablePartitioningSettings) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - - constexpr const char* table = "/Root/table"; - constexpr ui32 minPartitions = 10; - - TestTablePartitioningSettingsArePreserved( - table, - minPartitions, - session, - CreateBackupLambda(driver, pathToBackup, true), - CreateRestoreLambda(driver, pathToBackup) - ); - } - - Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - - constexpr const char* table = "/Root/table"; - constexpr const char* index = "byValue"; - constexpr ui32 minIndexPartitions = 10; - - TestIndexTablePartitioningSettingsArePreserved( - table, - index, - minIndexPartitions, - session, - CreateBackupLambda(driver, pathToBackup, true), - CreateRestoreLambda(driver, pathToBackup) - ); - } - - Y_UNIT_TEST(RestoreTableSplitBoundaries) { - TKikimrWithGrpcAndRootSchema server; - auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))); - TTableClient tableClient(driver); - auto session = tableClient.GetSession().ExtractValueSync().GetSession(); - TTempDir tempDir; - const auto& pathToBackup = tempDir.Path(); - - constexpr const char* table = "/Root/table"; - constexpr ui64 partitions = 10; - - TestTableSplitBoundariesArePreserved( - table, - partitions, - session, - CreateBackupLambda(driver, pathToBackup, true), - CreateRestoreLambda(driver, pathToBackup) - ); - } - - // TO DO: test index impl table split boundaries restoration from a backup -} - -Y_UNIT_TEST_SUITE(BackupRestoreS3) { - - Aws::SDKOptions Options; - - Y_TEST_HOOK_BEFORE_RUN(InitAwsAPI) { - Aws::InitAPI(Options); - } - - Y_TEST_HOOK_AFTER_RUN(ShutdownAwsAPI) { - Aws::ShutdownAPI(Options); - } - - using NKikimr::NWrappers::NTestHelpers::TS3Mock; - - class TS3TestEnv { - TKikimrWithGrpcAndRootSchema server; - TDriver driver; - TTableClient tableClient; - TSession session; - ui16 s3Port; - TS3Mock s3Mock; - // required for exports to function - TDataShardExportFactory dataShardExportFactory; - - public: - TS3TestEnv() - : driver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort()))) - , tableClient(driver) - , session(tableClient.CreateSession().ExtractValueSync().GetSession()) - , s3Port(server.GetPortManager().GetPort()) - , s3Mock({}, TS3Mock::TSettings(s3Port)) - { - UNIT_ASSERT_C(s3Mock.Start(), s3Mock.GetError()); - - auto& runtime = *server.GetRuntime(); - runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::EPriority::PRI_DEBUG); - runtime.GetAppData().DataShardExportFactory = &dataShardExportFactory; - } - - TKikimrWithGrpcAndRootSchema& GetServer() { - return server; - } - - const TDriver& GetDriver() const { - return driver; - } - - TSession& GetSession() { - return session; - } - - ui16 GetS3Port() const { - return s3Port; - } - }; - - template - bool WaitForOperation(NOperation::TOperationClient& client, NOperationId::TOperationId id, - int retries = 10, TDuration sleepDuration = TDuration::MilliSeconds(100) - ) { - for (int retry = 0; retry <= retries; ++retry) { - auto result = client.Get(id).ExtractValueSync(); - if (result.Ready()) { - UNIT_ASSERT_VALUES_EQUAL_C( - result.Status().GetStatus(), EStatus::SUCCESS, - result.Status().GetIssues().ToString() - ); - return true; - } - Sleep(sleepDuration *= 2); - } - return false; - } - - void ExportToS3(NExport::TExportClient& exportClient, ui16 s3Port, NOperation::TOperationClient& operationClient, - const TString& source, const TString& destination - ) { - // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. - // Any non-empty strings should do. - const auto exportSettings = NExport::TExportToS3Settings() - .Endpoint(Sprintf("localhost:%u", s3Port)) - .Scheme(ES3Scheme::HTTP) - .Bucket("test_bucket") - .AccessKey("test_key") - .SecretKey("test_secret") - .AppendItem(NExport::TExportToS3Settings::TItem{.Src = source, .Dst = destination}); - - auto response = exportClient.ExportToS3(exportSettings).ExtractValueSync(); - UNIT_ASSERT_C(WaitForOperation(operationClient, response.Id()), - Sprintf("The export from %s to %s did not complete within the allocated time.", - source.c_str(), destination.c_str() - ) - ); - } - - void ImportFromS3(NImport::TImportClient& importClient, ui16 s3Port, NOperation::TOperationClient& operationClient, - const TString& source, const TString& destination - ) { - // The exact values for Bucket, AccessKey and SecretKey do not matter if the S3 backend is TS3Mock. - // Any non-empty strings should do. - const auto importSettings = NImport::TImportFromS3Settings() - .Endpoint(Sprintf("localhost:%u", s3Port)) - .Scheme(ES3Scheme::HTTP) - .Bucket("test_bucket") - .AccessKey("test_key") - .SecretKey("test_secret") - .AppendItem(NImport::TImportFromS3Settings::TItem{.Src = source, .Dst = destination}); - - auto response = importClient.ImportFromS3(importSettings).ExtractValueSync(); - UNIT_ASSERT_C(WaitForOperation(operationClient, response.Id()), - Sprintf("The import from %s to %s did not complete within the allocated time.", - source.c_str(), destination.c_str() - ) - ); - } - - auto CreateBackupLambda(const TDriver& driver, ui16 s3Port) { - return [&driver, s3Port](const char* table) { - NExport::TExportClient exportClient(driver); - NOperation::TOperationClient operationClient(driver); - ExportToS3(exportClient, s3Port, operationClient, table, "table"); - }; - } - - auto CreateRestoreLambda(const TDriver& driver, ui16 s3Port) { - return [&driver, s3Port](const char* table) { - NImport::TImportClient importClient(driver); - NOperation::TOperationClient operationClient(driver); - ImportFromS3(importClient, s3Port, operationClient, "table", table); - }; - } - - Y_UNIT_TEST(RestoreTableContent) { - TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; - - TestTableContentIsPreserved( - table, - testEnv.GetSession(), - CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), - CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) - ); - } - - Y_UNIT_TEST(RestoreTablePartitioningSettings) { - TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; - constexpr ui32 minPartitions = 10; - - TestTablePartitioningSettingsArePreserved( - table, - minPartitions, - testEnv.GetSession(), - CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), - CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) - ); - } - - Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { - TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; - constexpr const char* index = "byValue"; - constexpr ui32 minIndexPartitions = 10; - - TestIndexTablePartitioningSettingsArePreserved( - table, - index, - minIndexPartitions, - testEnv.GetSession(), - CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), - CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) - ); - } - - Y_UNIT_TEST(RestoreTableSplitBoundaries) { - TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; - constexpr ui64 partitions = 10; - - TestTableSplitBoundariesArePreserved( - table, - partitions, - testEnv.GetSession(), - CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), - CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) - ); - } - - Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) { - TS3TestEnv testEnv; - constexpr const char* table = "/Root/table"; - constexpr const char* index = "byValue"; - constexpr ui64 indexPartitions = 10; - - TestIndexTableSplitBoundariesArePreserved( - table, - index, - indexPartitions, - testEnv.GetSession(), - CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()), - CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port()) - ); - } - -} diff --git a/ydb/services/ydb/ut/ya.make b/ydb/services/ydb/ut/ya.make index 442f7459444d..07a734b9f57b 100644 --- a/ydb/services/ydb/ut/ya.make +++ b/ydb/services/ydb/ut/ya.make @@ -45,6 +45,8 @@ PEERDIR( ydb/core/grpc_services/base ydb/core/testlib ydb/core/security + ydb/core/wrappers/ut_helpers + ydb/library/backup ydb/library/yql/minikql/dom ydb/library/yql/minikql/jsonpath ydb/library/testlib/service_mocks/ldap_mock @@ -53,6 +55,7 @@ PEERDIR( ydb/public/lib/yson_value ydb/public/lib/ut_helpers ydb/public/lib/ydb_cli/commands + ydb/public/lib/ydb_cli/dump ydb/public/sdk/cpp/client/draft ydb/public/sdk/cpp/client/ydb_coordination ydb/public/sdk/cpp/client/ydb_export diff --git a/ydb/services/ydb/ya.make b/ydb/services/ydb/ya.make index 34676794495e..d42bbf19283c 100644 --- a/ydb/services/ydb/ya.make +++ b/ydb/services/ydb/ya.make @@ -37,7 +37,6 @@ PEERDIR( END() RECURSE_FOR_TESTS( - backup_ut sdk_sessions_ut sdk_sessions_pool_ut table_split_ut diff --git a/ydb/services/ydb/ydb_import_ut.cpp b/ydb/services/ydb/ydb_import_ut.cpp index 961cd9679c85..a423246c30a5 100644 --- a/ydb/services/ydb/ydb_import_ut.cpp +++ b/ydb/services/ydb/ydb_import_ut.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include +#include #include #include @@ -128,3 +130,215 @@ Y_UNIT_TEST_SUITE(YdbImport) { } } + +Y_UNIT_TEST_SUITE(BackupRestore) { + + using namespace NYdb::NTable; + + void ExecuteDataDefinitionQuery(TSession& session, const TString& script) { + const auto result = session.ExecuteSchemeQuery(script).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); + } + + TDataQueryResult ExecuteDataModificationQuery(TSession& session, + const TString& script, + const TExecDataQuerySettings& settings = {} + ) { + const auto result = session.ExecuteDataQuery( + script, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + settings + ).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), "script:\n" << script << "\nissues:\n" << result.GetIssues().ToString()); + + return result; + } + + TValue GetSingleResult(const TDataQueryResult& rawResults) { + auto resultSetParser = rawResults.GetResultSetParser(0); + UNIT_ASSERT(resultSetParser.TryNextRow()); + return resultSetParser.GetValue(0); + } + + ui64 GetUint64(const TValue& value) { + return TValueParser(value).GetUint64(); + } + + void Restore(NDump::TClient& client, const TFsPath& sourceFile, const TString& dbPath) { + auto result = client.Restore(sourceFile, dbPath); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + auto CreateMinPartitionsChecker(ui64 expectedMinPartitions) { + return [=](const TTableDescription& tableDescription) { + return tableDescription.GetPartitioningSettings().GetMinPartitionsCount() == expectedMinPartitions; + }; + } + + void CheckTableDescription(TSession& session, const TString& path, auto&& checker) { + auto describeResult = session.DescribeTable(path).ExtractValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); + auto tableDescription = describeResult.GetTableDescription(); + Ydb::Table::CreateTableRequest descriptionProto; + // The purpose of translating to CreateTableRequest is solely to produce a clearer error message. + tableDescription.SerializeTo(descriptionProto); + UNIT_ASSERT_C( + checker(tableDescription), + descriptionProto.DebugString() + ); + } + + Y_UNIT_TEST(Basic) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + + constexpr const char* table = "/Root/table"; + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ); + )", + table + )); + ExecuteDataModificationQuery(session, Sprintf(R"( + UPSERT INTO `%s` ( + Key, + Value + ) + VALUES + (1, "one"), + (2, "two"), + (3, "three"), + (4, "four"), + (5, "five"); + )", + table + )); + + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder + NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); + + NDump::TClient backupClient(driver); + + // restore deleted rows in an existing table + ExecuteDataModificationQuery(session, Sprintf(R"( + DELETE FROM `%s` WHERE Key > 3; + )", table + )); + Restore(backupClient, pathToBackup, "/Root"); + { + auto result = ExecuteDataModificationQuery(session, Sprintf(R"( + SELECT COUNT(*) FROM `%s`; + )", table + )); + UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); + } + + // restore deleted table + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + Restore(backupClient, pathToBackup, "/Root"); + { + auto result = ExecuteDataModificationQuery(session, Sprintf(R"( + SELECT COUNT(*) FROM `%s`; + )", table + )); + UNIT_ASSERT_VALUES_EQUAL(GetUint64(GetSingleResult(result)), 5ull); + } + } + + Y_UNIT_TEST(RestoreTablePartitioningSettings) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + + constexpr const char* table = "/Root/table"; + constexpr int minPartitions = 10; + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Utf8, + PRIMARY KEY (Key) + ) + WITH ( + AUTO_PARTITIONING_BY_LOAD = ENABLED, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d + ); + )", + table, minPartitions + )); + + CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions)); + + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder + NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); + + NDump::TClient backupClient(driver); + + // restore deleted table + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + Restore(backupClient, pathToBackup, "/Root"); + CheckTableDescription(session, table, CreateMinPartitionsChecker(minPartitions)); + } + + Y_UNIT_TEST(RestoreIndexTablePartitioningSettings) { + TKikimrWithGrpcAndRootSchema server; + auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort()))); + TTableClient tableClient(driver); + auto session = tableClient.GetSession().ExtractValueSync().GetSession(); + + constexpr const char* table = "/Root/table"; + constexpr const char* index = "byValue"; + const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable"); + constexpr int minPartitions = 10; + ExecuteDataDefinitionQuery(session, Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint32, + Value Uint32, + PRIMARY KEY (Key), + INDEX %s GLOBAL ON (Value) + ); + )", + table, index + )); + ExecuteDataDefinitionQuery(session, Sprintf(R"( + ALTER TABLE `%s` ALTER INDEX %s SET ( + AUTO_PARTITIONING_BY_LOAD = ENABLED, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d + ); + )", table, index, minPartitions + )); + + CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); + + TTempDir tempDir; + const auto& pathToBackup = tempDir.Path(); + // TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder + NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, false, false); + + NDump::TClient backupClient(driver); + + // restore deleted table + ExecuteDataDefinitionQuery(session, Sprintf(R"( + DROP TABLE `%s`; + )", table + )); + Restore(backupClient, pathToBackup, "/Root"); + CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minPartitions)); + } + +}