diff --git a/.clang-format b/.clang-format index fd84858b672..8e37396e34a 100644 --- a/.clang-format +++ b/.clang-format @@ -179,7 +179,7 @@ PenaltyBreakString: 1000 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyIndentedWhitespace: 0 -PenaltyReturnTypeOnItsOwnLine: 20000 +PenaltyReturnTypeOnItsOwnLine: 0 PointerAlignment: Left PPIndentWidth: -1 QualifierAlignment: Leave diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index ef65d8bd05d..634832abe26 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -5,6 +5,7 @@ cloud/blockstore/tests/resize-disk * cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_s3 cloud/disk_manager/internal/pkg/dataplane/snapshot/storage/tests tests.TestShallowCopySnapshotWithRandomFailure/store_chunks_in_ydb +cloud/disk_manager/internal/pkg/facade/disk_service_nemesis_test tests.TestDiskServiceMigrate* cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-kikimr-nemesis-test * cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-kikimr-test * cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-local-test * diff --git a/.gitignore b/.gitignore index 21b77be6537..9bce80cec2e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ __pycache__ *.py[cod] *$py.class +# distclang +.distclang* + # MacOS specific .DS_Store @@ -86,6 +89,9 @@ test-results # IDE files *.swp +.vscode/* +*.code-workspace +*.code-workspace.bak # act files .input diff --git a/CLANG-FORMAT.md b/CLANG-FORMAT.md index aba3896edf8..591d0e09d4e 100644 --- a/CLANG-FORMAT.md +++ b/CLANG-FORMAT.md @@ -4,10 +4,14 @@ Use [this](.clang-format) clang-format config to format new and modified code. ## Install +For VSCode install and use clangd plugin https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd + +For other development environments, you should install clang-format-18 manually. + ``` sudo apt-get install clang-format-18 ``` Configure the plugin for your favorite code editor to use the [config](.clang-format) to format the code. For VS Code, for example, you can use https://marketplace.visualstudio.com/items?itemName=xaver.clang-format -Note. clang-format searches for the .clang-format configuration file in the directory of the formatted file and in all parent directories. Therefore, no additional settings are required. \ No newline at end of file +Note. clang-format searches for the .clang-format configuration file in the directory of the formatted file and in all parent directories. Therefore, no additional settings are required. diff --git a/README.md b/README.md index aa46283d920..f86938719fd 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Network Block Device implementation over YDB BlobStorage or over our own storage ### Quickstart +Follow the instructions [here](VSCODE.md) to generate workspace and install the necessary plugins. + Follow the instructions [here](example/README.md) to build and run NBS on your machine and to attach an NBS-based disk via NBD. NBS-based disks can be attached via vhost-user-blk as well. Follow the instructions [here](CLANG-FORMAT.md) to install clang-format for formatting the code. diff --git a/VSCODE.md b/VSCODE.md new file mode 100644 index 00000000000..80f249cab66 --- /dev/null +++ b/VSCODE.md @@ -0,0 +1,28 @@ +## Visual Studio Code + +### Generate workspace + +Run vscode_generate_workspace.sh to generate a workspace in the root of the repository. +``` +.vscode_generate_workspace.sh +``` + +### Open workspace + +Open workspace from menu "Open workspace from file..." And select the newly created file "nbs.code-workspace" in the root of the repository. + +Or execute +``` +code nbs.code-workspace +``` + +### Code competition + +Install all recommended by workspace plugins. +The most important one is https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd used for code competition and formatting. + +### Formatting + +Enable feature "Trim trailing whitespace" for user or workspace. +Enable feature "Trim final newlines" for user or workspace. +Enable feature "Insert final newline" for user or workspace. diff --git a/cloud/blockstore/libs/.gitignore b/cloud/blockstore/libs/.gitignore index bdba5625e28..51c717af23d 100644 --- a/cloud/blockstore/libs/.gitignore +++ b/cloud/blockstore/libs/.gitignore @@ -1,18 +1,19 @@ client/bench/blockstore-client-bench -client/ut/cloud-blockstore-libs-client-ut client/ut_throttling/cloud-blockstore-libs-client-ut_throttling +client/ut/cloud-blockstore-libs-client-ut common/ut/cloud-blockstore-libs-common-ut daemon/ydb/ut/cloud-blockstore-libs-daemon-ydb-ut diagnostics/ut/cloud-blockstore-libs-diagnostics-ut discovery/ut/cloud-blockstore-libs-discovery-ut disk_agent/ut/cloud-blockstore-libs-disk_agent-ut -encryption/ut/cloud-blockstore-libs-encryption-ut encryption/ut_keyring/bin/cloud-blockstore-libs-encryption-ut_keyring-bin encryption/ut_keyring/cloud-blockstore-libs-encryption-ut_keyring +encryption/ut/cloud-blockstore-libs-encryption-ut endpoints_grpc/ut/cloud-blockstore-libs-endpoints_grpc-ut -endpoints/ut/cloud-blockstore-libs-endpoints-ut endpoints_vhost/ut/cloud-blockstore-libs-endpoints_vhost-ut +endpoints/ut/cloud-blockstore-libs-endpoints-ut kms/iface/ut/cloud-blockstore-libs-kms-iface-ut +kms/impl/example/kms-example logbroker/iface/ut/cloud-blockstore-libs-logbroker-iface-ut logbroker/topic_api_impl/ut/cloud-blockstore-libs-logbroker-topic_api_impl-ut nbd/bench/blockstore-nbd-bench @@ -22,26 +23,25 @@ rdma/iface/ut/cloud-blockstore-libs-rdma-iface-ut rdma/impl/ut/cloud-blockstore-libs-rdma-impl-ut server/ut/cloud-blockstore-libs-server-ut service_kikimr/ut/cloud-blockstore-libs-service_kikimr-ut -service_local/ut/cloud-blockstore-libs-service_local-ut service_local/ut_large/cloud-blockstore-libs-service_local-ut_large -service_throttling/ut/cloud-blockstore-libs-service_throttling-ut +service_local/ut/cloud-blockstore-libs-service_local-ut service_throttling/ut_logger/blockstore-libs-service_throttling-ut_logger service_throttling/ut_metrics/blockstore-libs-service_throttling-ut_metrics service_throttling/ut_policy/blockstore-libs-service_throttling-ut_policy +service_throttling/ut/cloud-blockstore-libs-service_throttling-ut service/ut/cloud-blockstore-libs-service-ut storage/bootstrapper/ut/cloud-blockstore-libs-storage-bootstrapper-ut storage/core/ut/cloud-blockstore-libs-storage-core-ut storage/disk_agent/benchmark/benchmark storage/disk_agent/model/ut/cloud-blockstore-libs-storage-disk_agent-model-ut storage/disk_agent/ut_actor/cloud-blockstore-libs-storage-disk_agent-ut_actor -storage/disk_agent/ut/cloud-blockstore-libs-storage-disk_agent-ut storage/disk_agent/ut_large/cloud-blockstore-libs-storage-disk_agent-ut_large +storage/disk_agent/ut/cloud-blockstore-libs-storage-disk_agent-ut +storage/disk_registry_proxy/ut/blockstore-libs-storage-disk_registry_proxy-ut storage/disk_registry/actors/ut/libs-storage-disk_registry-actors-ut storage/disk_registry/model/ut/blockstore-libs-storage-disk_registry-model-ut -storage/disk_registry_proxy/ut/blockstore-libs-storage-disk_registry_proxy-ut storage/disk_registry/ut_allocation/libs-storage-disk_registry-ut_allocation storage/disk_registry/ut_checkpoint/libs-storage-disk_registry-ut_checkpoint -storage/disk_registry/ut/cloud-blockstore-libs-storage-disk_registry-ut storage/disk_registry/ut_cms/blockstore-libs-storage-disk_registry-ut_cms storage/disk_registry/ut_config/libs-storage-disk_registry-ut_config storage/disk_registry/ut_create/libs-storage-disk_registry-ut_create @@ -54,27 +54,28 @@ storage/disk_registry/ut_restore/libs-storage-disk_registry-ut_restore storage/disk_registry/ut_session/libs-storage-disk_registry-ut_session storage/disk_registry/ut_suspend/libs-storage-disk_registry-ut_suspend storage/disk_registry/ut_wait_device/storage-disk_registry-ut_wait_device +storage/disk_registry/ut/cloud-blockstore-libs-storage-disk_registry-ut storage/model/ut/cloud-blockstore-libs-storage-model-ut -storage/partition2/model/ut/cloud-blockstore-libs-storage-partition2-model-ut -storage/partition2/ut/cloud-blockstore-libs-storage-partition2-ut storage/partition_common/ut/cloud-blockstore-libs-storage-partition_common-ut -storage/partition/model/ut/cloud-blockstore-libs-storage-partition-model-ut storage/partition_nonrepl/model/ut/libs-storage-partition_nonrepl-model-ut storage/partition_nonrepl/ut/blockstore-libs-storage-partition_nonrepl-ut +storage/partition/model/ut/cloud-blockstore-libs-storage-partition-model-ut storage/partition/ut/cloud-blockstore-libs-storage-partition-ut +storage/partition2/model/ut/cloud-blockstore-libs-storage-partition2-model-ut +storage/partition2/ut/cloud-blockstore-libs-storage-partition2-ut storage/perf/cloud-blockstore-libs-storage-perf storage/service/model/ut/cloud-blockstore-libs-storage-service-model-ut storage/service/ut/cloud-blockstore-libs-storage-service-ut storage/ss_proxy/ut/cloud-blockstore-libs-storage-ss_proxy-ut storage/stats_service/ut/cloud-blockstore-libs-storage-stats_service-ut storage/undelivered/ut/cloud-blockstore-libs-storage-undelivered-ut -storage/volume/actors/ut/cloud-blockstore-libs-storage-volume-actors-ut storage/volume_balancer/ut/cloud-blockstore-libs-storage-volume_balancer-ut -storage/volume/model/ut/cloud-blockstore-libs-storage-volume-model-ut storage/volume_proxy/ut/cloud-blockstore-libs-storage-volume_proxy-ut +storage/volume/actors/ut/cloud-blockstore-libs-storage-volume-actors-ut +storage/volume/model/ut/cloud-blockstore-libs-storage-volume-model-ut storage/volume/ut/cloud-blockstore-libs-storage-volume-ut throttling/ut/cloud-blockstore-libs-throttling-ut validation/ut/cloud-blockstore-libs-validation-ut -vhost/ut/cloud-blockstore-libs-vhost-ut vhost/ut_stress/cloud-blockstore-libs-vhost-ut_stress +vhost/ut/cloud-blockstore-libs-vhost-ut ydbstats/ut/cloud-blockstore-libs-ydbstats-ut diff --git a/cloud/blockstore/libs/rdma/impl/client.cpp b/cloud/blockstore/libs/rdma/impl/client.cpp index a447a320885..a80283e39fb 100644 --- a/cloud/blockstore/libs/rdma/impl/client.cpp +++ b/cloud/blockstore/libs/rdma/impl/client.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -53,6 +54,8 @@ constexpr TDuration FLUSH_TIMEOUT = TDuration::Seconds(10); constexpr TDuration LOG_THROTTLER_PERIOD = TDuration::Seconds(60); constexpr TDuration MIN_RECONNECT_DELAY = TDuration::MilliSeconds(10); +constexpr size_t REQUEST_HISTORY_SIZE = 1024; + //////////////////////////////////////////////////////////////////////////////// struct TRequest; @@ -104,7 +107,8 @@ class TActiveRequests { private: THashMap Requests; - THashSet Pointers; + THistory CompletedRequests = THistory(REQUEST_HISTORY_SIZE); + THistory TimedOutRequests = THistory(REQUEST_HISTORY_SIZE); public: ui32 CreateId() @@ -120,7 +124,6 @@ class TActiveRequests void Push(TRequestPtr req) { - Y_ABORT_UNLESS(Pointers.emplace(req.get()).second); Y_ABORT_UNLESS(Requests.emplace(req->ReqId, std::move(req)).second); } @@ -129,7 +132,7 @@ class TActiveRequests auto it = Requests.find(reqId); if (it != Requests.end()) { TRequestPtr req = std::move(it->second); - Pointers.erase(req.get()); + CompletedRequests.Put(it->first); Requests.erase(it); return req; } @@ -143,14 +146,28 @@ class TActiveRequests } auto it = std::begin(Requests); TRequestPtr req = std::move(it->second); - Pointers.erase(req.get()); + CompletedRequests.Put(it->first); Requests.erase(it); return req; } - bool Contains(TRequest* ptr) + TRequest* Get(ui32 reqId) + { + auto it = Requests.find(reqId); + if (it != Requests.end()) { + return it->second.get(); + } + return nullptr; + } + + bool TimedOut(ui32 reqId) const { - return Pointers.contains(ptr); + return TimedOutRequests.Contains(reqId); + } + + bool Completed(ui32 reqId) const + { + return CompletedRequests.Contains(reqId); } TVector PopTimedOutRequests(ui64 timeoutCycles) @@ -166,9 +183,7 @@ class TActiveRequests } for (const auto& x: requests) { - // TODO: keep tombstones to distinguish between timed out requests - // and unknown reqIds - Pointers.erase(x.get()); + TimedOutRequests.Put(x->ReqId); Requests.erase(x->ReqId); } @@ -447,18 +462,18 @@ class TClientEndpoint final TString PeerAddress() const; int ReconnectTimerHandle() const; - // called from client thread + // called from external thread TResultOrError AllocateRequest( IClientHandlerPtr handler, std::unique_ptr context, size_t requestBytes, size_t responseBytes) noexcept override; - void SendRequest( TClientRequestPtr creq, TCallContextPtr callContext) noexcept override; // called from CQ thread + void HandleCompletionEvent(ibv_wc* wc) override; bool HandleInputRequests(); bool HandleCompletionEvents(); bool AbortRequests() noexcept; @@ -466,18 +481,15 @@ class TClientEndpoint final bool FlushHanging() const; private: + // called from CQ thread void HandleQueuedRequests(); bool IsWorkRequestValid(const TWorkRequestId& id) const; void HandleFlush(const TWorkRequestId& id) noexcept; - void HandleCompletionEvent(ibv_wc* wc) override; - void SendRequest(TRequestPtr req, TSendWr* send); void SendRequestCompleted( TSendWr* send, ibv_wc_status status) noexcept; - void RecvResponse(TRecvWr* recv); void RecvResponseCompleted(TRecvWr* recv, ibv_wc_status status); - void AbortRequest(TRequestPtr req, ui32 err, const TString& msg) noexcept; void FreeRequest(TRequest* creq) noexcept; }; @@ -683,6 +695,7 @@ void TClientEndpoint::StartReceive() } } +// implements IClientEndpoint TResultOrError TClientEndpoint::AllocateRequest( IClientHandlerPtr handler, std::unique_ptr context, @@ -731,6 +744,7 @@ TResultOrError TClientEndpoint::AllocateRequest( return TClientRequestPtr(std::move(req)); } +// implements IClientEndpoint void TClientEndpoint::SendRequest( TClientRequestPtr creq, TCallContextPtr callContext) noexcept @@ -879,12 +893,13 @@ void TClientEndpoint::HandleFlush(const TWorkRequestId& id) noexcept } } +// implements NVerbs::ICompletionHandler void TClientEndpoint::HandleCompletionEvent(ibv_wc* wc) { auto id = TWorkRequestId(wc->wr_id); RDMA_TRACE(NVerbs::GetOpcodeName(wc->opcode) << " " << id - << " " << NVerbs::GetStatusString(wc->status)); + << " completed with " << NVerbs::GetStatusString(wc->status)); if (!IsWorkRequestValid(id)) { RDMA_ERROR(LogThrottler.Unexpected, Log, @@ -939,7 +954,7 @@ void TClientEndpoint::SendRequest(TRequestPtr req, TSendWr* send) Counters->SendRequestStarted(); - send->context = req.get(); + send->context = reinterpret_cast(static_cast(req->ReqId)); ActiveRequests.Push(std::move(req)); } @@ -954,7 +969,7 @@ void TClientEndpoint::SendRequestCompleted( if (status != IBV_WC_SUCCESS) { RDMA_ERROR("SEND " << TWorkRequestId(send->wr.wr_id) - << " error: " << NVerbs::GetStatusString(status)); + << ": " << NVerbs::GetStatusString(status)); Counters->SendRequestError(); ReportRdmaError(); @@ -962,20 +977,27 @@ void TClientEndpoint::SendRequestCompleted( return; } - auto* req = static_cast(send->context); + auto reqId = SafeCast(reinterpret_cast(send->context)); + + if (auto* req = ActiveRequests.Get(reqId)) { + LWTRACK( + SendRequestCompleted, + req->CallContext->LWOrbit, + req->CallContext->RequestId); + + } else if (ActiveRequests.TimedOut(reqId)) { + RDMA_INFO("SEND " << TWorkRequestId(send->wr.wr_id) + << ": request has timed out before receiving send wc"); - if (!ActiveRequests.Contains(req)) { - RDMA_WARN("SEND " << TWorkRequestId(send->wr.wr_id) << " error: " - << "request " << reinterpret_cast(req) << " not found") + } else if (ActiveRequests.Completed(reqId)) { + RDMA_INFO("SEND " << TWorkRequestId(send->wr.wr_id) + << ": request has been completed before receiving send wc"); + } else { + RDMA_ERROR("SEND " << TWorkRequestId(send->wr.wr_id) + << ": request not found") Counters->UnknownRequest(); - return; } - - LWTRACK( - SendRequestCompleted, - req->CallContext->LWOrbit, - req->CallContext->RequestId); } void TClientEndpoint::RecvResponse(TRecvWr* recv) @@ -995,7 +1017,7 @@ void TClientEndpoint::RecvResponseCompleted( { if (wc_status != IBV_WC_SUCCESS) { RDMA_ERROR("RECV " << TWorkRequestId(recv->wr.wr_id) - << " error: " << NVerbs::GetStatusString(wc_status)); + << ": " << NVerbs::GetStatusString(wc_status)); Counters->RecvResponseError(); RecvResponse(recv); @@ -1008,7 +1030,7 @@ void TClientEndpoint::RecvResponseCompleted( int version = ParseMessageHeader(msg); if (version != RDMA_PROTO_VERSION) { RDMA_ERROR("RECV " << TWorkRequestId(recv->wr.wr_id) - << " error: unrecognized protocol version") + << ": unrecognized protocol version") Counters->RecvResponseError(); RecvResponse(recv); @@ -1024,7 +1046,7 @@ void TClientEndpoint::RecvResponseCompleted( auto req = ActiveRequests.Pop(reqId); if (!req) { RDMA_ERROR("RECV " << TWorkRequestId(recv->wr.wr_id) - << " error: request " << reqId << " not found"); + << ": request not found"); Counters->UnknownRequest(); return; @@ -1546,31 +1568,30 @@ class TClient final IMonitoringServicePtr monitoring, TClientConfigPtr config); + // called from external thread void Start() noexcept override; void Stop() noexcept override; - TFuture StartEndpoint( TString host, ui32 port) noexcept override; private: + // called from external thread void HandleConnectionEvent( NVerbs::TConnectionEventPtr event) noexcept override; + // called from CM thread void Reconnect(TClientEndpoint* endpont) noexcept override; void BeginResolveAddress(TClientEndpoint* endpoint) noexcept; void BeginResolveRoute(TClientEndpoint* endpoint) noexcept; void BeginConnect(TClientEndpoint* endpoint) noexcept; void HandleDisconnected(TClientEndpoint* endpoint) noexcept; - void HandleConnected( TClientEndpoint* endpoint, NVerbs::TConnectionEventPtr event) noexcept; - void HandleRejected( TClientEndpoint* endpoint, NVerbs::TConnectionEventPtr event) noexcept; - TCompletionPoller& PickPoller() noexcept; }; @@ -1636,6 +1657,7 @@ void TClient::Stop() noexcept CompletionPollers.clear(); } +// implements IClient TFuture TClient::StartEndpoint( TString host, ui32 port) noexcept @@ -1674,6 +1696,7 @@ TFuture TClient::StartEndpoint( //////////////////////////////////////////////////////////////////////////////// +// implements IConnectionEventHandler void TClient::HandleConnectionEvent(NVerbs::TConnectionEventPtr event) noexcept { TClientEndpoint* endpoint = TClientEndpoint::FromEvent(event.get()); @@ -1729,6 +1752,7 @@ void TClient::HandleConnectionEvent(NVerbs::TConnectionEventPtr event) noexcept } } +// implements IConnectionEventHandler void TClient::Reconnect(TClientEndpoint* endpoint) noexcept { if (endpoint->Reconnect.Hanging()) { diff --git a/cloud/blockstore/libs/rdma/impl/server.cpp b/cloud/blockstore/libs/rdma/impl/server.cpp index 707bc9c8caa..5469c4e6356 100644 --- a/cloud/blockstore/libs/rdma/impl/server.cpp +++ b/cloud/blockstore/libs/rdma/impl/server.cpp @@ -321,15 +321,15 @@ class TServerSession final void EnqueueRequest(TRequestPtr req) noexcept; // called from CQ thread + void HandleCompletionEvent(ibv_wc* wc) override; bool HandleInputRequests(); bool HandleCompletionEvents(); bool IsFlushed(); private: - // called inderectly from CQ by 2 previous functions + // called from CQ thread void HandleQueuedRequests(); bool IsWorkRequestValid(const TWorkRequestId& id) const; - void HandleCompletionEvent(ibv_wc* wc) override; void RecvRequest(TRecvWr* recv); void RecvRequestCompleted(TRecvWr* recv, ibv_wc_status status); void ReadRequestData(TRequestPtr req, TSendWr* send); @@ -365,7 +365,7 @@ TServerSession::TServerSession( { Connection->context = this; - STORAGE_INFO("start new session [send_magic=%X recv_magic=%X]", + STORAGE_INFO("new session [send_magic=%X recv_magic=%X]", SendMagic, RecvMagic); CompletionChannel = Verbs->CreateCompletionChannel(Connection->verbs); @@ -610,6 +610,7 @@ bool TServerSession::IsWorkRequestValid(const TWorkRequestId& id) const return false; } +// implements NVerbs::ICompletionHandler void TServerSession::HandleCompletionEvent(ibv_wc* wc) { auto id = TWorkRequestId(wc->wr_id); @@ -1427,13 +1428,13 @@ class TServer final // called from external thread void Start() override; void Stop() override; - IServerEndpointPtr StartEndpoint( TString host, ui32 port, IServerHandlerPtr handler) override; private: + // called from external thread void Listen(TServerEndpoint* endpoint); // called from CM thread @@ -1441,12 +1442,9 @@ class TServer final void HandleConnectRequest( TServerEndpoint* endpoint, rdma_cm_event* event) noexcept; void Accept(TServerEndpoint* endpoint, rdma_cm_event* event) noexcept; - void HandleConnected(TServerSession* session) noexcept; void HandleDisconnected(TServerSession* session) noexcept; - void Reject(rdma_cm_id* id, int status) noexcept; - TCompletionPoller* PickPoller() noexcept; }; @@ -1513,6 +1511,7 @@ void TServer::Stop() CompletionPollers.clear(); } +// implements IServer IServerEndpointPtr TServer::StartEndpoint( TString host, ui32 port, @@ -1567,6 +1566,7 @@ void TServer::Listen(TServerEndpoint* endpoint) //////////////////////////////////////////////////////////////////////////////// +// implements IConnectionEventHandler void TServer::HandleConnectionEvent(rdma_cm_event* event) noexcept { STORAGE_INFO(NVerbs::GetEventName(event->event) << " received"); diff --git a/cloud/blockstore/libs/rdma/impl/work_queue.h b/cloud/blockstore/libs/rdma/impl/work_queue.h index bbe3b2cef6a..1322f6da133 100644 --- a/cloud/blockstore/libs/rdma/impl/work_queue.h +++ b/cloud/blockstore/libs/rdma/impl/work_queue.h @@ -42,7 +42,7 @@ union TWorkRequestId inline IOutputStream& operator<<(IOutputStream& out, const TWorkRequestId& id) { - Printf(out, "%08X:%X:%X", id.Magic, id.Generation, id.Index); + Printf(out, "%08X.%X.%X", id.Magic, id.Generation, id.Index); return out; } diff --git a/cloud/blockstore/libs/service/auth_scheme.cpp b/cloud/blockstore/libs/service/auth_scheme.cpp index 9d473fba736..e37f12478cf 100644 --- a/cloud/blockstore/libs/service/auth_scheme.cpp +++ b/cloud/blockstore/libs/service/auth_scheme.cpp @@ -193,6 +193,7 @@ TPermissionList GetRequestPermissions( perms("reallocatedisk", {EPermission::Update}), perms("rebindvolumes", {EPermission::Update}), perms("setuserid", {EPermission::Update}), + perms("cms", {EPermission::Update}), // Delete perms("deletecheckpointdata", {EPermission::Delete}), diff --git a/cloud/blockstore/libs/storage/core/proto_helpers.h b/cloud/blockstore/libs/storage/core/proto_helpers.h index cdb5fb79585..4517e43ec6f 100644 --- a/cloud/blockstore/libs/storage/core/proto_helpers.h +++ b/cloud/blockstore/libs/storage/core/proto_helpers.h @@ -171,6 +171,14 @@ inline bool RequiresCheckpointSupport(const NProto::TChecksumDeviceBlocksRequest return false; } +inline NProto::EStorageMediaKind GetCheckpointShadowDiskType( + NProto::EStorageMediaKind srcMediaKind) +{ + return srcMediaKind == NProto::STORAGE_MEDIA_HDD_NONREPLICATED + ? NProto::STORAGE_MEDIA_HDD_NONREPLICATED + : NProto::STORAGE_MEDIA_SSD_NONREPLICATED; +} + //////////////////////////////////////////////////////////////////////////////// TBlockRange64 BuildRequestBlockRange( diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp index 7fd1ddaa5ed..717c46cb70a 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp @@ -23,6 +23,8 @@ class TAcquireDiskActor final private: const TActorId Owner; TRequestInfoPtr RequestInfo; + TVector Devices; + const ui32 LogicalBlockSize = 0; const TString DiskId; const TString ClientId; const NProto::EVolumeAccessMode AccessMode; @@ -30,16 +32,14 @@ class TAcquireDiskActor final const ui32 VolumeGeneration; const TDuration RequestTimeout; - TVector Devices; - ui32 LogicalBlockSize = 0; - - NProto::TError AcquireError; int PendingRequests = 0; public: TAcquireDiskActor( const TActorId& owner, TRequestInfoPtr requestInfo, + TVector devices, + ui32 logicalBlockSize, TString diskId, TString clientId, NProto::EVolumeAccessMode accessMode, @@ -53,8 +53,6 @@ class TAcquireDiskActor final void PrepareRequest(NProto::TAcquireDevicesRequest& request); void PrepareRequest(NProto::TReleaseDevicesRequest& request); - void StartAcquireDisk(const TActorContext& ctx); - void FinishAcquireDisk(const TActorContext& ctx); void FinishAcquireDisk(const TActorContext& ctx, NProto::TError error); void ReplyAndDie(const TActorContext& ctx, NProto::TError error); @@ -64,12 +62,11 @@ class TAcquireDiskActor final ui32 nodeId, NProto::TError error); - template + template void SendRequests(const TActorContext& ctx); private: STFUNC(StateAcquire); - STFUNC(StateFinish); void HandlePoisonPill( const TEvents::TEvPoisonPill::TPtr& ev, @@ -79,14 +76,6 @@ class TAcquireDiskActor final const TEvDiskAgent::TEvAcquireDevicesResponse::TPtr& ev, const TActorContext& ctx); - void HandleStartAcquireDiskResponse( - const TEvDiskRegistryPrivate::TEvStartAcquireDiskResponse::TPtr& ev, - const TActorContext& ctx); - - void HandleFinishAcquireDiskResponse( - const TEvDiskRegistryPrivate::TEvFinishAcquireDiskResponse::TPtr& ev, - const TActorContext& ctx); - void HandleAcquireDevicesUndelivery( const TEvDiskAgent::TEvAcquireDevicesRequest::TPtr& ev, const TActorContext& ctx); @@ -103,6 +92,8 @@ class TAcquireDiskActor final TAcquireDiskActor::TAcquireDiskActor( const TActorId& owner, TRequestInfoPtr requestInfo, + TVector devices, + ui32 logicalBlockSize, TString diskId, TString clientId, NProto::EVolumeAccessMode accessMode, @@ -111,6 +102,8 @@ TAcquireDiskActor::TAcquireDiskActor( TDuration requestTimeout) : Owner(owner) , RequestInfo(std::move(requestInfo)) + , Devices(std::move(devices)) + , LogicalBlockSize(logicalBlockSize) , DiskId(std::move(diskId)) , ClientId(std::move(clientId)) , AccessMode(accessMode) @@ -119,35 +112,40 @@ TAcquireDiskActor::TAcquireDiskActor( , RequestTimeout(requestTimeout) { ActivityType = TBlockStoreActivities::DISK_REGISTRY_WORKER; + + SortBy(Devices, [] (auto& d) { + return d.GetNodeId(); + }); } void TAcquireDiskActor::Bootstrap(const TActorContext& ctx) { Become(&TThis::StateAcquire); - StartAcquireDisk(ctx); - ctx.Schedule(RequestTimeout, new TEvents::TEvWakeup()); -} -void TAcquireDiskActor::StartAcquireDisk(const TActorContext& ctx) -{ - using TType = TEvDiskRegistryPrivate::TEvStartAcquireDiskRequest; - NCloud::Send(ctx, Owner, std::make_unique(DiskId, ClientId)); -} + if (Devices.empty()) { + FinishAcquireDisk(ctx, MakeError(E_REJECTED, "nothing to acquire")); + return; + } -void TAcquireDiskActor::FinishAcquireDisk(const TActorContext& ctx) -{ - Become(&TThis::StateFinish); + ctx.Schedule(RequestTimeout, new TEvents::TEvWakeup()); - using TType = TEvDiskRegistryPrivate::TEvFinishAcquireDiskRequest; - NCloud::Send(ctx, Owner, std::make_unique(DiskId, ClientId)); + LOG_DEBUG(ctx, TBlockStoreComponents::DISK_REGISTRY_WORKER, + "[%s] Sending acquire devices requests for disk %s, targets %s", + ClientId.c_str(), + DiskId.c_str(), + LogTargets().c_str()); + + SendRequests(ctx); } void TAcquireDiskActor::FinishAcquireDisk( const TActorContext& ctx, NProto::TError error) { - AcquireError = std::move(error); - FinishAcquireDisk(ctx); + using TType = TEvDiskRegistryPrivate::TEvFinishAcquireDiskRequest; + NCloud::Send(ctx, Owner, std::make_unique(DiskId, ClientId)); + + ReplyAndDie(ctx, std::move(error)); } void TAcquireDiskActor::PrepareRequest(NProto::TAcquireDevicesRequest& request) @@ -164,12 +162,12 @@ void TAcquireDiskActor::PrepareRequest(NProto::TReleaseDevicesRequest& request) request.MutableHeaders()->SetClientId(ClientId); } -template +template void TAcquireDiskActor::SendRequests(const TActorContext& ctx) { auto it = Devices.begin(); while (it != Devices.end()) { - auto request = std::make_unique(); + auto request = std::make_unique(); PrepareRequest(request->Record); const ui32 nodeId = it->GetNodeId(); @@ -270,7 +268,7 @@ void TAcquireDiskActor::OnAcquireResponse( } if (--PendingRequests == 0) { - FinishAcquireDisk(ctx); + FinishAcquireDisk(ctx, {}); } } @@ -304,47 +302,6 @@ void TAcquireDiskActor::HandleWakeup( MakeError(E_REJECTED, "timeout")); } -void TAcquireDiskActor::HandleStartAcquireDiskResponse( - const TEvDiskRegistryPrivate::TEvStartAcquireDiskResponse::TPtr& ev, - const TActorContext& ctx) -{ - const auto* msg = ev->Get(); - - if (msg->GetStatus() != S_OK) { - ReplyAndDie(ctx, msg->GetError()); - return; - } - - LogicalBlockSize = msg->LogicalBlockSize; - Devices = msg->Devices; - - if (Devices.empty()) { - FinishAcquireDisk(ctx, MakeError(E_REJECTED, "nothing to acquire")); - return; - } - - SortBy(Devices, [] (auto& d) { - return d.GetNodeId(); - }); - - LOG_DEBUG(ctx, TBlockStoreComponents::DISK_REGISTRY_WORKER, - "[%s] Sending acquire devices requests for disk %s, targets %s", - ClientId.c_str(), - DiskId.c_str(), - LogTargets().c_str()); - - SendRequests(ctx); -} - -void TAcquireDiskActor::HandleFinishAcquireDiskResponse( - const TEvDiskRegistryPrivate::TEvFinishAcquireDiskResponse::TPtr& ev, - const TActorContext& ctx) -{ - Y_UNUSED(ev); - - ReplyAndDie(ctx, AcquireError); -} - //////////////////////////////////////////////////////////////////////////////// TString TAcquireDiskActor::LogTargets() const @@ -364,9 +321,6 @@ STFUNC(TAcquireDiskActor::StateAcquire) HFunc(TEvDiskAgent::TEvAcquireDevicesRequest, HandleAcquireDevicesUndelivery); - HFunc(TEvDiskRegistryPrivate::TEvStartAcquireDiskResponse, - HandleStartAcquireDiskResponse); - HFunc(TEvents::TEvWakeup, HandleWakeup); default: @@ -375,28 +329,6 @@ STFUNC(TAcquireDiskActor::StateAcquire) } } -STFUNC(TAcquireDiskActor::StateFinish) -{ - switch (ev->GetTypeRewrite()) { - HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); - - HFunc(TEvDiskRegistryPrivate::TEvFinishAcquireDiskResponse, - HandleFinishAcquireDiskResponse); - - IgnoreFunc(TEvDiskRegistryPrivate::TEvStartAcquireDiskResponse); - - IgnoreFunc(TEvents::TEvWakeup); - IgnoreFunc(TEvDiskAgent::TEvAcquireDevicesResponse); - IgnoreFunc(TEvDiskAgent::TEvAcquireDevicesRequest); - IgnoreFunc(TEvDiskAgent::TEvReleaseDevicesResponse); - IgnoreFunc(TEvDiskAgent::TEvReleaseDevicesRequest); - - default: - HandleUnexpectedEvent(ev, TBlockStoreComponents::DISK_REGISTRY_WORKER); - break; - } -} - } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -409,66 +341,70 @@ void TDiskRegistryActor::HandleAcquireDisk( const auto* msg = ev->Get(); - auto requestInfo = CreateRequestInfo( - ev->Sender, - ev->Cookie, - msg->CallContext - ); - auto clientId = msg->Record.GetHeaders().GetClientId(); + auto diskId = msg->Record.GetDiskId(); LOG_DEBUG(ctx, TBlockStoreComponents::DISK_REGISTRY, "[%lu] Received AcquireDisk request: " "DiskId=%s, ClientId=%s, AccessMode=%u, MountSeqNumber=%lu" ", VolumeGeneration=%u", TabletID(), - msg->Record.GetDiskId().c_str(), + diskId.c_str(), clientId.c_str(), static_cast(msg->Record.GetAccessMode()), msg->Record.GetMountSeqNumber(), msg->Record.GetVolumeGeneration()); - auto actor = NCloud::Register( - ctx, - ctx.SelfID, - std::move(requestInfo), - msg->Record.GetDiskId(), - std::move(clientId), - msg->Record.GetAccessMode(), - msg->Record.GetMountSeqNumber(), - msg->Record.GetVolumeGeneration(), - Config->GetAgentRequestTimeout()); - Actors.insert(actor); -} - -//////////////////////////////////////////////////////////////////////////////// + TDiskInfo diskInfo; + auto error = State->StartAcquireDisk(diskId, diskInfo); -void TDiskRegistryActor::HandleStartAcquireDisk( - const TEvDiskRegistryPrivate::TEvStartAcquireDiskRequest::TPtr& ev, - const TActorContext& ctx) -{ - const auto* msg = ev->Get(); + if (HasError(error)) { + LOG_ERROR(ctx, TBlockStoreComponents::DISK_REGISTRY_WORKER, + "[%s] AcquireDisk %s error: %s", + clientId.c_str(), + diskId.c_str(), + FormatError(error).c_str()); - TDiskInfo diskInfo; + NCloud::Reply( + ctx, + *ev, + std::make_unique( + std::move(error))); + return; + } - auto error = State->StartAcquireDisk(msg->DiskId, diskInfo); State->FilterDevicesAtUnavailableAgents(diskInfo); - auto devices = std::move(diskInfo.Devices); + TVector devices = std::move(diskInfo.Devices); for (auto& migration: diskInfo.Migrations) { devices.push_back(std::move(*migration.MutableTargetDevice())); } for (auto& replica: diskInfo.Replicas) { - for (auto& device: replica) { - devices.push_back(std::move(device)); - } + devices.insert(devices.end(), + std::make_move_iterator(replica.begin()), + std::make_move_iterator(replica.end())); } - auto response = std::make_unique( - std::move(error), std::move(devices), diskInfo.LogicalBlockSize); - NCloud::Reply(ctx, *ev, std::move(response)); + auto actor = NCloud::Register( + ctx, + ctx.SelfID, + CreateRequestInfo( + ev->Sender, + ev->Cookie, + msg->CallContext), + std::move(devices), + diskInfo.LogicalBlockSize, + std::move(diskId), + std::move(clientId), + msg->Record.GetAccessMode(), + msg->Record.GetMountSeqNumber(), + msg->Record.GetVolumeGeneration(), + Config->GetAgentRequestTimeout()); + Actors.insert(actor); } +//////////////////////////////////////////////////////////////////////////////// + void TDiskRegistryActor::HandleFinishAcquireDisk( const TEvDiskRegistryPrivate::TEvFinishAcquireDiskRequest::TPtr& ev, const TActorContext& ctx) diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_private.h b/cloud/blockstore/libs/storage/disk_registry/disk_registry_private.h index 1d5fa4d7aa7..be70a3bbf6b 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_private.h +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_private.h @@ -146,7 +146,6 @@ using TVolumeConfig = NKikimrBlockStore::TVolumeConfig; xxx(CleanupDisks, __VA_ARGS__) \ xxx(SecureErase, __VA_ARGS__) \ xxx(CleanupDevices, __VA_ARGS__) \ - xxx(StartAcquireDisk, __VA_ARGS__) \ xxx(FinishAcquireDisk, __VA_ARGS__) \ xxx(RemoveDiskSession, __VA_ARGS__) \ xxx(DestroyBrokenDisks, __VA_ARGS__) \ @@ -171,36 +170,6 @@ using TVolumeConfig = NKikimrBlockStore::TVolumeConfig; struct TEvDiskRegistryPrivate { - // - // StartAcquireDisk - // - - struct TStartAcquireDiskRequest - { - TString DiskId; - TString ClientId; - - TStartAcquireDiskRequest(TString diskId, TString clientId) - : DiskId(std::move(diskId)) - , ClientId(std::move(clientId)) - {} - }; - - struct TStartAcquireDiskResponse - { - TVector Devices; - ui32 LogicalBlockSize = 0; - - TStartAcquireDiskResponse() = default; - - explicit TStartAcquireDiskResponse( - TVector devices, - ui32 logicalBlockSize) - : Devices(std::move(devices)) - , LogicalBlockSize(logicalBlockSize) - {} - }; - // // FinishAcquireDisk // diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp index 12455fa6bb1..27264847990 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -2022,10 +2023,7 @@ NProto::TError TDiskRegistryState::AllocateCheckpoint( auto checkpointDiskId = TCheckpointInfo::MakeCheckpointDiskId(sourceDiskId, checkpointId); - auto checkpointMediaKind = - diskInfo.MediaKind == NProto::STORAGE_MEDIA_HDD_NONREPLICATED - ? NProto::STORAGE_MEDIA_HDD_NONREPLICATED - : NProto::STORAGE_MEDIA_SSD_NONREPLICATED; + auto checkpointMediaKind = GetCheckpointShadowDiskType(diskInfo.MediaKind); TAllocateDiskParams diskParams{ checkpointDiskId, diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp index 7d51cee4fe4..0e09b1a58db 100644 --- a/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp +++ b/cloud/blockstore/libs/storage/disk_registry/disk_registry_ut_session.cpp @@ -335,6 +335,19 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) runtime->GetNodeId(0)); } + bool finished = false; + + runtime->SetObserverFunc( [&] (TAutoPtr& event) { + switch (event->GetTypeRewrite()) { + case TEvDiskRegistryPrivate::EvFinishAcquireDiskResponse: { + finished = true; + break; + } + } + + return TTestActorRuntime::DefaultObserverFunc(event); + }); + { auto response = diskRegistry.AcquireDisk("disk-1", "session-1"); const auto& msg = response->Record; @@ -352,6 +365,8 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) UNIT_ASSERT_VALUES_EQUAL( msg.GetDevices(0).GetNodeId(), runtime->GetNodeId(0)); + + UNIT_ASSERT(finished); } diskRegistry.ReleaseDisk("disk-1", "session-1"); @@ -466,14 +481,9 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) diskRegistry.AllocateDisk("disk-1", 20_GB); { - auto response = diskRegistry.StartAcquireDisk("disk-1", "session-1"); - UNIT_ASSERT(!HasError(response->GetError())); - UNIT_ASSERT_VALUES_EQUAL(response->Devices.size(), 2); - } - - { - auto response = diskRegistry.FinishAcquireDisk("disk-1", "session-1"); + auto response = diskRegistry.AcquireDisk("disk-1", "session-1"); UNIT_ASSERT(!HasError(response->GetError())); + UNIT_ASSERT_VALUES_EQUAL(2, response->Record.DevicesSize()); } { @@ -507,65 +517,6 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetStatus()); } - Y_UNIT_TEST(ShouldHandleTimeoutedStartSession) - { - const auto agent1 = CreateAgentConfig("agent-1", { - Device("dev-1", "uuid-1", "rack-1", 10_GB), - Device("dev-2", "uuid-2", "rack-1", 10_GB) - }); - - auto runtime = TTestRuntimeBuilder() - .WithAgents({ agent1 }) - .Build(); - - TDiskRegistryClient diskRegistry(*runtime); - diskRegistry.WaitReady(); - diskRegistry.SetWritableState(true); - - diskRegistry.UpdateConfig(CreateRegistryConfig(0, { agent1 })); - - RegisterAgents(*runtime, 1); - WaitForAgents(*runtime, 1); - WaitForSecureErase(*runtime, {agent1}); - - diskRegistry.AllocateDisk("disk-1", 20_GB); - - TAutoPtr startAcquireDiskResp; - TAutoPtr finishAcquireDiskReq; - - auto observerFunc = runtime->SetObserverFunc([&] (TAutoPtr& event) { - switch (event->GetTypeRewrite()) { - case TEvDiskRegistryPrivate::EvStartAcquireDiskResponse: { - startAcquireDiskResp = std::move(event); - return TTestActorRuntime::EEventAction::DROP; - } - case TEvDiskRegistryPrivate::EvFinishAcquireDiskRequest: { - finishAcquireDiskReq = std::move(event); - return TTestActorRuntime::EEventAction::DROP; - } - } - - return TTestActorRuntime::DefaultObserverFunc(event); - }); - - diskRegistry.SendAcquireDiskRequest("disk-1", "session-1"); - runtime->AdvanceCurrentTime(TDuration::Seconds(1)); - runtime->DispatchEvents({}, TDuration::MilliSeconds(10)); - - while (!finishAcquireDiskReq) { - runtime->AdvanceCurrentTime(TDuration::Seconds(1)); - runtime->DispatchEvents({}, TDuration::MilliSeconds(10)); - } - - runtime->SetObserverFunc(observerFunc); - - runtime->Send(startAcquireDiskResp.Release()); - runtime->Send(finishAcquireDiskReq.Release()); - - auto response = diskRegistry.RecvAcquireDiskResponse(); - UNIT_ASSERT_VALUES_EQUAL(E_REJECTED, response->GetError().GetCode()); - } - Y_UNIT_TEST(ShouldFailReleaseSessionIfDiskRegistryRestarts) { const auto agent1 = CreateAgentConfig("agent-1", { @@ -590,14 +541,9 @@ Y_UNIT_TEST_SUITE(TDiskRegistryTest) diskRegistry.AllocateDisk("disk-1", 20_GB); { - auto response = diskRegistry.StartAcquireDisk("disk-1", "session-1"); - UNIT_ASSERT(!HasError(response->GetError())); - UNIT_ASSERT_VALUES_EQUAL(response->Devices.size(), 2); - } - - { - auto response = diskRegistry.FinishAcquireDisk("disk-1", "session-1"); + auto response = diskRegistry.AcquireDisk("disk-1", "session-1"); UNIT_ASSERT(!HasError(response->GetError())); + UNIT_ASSERT_VALUES_EQUAL(2, response->Record.DevicesSize()); } auto observerFunc = runtime->SetObserverFunc( [&] (TAutoPtr& event) { diff --git a/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h b/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h index 3bac0753558..8477099ab56 100644 --- a/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h +++ b/cloud/blockstore/libs/storage/disk_registry/testlib/test_env.h @@ -744,18 +744,6 @@ class TDiskRegistryClient std::move(devices)); } - auto CreateStartAcquireDiskRequest(TString diskId, TString clientId) - { - return std::make_unique( - std::move(diskId), std::move(clientId)); - } - - auto CreateFinishAcquireDiskRequest(TString diskId, TString clientId) - { - return std::make_unique( - std::move(diskId), std::move(clientId)); - } - auto CreateRemoveDiskSessionRequest(TString diskId, TString clientId) { return std::make_unique( diff --git a/cloud/blockstore/libs/storage/partition/model/unconfirmed_blob.cpp b/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.cpp similarity index 78% rename from cloud/blockstore/libs/storage/partition/model/unconfirmed_blob.cpp rename to cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.cpp index 1be520e6139..7e1c2337d33 100644 --- a/cloud/blockstore/libs/storage/partition/model/unconfirmed_blob.cpp +++ b/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.cpp @@ -1,11 +1,11 @@ -#include "unconfirmed_blob.h" +#include "blob_unique_id_with_range.h" namespace NCloud::NBlockStore::NStorage::NPartition { //////////////////////////////////////////////////////////////////////////////// bool Overlaps( - const TUnconfirmedBlobs& blobs, + const TCommitIdToBlobUniqueIdWithRange& blobs, ui64 lowCommitId, ui64 highCommitId, const TBlockRange32& blockRange) @@ -31,12 +31,4 @@ bool Overlaps( return false; } -bool Overlaps( - const TUnconfirmedBlobs& blobs, - ui64 commitId, - const TBlockRange32& blockRange) -{ - return Overlaps(blobs, 0, commitId, blockRange); -} - } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/model/unconfirmed_blob.h b/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.h similarity index 57% rename from cloud/blockstore/libs/storage/partition/model/unconfirmed_blob.h rename to cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.h index 3e4e8d0a6b0..d9aed76c194 100644 --- a/cloud/blockstore/libs/storage/partition/model/unconfirmed_blob.h +++ b/cloud/blockstore/libs/storage/partition/model/blob_unique_id_with_range.h @@ -11,32 +11,26 @@ namespace NCloud::NBlockStore::NStorage::NPartition { //////////////////////////////////////////////////////////////////////////////// -struct TUnconfirmedBlob +struct TBlobUniqueIdWithRange { ui64 UniqueId = 0; TBlockRange32 BlockRange; - TUnconfirmedBlob() = default; + TBlobUniqueIdWithRange() = default; - TUnconfirmedBlob(ui64 uniqueId, const TBlockRange32& blockRange) + TBlobUniqueIdWithRange(ui64 uniqueId, const TBlockRange32& blockRange) : UniqueId(uniqueId) , BlockRange(blockRange) {} }; -// mapping from CommitId -using TUnconfirmedBlobs = THashMap>; -using TConfirmedBlobs = THashMap>; +using TCommitIdToBlobUniqueIdWithRange = + THashMap>; bool Overlaps( - const TUnconfirmedBlobs& blobs, + const TCommitIdToBlobUniqueIdWithRange& blobs, ui64 lowCommitId, ui64 highCommitId, const TBlockRange32& blockRange); -bool Overlaps( - const TUnconfirmedBlobs& blobs, - ui64 commitId, - const TBlockRange32& blockRange); - } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/model/ya.make b/cloud/blockstore/libs/storage/partition/model/ya.make index e8e121e7100..da9adb48031 100644 --- a/cloud/blockstore/libs/storage/partition/model/ya.make +++ b/cloud/blockstore/libs/storage/partition/model/ya.make @@ -8,6 +8,7 @@ GENERATE_ENUM_SERIALIZATION(operation_status.h) SRCS( barrier.cpp blob_index.cpp + blob_unique_id_with_range.cpp block.cpp block_index.cpp block_mask.cpp @@ -18,7 +19,6 @@ SRCS( garbage_queue.cpp mixed_index_cache.cpp operation_status.cpp - unconfirmed_blob.cpp ) PEERDIR( diff --git a/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp index a34fbe2f0df..84a24f89421 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_describeblocks.cpp @@ -203,14 +203,14 @@ bool TPartitionActor::PrepareDescribeBlocks( ui64 commitId = args.CommitId; - if (State->OverlapsUnconfirmedBlobs(commitId, args.DescribeRange)) { + if (State->OverlapsUnconfirmedBlobs(0, commitId, args.DescribeRange)) { args.Interrupted = true; return true; } // NOTE: we should also look in confirmed blobs because they are not added // yet - if (State->OverlapsConfirmedBlobs(commitId, args.DescribeRange)) { + if (State->OverlapsConfirmedBlobs(0, commitId, args.DescribeRange)) { args.Interrupted = true; return true; } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp index 3bb5604c47c..0c0ebba37ad 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_readblocks.cpp @@ -987,14 +987,14 @@ bool TPartitionActor::PrepareReadBlocks( ui64 commitId = args.CommitId; - if (State->OverlapsUnconfirmedBlobs(commitId, args.ReadRange)) { + if (State->OverlapsUnconfirmedBlobs(0, commitId, args.ReadRange)) { args.Interrupted = true; return true; } // NOTE: we should also look in confirmed blobs because they are not added // yet - if (State->OverlapsConfirmedBlobs(commitId, args.ReadRange)) { + if (State->OverlapsConfirmedBlobs(0, commitId, args.ReadRange)) { args.Interrupted = true; return true; } diff --git a/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp b/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp index ec5751f63ce..b4daa7d742c 100644 --- a/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp +++ b/cloud/blockstore/libs/storage/partition/part_actor_writemergedblocks.cpp @@ -247,12 +247,10 @@ void TWriteMergedBlocksActor::AddBlobs( ADD_WRITE_RESULT ); } else { - TVector blobs(Reserve(WriteBlobRequests.size())); + TVector blobs(Reserve(WriteBlobRequests.size())); for (const auto& req: WriteBlobRequests) { - blobs.emplace_back( - req.BlobId.UniqueId(), - req.WriteRange); + blobs.emplace_back(req.BlobId.UniqueId(), req.WriteRange); } request = std::make_unique( diff --git a/cloud/blockstore/libs/storage/partition/part_database.cpp b/cloud/blockstore/libs/storage/partition/part_database.cpp index a82eef9da6e..856aaa5912e 100644 --- a/cloud/blockstore/libs/storage/partition/part_database.cpp +++ b/cloud/blockstore/libs/storage/partition/part_database.cpp @@ -1194,7 +1194,7 @@ bool TPartitionDatabase::ReadGarbageBlobs(TVector& blobIds) void TPartitionDatabase::WriteUnconfirmedBlob( const TPartialBlobId& blobId, - const TUnconfirmedBlob& blob) + const TBlobUniqueIdWithRange& blob) { using TTable = TPartitionSchema::UnconfirmedBlobs; @@ -1215,7 +1215,8 @@ void TPartitionDatabase::DeleteUnconfirmedBlob(const TPartialBlobId& blobId) .Delete(); } -bool TPartitionDatabase::ReadUnconfirmedBlobs(TUnconfirmedBlobs& blobs) +bool TPartitionDatabase::ReadUnconfirmedBlobs( + TCommitIdToBlobUniqueIdWithRange& blobs) { using TTable = TPartitionSchema::UnconfirmedBlobs; diff --git a/cloud/blockstore/libs/storage/partition/part_database.h b/cloud/blockstore/libs/storage/partition/part_database.h index 57302364bc7..1bcf3b5cea2 100644 --- a/cloud/blockstore/libs/storage/partition/part_database.h +++ b/cloud/blockstore/libs/storage/partition/part_database.h @@ -5,12 +5,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -209,10 +209,10 @@ class TPartitionDatabase void WriteUnconfirmedBlob( const TPartialBlobId& blobId, - const TUnconfirmedBlob& blob); + const TBlobUniqueIdWithRange& blob); void DeleteUnconfirmedBlob(const TPartialBlobId& blobId); - bool ReadUnconfirmedBlobs(TUnconfirmedBlobs& blobs); + bool ReadUnconfirmedBlobs(TCommitIdToBlobUniqueIdWithRange& blobs); }; } // namespace NCloud::NBlockStore::NStorage::NPartition diff --git a/cloud/blockstore/libs/storage/partition/part_events_private.h b/cloud/blockstore/libs/storage/partition/part_events_private.h index 9696201b3dc..35afa546e32 100644 --- a/cloud/blockstore/libs/storage/partition/part_events_private.h +++ b/cloud/blockstore/libs/storage/partition/part_events_private.h @@ -9,9 +9,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -656,13 +656,13 @@ struct TEvPartitionPrivate struct TAddUnconfirmedBlobsRequest { ui64 CommitId = 0; - TVector Blobs; + TVector Blobs; TAddUnconfirmedBlobsRequest() = default; TAddUnconfirmedBlobsRequest( ui64 commitId, - TVector blobs) + TVector blobs) : CommitId(commitId) , Blobs(std::move(blobs)) {} diff --git a/cloud/blockstore/libs/storage/partition/part_state.cpp b/cloud/blockstore/libs/storage/partition/part_state.cpp index 88ed0a320d2..8a82682badb 100644 --- a/cloud/blockstore/libs/storage/partition/part_state.cpp +++ b/cloud/blockstore/libs/storage/partition/part_state.cpp @@ -608,13 +608,6 @@ bool TPartitionState::OverlapsUnconfirmedBlobs( return Overlaps(UnconfirmedBlobs, lowCommitId, highCommitId, blockRange); } -bool TPartitionState::OverlapsUnconfirmedBlobs( - ui64 commitId, - const TBlockRange32& blockRange) const -{ - return Overlaps(UnconfirmedBlobs, commitId, blockRange); -} - bool TPartitionState::OverlapsConfirmedBlobs( ui64 lowCommitId, ui64 highCommitId, @@ -623,14 +616,8 @@ bool TPartitionState::OverlapsConfirmedBlobs( return Overlaps(ConfirmedBlobs, lowCommitId, highCommitId, blockRange); } -bool TPartitionState::OverlapsConfirmedBlobs( - ui64 commitId, - const TBlockRange32& blockRange) const -{ - return Overlaps(ConfirmedBlobs, commitId, blockRange); -} - -void TPartitionState::InitUnconfirmedBlobs(TUnconfirmedBlobs blobs) +void TPartitionState::InitUnconfirmedBlobs( + TCommitIdToBlobUniqueIdWithRange blobs) { UnconfirmedBlobs = std::move(blobs); for (const auto& entry: UnconfirmedBlobs) { @@ -643,7 +630,7 @@ void TPartitionState::InitUnconfirmedBlobs(TUnconfirmedBlobs blobs) void TPartitionState::WriteUnconfirmedBlob( TPartitionDatabase& db, ui64 commitId, - const TUnconfirmedBlob& blob) + const TBlobUniqueIdWithRange& blob) { auto blobId = MakePartialBlobId(commitId, blob.UniqueId); db.WriteUnconfirmedBlob(blobId, blob); diff --git a/cloud/blockstore/libs/storage/partition/part_state.h b/cloud/blockstore/libs/storage/partition/part_state.h index d309278a1d1..cb161f07dcf 100644 --- a/cloud/blockstore/libs/storage/partition/part_state.h +++ b/cloud/blockstore/libs/storage/partition/part_state.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -1298,16 +1298,18 @@ class TPartitionState } private: - TUnconfirmedBlobs UnconfirmedBlobs; - TConfirmedBlobs ConfirmedBlobs; + TCommitIdToBlobUniqueIdWithRange UnconfirmedBlobs; + // contains entries from UnconfirmedBlobs that have been confirmed but have + // not yet been added to the index + TCommitIdToBlobUniqueIdWithRange ConfirmedBlobs; public: - const TUnconfirmedBlobs& GetUnconfirmedBlobs() const + const TCommitIdToBlobUniqueIdWithRange& GetUnconfirmedBlobs() const { return UnconfirmedBlobs; } - const TConfirmedBlobs& GetConfirmedBlobs() const + const TCommitIdToBlobUniqueIdWithRange& GetConfirmedBlobs() const { return ConfirmedBlobs; } @@ -1317,25 +1319,17 @@ class TPartitionState ui64 highCommitId, const TBlockRange32& blockRange) const; - bool OverlapsUnconfirmedBlobs( - ui64 commitId, - const TBlockRange32& blockRange) const; - bool OverlapsConfirmedBlobs( ui64 lowCommitId, ui64 highCommitId, const TBlockRange32& blockRange) const; - bool OverlapsConfirmedBlobs( - ui64 commitId, - const TBlockRange32& blockRange) const; - - void InitUnconfirmedBlobs(TUnconfirmedBlobs blobs); + void InitUnconfirmedBlobs(TCommitIdToBlobUniqueIdWithRange blobs); void WriteUnconfirmedBlob( TPartitionDatabase& db, ui64 commitId, - const TUnconfirmedBlob& blob); + const TBlobUniqueIdWithRange& blob); void ConfirmedBlobsAdded(TPartitionDatabase& db, ui64 commitId); diff --git a/cloud/blockstore/libs/storage/partition/part_tx.h b/cloud/blockstore/libs/storage/partition/part_tx.h index 299b81e3a5a..e7428cb1fe4 100644 --- a/cloud/blockstore/libs/storage/partition/part_tx.h +++ b/cloud/blockstore/libs/storage/partition/part_tx.h @@ -9,12 +9,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -109,7 +109,7 @@ struct TTxPartition TVector CleanupQueue; TVector NewBlobs; TVector GarbageBlobs; - TUnconfirmedBlobs UnconfirmedBlobs; + TCommitIdToBlobUniqueIdWithRange UnconfirmedBlobs; explicit TLoadState(ui64 blocksCount) : UsedBlocks(blocksCount) @@ -1100,12 +1100,12 @@ struct TTxPartition { const TRequestInfoPtr RequestInfo; ui64 CommitId = 0; - TVector Blobs; + TVector Blobs; TAddUnconfirmedBlobs( TRequestInfoPtr requestInfo, ui64 commitId, - TVector blobs) + TVector blobs) : RequestInfo(std::move(requestInfo)) , CommitId(commitId) , Blobs(std::move(blobs)) diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp index 0f1405e2d4e..57dbf0f45ba 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.cpp @@ -39,27 +39,28 @@ TNonreplicatedPartitionMigrationActor::TNonreplicatedPartitionMigrationActor( , TimeoutCalculator(Config, SrcConfig) {} -void TNonreplicatedPartitionMigrationActor::Bootstrap( +void TNonreplicatedPartitionMigrationActor::OnBootstrap( const NActors::TActorContext& ctx) { - TNonreplicatedPartitionMigrationCommonActor::Bootstrap(ctx); - StartWork(ctx, CreateSrcActor(ctx), CreateDstActor(ctx)); } -void TNonreplicatedPartitionMigrationActor::OnMessage( +bool TNonreplicatedPartitionMigrationActor::OnMessage( + const NActors::TActorContext& ctx, TAutoPtr& ev) { + Y_UNUSED(ctx); + switch (ev->GetTypeRewrite()) { HFunc(TEvVolume::TEvMigrationStateUpdated, HandleMigrationStateUpdated); HFunc( TEvDiskRegistry::TEvFinishMigrationResponse, HandleFinishMigrationResponse); - default: - HandleUnexpectedEvent(ev, TBlockStoreComponents::PARTITION); + return false; break; } + return true; } TDuration TNonreplicatedPartitionMigrationActor::CalculateMigrationTimeout() diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.h index d2c94ac4626..ec3408e45c3 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_actor.h @@ -35,10 +35,10 @@ class TNonreplicatedPartitionMigrationActor final NRdma::IClientPtr rdmaClient, NActors::TActorId statActorId); - void Bootstrap(const NActors::TActorContext& ctx) override; - // IMigrationOwner implementation - void OnMessage(TAutoPtr& ev) override; + void OnBootstrap(const NActors::TActorContext& ctx) override; + bool OnMessage(const NActors::TActorContext& ctx, + TAutoPtr& ev) override; TDuration CalculateMigrationTimeout() override; void OnMigrationProgress( const NActors::TActorContext& ctx, diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.cpp index 85fabc310e6..088a040b648 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.cpp @@ -49,6 +49,8 @@ void TNonreplicatedPartitionMigrationCommonActor::Bootstrap( ScheduleCountersUpdate(ctx); Become(&TThis::StateWork); + + MigrationOwner->OnBootstrap(ctx); } void TNonreplicatedPartitionMigrationCommonActor::MarkMigratedBlocks( @@ -122,6 +124,11 @@ BLOCKSTORE_HANDLE_UNIMPLEMENTED_REQUEST(GetScanDiskStatus, TEvVolume); STFUNC(TNonreplicatedPartitionMigrationCommonActor::StateWork) { + // Give the inheritor the opportunity to process the message first. + if (MigrationOwner->OnMessage(this->ActorContext(), ev)) { + return; + } + switch (ev->GetTypeRewrite()) { HFunc( TEvNonreplPartitionPrivate::TEvUpdateCounters, @@ -167,14 +174,18 @@ STFUNC(TNonreplicatedPartitionMigrationCommonActor::StateWork) HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); default: - // Give the inheritor a chance to process the message. - MigrationOwner->OnMessage(ev); + HandleUnexpectedEvent(ev, TBlockStoreComponents::VOLUME); break; } } STFUNC(TNonreplicatedPartitionMigrationCommonActor::StateZombie) { + // Give the inheritor the opportunity to process the message first. + if (MigrationOwner->OnMessage(this->ActorContext(), ev)) { + return; + } + switch (ev->GetTypeRewrite()) { IgnoreFunc(TEvNonreplPartitionPrivate::TEvUpdateCounters); @@ -211,8 +222,7 @@ STFUNC(TNonreplicatedPartitionMigrationCommonActor::StateZombie) HFunc(TEvents::TEvPoisonTaken, PoisonPillHelper.HandlePoisonTaken); default: - // Give the inheritor a chance to process the message. - MigrationOwner->OnMessage(ev); + HandleUnexpectedEvent(ev, TBlockStoreComponents::VOLUME); break; } } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.h index 5b2f78a5fbd..c9a59a807ac 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_migration_common_actor.h @@ -33,8 +33,14 @@ class IMigrationOwner public: virtual ~IMigrationOwner() = default; - // Delegates the processing of unknown messages to the owner. - virtual void OnMessage(TAutoPtr& ev) = 0; + // Bootstrap for migration owner. + virtual void OnBootstrap(const NActors::TActorContext& ctx) = 0; + + // Delegates the processing of messages to the owner first. + // If true is returned, then the message has been processed. + virtual bool OnMessage( + const NActors::TActorContext& ctx, + TAutoPtr& ev) = 0; // Calculates the time during which a 4MB block should migrate. [[nodiscard]] virtual TDuration CalculateMigrationTimeout() = 0; @@ -99,13 +105,14 @@ class TNonreplicatedPartitionMigrationCommonActor TPartitionDiskCountersPtr SrcCounters; TPartitionDiskCountersPtr DstCounters; - // PoisonPill - TPoisonPillHelper PoisonPillHelper; - // Usage statistics ui64 NetworkBytes = 0; TDuration CpuUsage; +protected: + // PoisonPill + TPoisonPillHelper PoisonPillHelper; + public: TNonreplicatedPartitionMigrationCommonActor( IMigrationOwner* migrationOwner, @@ -129,7 +136,8 @@ class TNonreplicatedPartitionMigrationCommonActor NActors::TActorId srcActorId, NActors::TActorId dstActorId); - // Called from the inheritor to mark ranges that do not need to be processed. + // Called from the inheritor to mark ranges that do not need to be + // processed. void MarkMigratedBlocks(TBlockRange64 range); // Called from the inheritor to get the next processing range. diff --git a/cloud/blockstore/libs/storage/service/service_actor.h b/cloud/blockstore/libs/storage/service/service_actor.h index f56ebf37fc5..854b54db5b3 100644 --- a/cloud/blockstore/libs/storage/service/service_actor.h +++ b/cloud/blockstore/libs/storage/service/service_actor.h @@ -366,6 +366,10 @@ class TServiceActor final TResultOrError CreateGetNameserverNodesActionActor( TRequestInfoPtr requestInfo, TString input); + + TResultOrError CreateCmsActionActor( + TRequestInfoPtr requestInfo, + TString input); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/blockstore/libs/storage/service/service_actor_actions.cpp b/cloud/blockstore/libs/storage/service/service_actor_actions.cpp index 54cba78b27d..44089570db7 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_actions.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_actions.cpp @@ -80,6 +80,7 @@ void TServiceActor::HandleExecuteAction( {"updatediskregistryagentlistparams", &TServiceActor::CreateUpdateDiskRegistryAgentListParamsActor }, {"changestorageconfig", &TServiceActor::CreateChangeStorageConfigActionActor }, {"getnameservernodes", &TServiceActor::CreateGetNameserverNodesActionActor }, + {"cms", &TServiceActor::CreateCmsActionActor }, }; NProto::TError error; diff --git a/cloud/blockstore/libs/storage/service/service_actor_actions_cms.cpp b/cloud/blockstore/libs/storage/service/service_actor_actions_cms.cpp new file mode 100644 index 00000000000..9093ca76538 --- /dev/null +++ b/cloud/blockstore/libs/storage/service/service_actor_actions_cms.cpp @@ -0,0 +1,141 @@ +#include "service_actor.h" + +#include + +#include +#include +#include +#include + +#include + +#include + +namespace NCloud::NBlockStore::NStorage { + +using namespace NActors; + +using namespace NKikimr; + +LWTRACE_USING(BLOCKSTORE_STORAGE_PROVIDER) + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TCmsActor final + : public TActorBootstrapped +{ +private: + const TRequestInfoPtr RequestInfo; + const TString Input; + +public: + TCmsActor( + TRequestInfoPtr requestInfo, + TString input); + + void Bootstrap(const TActorContext& ctx); + +private: + void ReplyAndDie( + const TActorContext& ctx, + std::unique_ptr response); + +private: + STFUNC(StateWork); + + void HandleCmsActionResponse( + const TEvService::TEvCmsActionResponse::TPtr& ev, + const TActorContext& ctx); +}; + +//////////////////////////////////////////////////////////////////////////////// + +TCmsActor::TCmsActor( + TRequestInfoPtr requestInfo, + TString input) + : RequestInfo(std::move(requestInfo)) + , Input(std::move(input)) +{ + ActivityType = TBlockStoreActivities::SERVICE; +} + +void TCmsActor::Bootstrap(const TActorContext& ctx) +{ + auto request = std::make_unique(); + + if (!google::protobuf::util::JsonStringToMessage(Input, &request->Record).ok()) { + auto response = std::make_unique( + MakeError(E_ARGUMENT, "Failed to parse input")); + ReplyAndDie(ctx, std::move(response)); + return; + } + + Become(&TThis::StateWork); + + NCloud::Send(ctx, MakeStorageServiceId(), std::move(request)); +} + +void TCmsActor::ReplyAndDie( + const TActorContext& ctx, + std::unique_ptr response) +{ + LWTRACK( + ResponseSent_Service, + RequestInfo->CallContext->LWOrbit, + "ExecuteAction_cms", + RequestInfo->CallContext->RequestId); + + NCloud::Reply(ctx, *RequestInfo, std::move(response)); + Die(ctx); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TCmsActor::HandleCmsActionResponse( + const TEvService::TEvCmsActionResponse::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + + auto response = std::make_unique( + msg->GetError()); + + google::protobuf::util::MessageToJsonString( + msg->Record, + response->Record.MutableOutput() + ); + + ReplyAndDie(ctx, std::move(response)); +} + +//////////////////////////////////////////////////////////////////////////////// + +STFUNC(TCmsActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + HFunc( + TEvService::TEvCmsActionResponse, + HandleCmsActionResponse); + + default: + HandleUnexpectedEvent(ev, TBlockStoreComponents::SERVICE); + break; + } +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TResultOrError TServiceActor::CreateCmsActionActor( + TRequestInfoPtr requestInfo, + TString input) +{ + return {std::make_unique( + std::move(requestInfo), + std::move(input))}; +} + +} // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp b/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp index 885e76af38d..5bf16140f3b 100644 --- a/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp +++ b/cloud/blockstore/libs/storage/service/service_actor_actions_get_dependent_disks.cpp @@ -151,7 +151,7 @@ void TGetDependentDisksActor::HandleGetDependentDisksResponse( auto* msg = ev->Get(); const auto& error = msg->GetError(); - if (HasError(error)) { + if (HasError(error) && error.GetCode() != E_NOT_FOUND) { HandleError(ctx, error); return; } diff --git a/cloud/blockstore/libs/storage/service/service_ut_actions.cpp b/cloud/blockstore/libs/storage/service/service_ut_actions.cpp index 6532ea2bc0a..8619ef697d4 100644 --- a/cloud/blockstore/libs/storage/service/service_ut_actions.cpp +++ b/cloud/blockstore/libs/storage/service/service_ut_actions.cpp @@ -1352,6 +1352,60 @@ Y_UNIT_TEST_SUITE(TServiceActionsTest) service, runtime); } + + Y_UNIT_TEST(ShouldExecuteCmsAction) + { + TTestEnv env; + TServiceClient service(env.GetRuntime(), SetupTestEnv(env)); + + env.GetRuntime().SetEventFilter([] (auto&, auto& event) { + switch (event->GetTypeRewrite()) { + case TEvService::EvCmsActionRequest: { + auto* msg = event->template Get(); + UNIT_ASSERT_VALUES_EQUAL(1, msg->Record.ActionsSize()); + const auto& action = msg->Record.GetActions(0); + UNIT_ASSERT_EQUAL(NProto::TAction_EType_REMOVE_HOST, action.GetType()); + UNIT_ASSERT_EQUAL("localhost", action.GetHost()); + UNIT_ASSERT(!action.GetDryRun()); + break; + } + case TEvService::EvCmsActionResponse: { + auto* msg = event->template Get(); + auto& r = *msg->Record.AddActionResults(); + *r.MutableResult() = MakeError(E_TRY_AGAIN); + r.SetTimeout(42); + r.AddDependentDisks("vol0"); + break; + } + } + return false; + }); + + NProto::TCmsActionRequest request; + auto& action = *request.AddActions(); + action.SetType(NProto::TAction_EType_REMOVE_HOST); + action.SetHost("localhost"); + + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + + const auto response = service.ExecuteAction("cms", buf); + + NProto::TCmsActionResponse responseProto; + if (!google::protobuf::util::JsonStringToMessage( + response->Record.GetOutput(), + &responseProto).ok()) + { + UNIT_ASSERT(false); + } + + UNIT_ASSERT_VALUES_EQUAL(1, responseProto.ActionResultsSize()); + const auto& r = responseProto.GetActionResults(0); + UNIT_ASSERT_VALUES_EQUAL(42, r.GetTimeout()); + UNIT_ASSERT_VALUES_EQUAL(E_TRY_AGAIN, r.GetResult().GetCode()); + UNIT_ASSERT_VALUES_EQUAL(1, r.DependentDisksSize()); + UNIT_ASSERT_VALUES_EQUAL("vol0", r.GetDependentDisks(0)); + } } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/service/ya.make b/cloud/blockstore/libs/storage/service/ya.make index c4223554768..b777cfabaf0 100644 --- a/cloud/blockstore/libs/storage/service/ya.make +++ b/cloud/blockstore/libs/storage/service/ya.make @@ -7,6 +7,7 @@ SRCS( service_actor_actions_change_disk_device.cpp service_actor_actions_change_storage_config.cpp service_actor_actions_check_blob.cpp + service_actor_actions_cms.cpp service_actor_actions_compact_range.cpp service_actor_actions_configure_volume_balancer.cpp service_actor_actions_create_disk_from_devices.cpp diff --git a/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h b/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h index d55e1d62ea2..2b59b838dfd 100644 --- a/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h +++ b/cloud/blockstore/libs/storage/testlib/disk_registry_proxy_mock.h @@ -135,6 +135,10 @@ class TDiskRegistryProxyMock final TEvDiskRegistry::TEvDeallocateCheckpointRequest, HandleDeallocateCheckpoint); + HFunc( + TEvService::TEvCmsActionRequest, + HandleCmsAction); + IgnoreFunc(NKikimr::TEvLocal::TEvTabletMetrics); default: @@ -179,31 +183,28 @@ class TDiskRegistryProxyMock final const TEvDiskRegistry::TEvAllocateDiskRequest::TPtr& ev, const NActors::TActorContext& ctx) { - const auto* msg = ev->Get(); - auto response = std::make_unique(); + NCloud::Reply(ctx, *ev, DoHandleAllocateDisk(ev->Get())); + } + std::unique_ptr DoHandleAllocateDisk( + const TEvDiskRegistry::TEvAllocateDiskRequest* msg) + { TDiskRegistryState::TPlacementGroup* group = nullptr; if (msg->Record.GetPlacementGroupId()) { - group = State->PlacementGroups.FindPtr(msg->Record.GetPlacementGroupId()); + group = State->PlacementGroups.FindPtr( + msg->Record.GetPlacementGroupId()); if (!group) { - response->Record.MutableError()->CopyFrom( - MakeError(E_NOT_FOUND, "no such group") - ); + return std::make_unique< + TEvDiskRegistry::TEvAllocateDiskResponse>( + MakeError(E_NOT_FOUND, "no such group")); } } - if (response->Record.HasError()) { - NCloud::Reply(ctx, *ev, std::move(response)); - return; - } - if (FAILED(State->CurrentErrorCode)) { - response->Record.MutableError()->CopyFrom( - MakeError(State->CurrentErrorCode, "disk allocation has failed") - ); - - NCloud::Reply(ctx, *ev, std::move(response)); - return; + return std::make_unique( + MakeError( + State->CurrentErrorCode, + "disk allocation has failed")); } const auto& diskId = msg->Record.GetDiskId(); @@ -269,6 +270,12 @@ class TDiskRegistryProxyMock final ++i; } + auto response = std::make_unique(); + + for (const auto& deviceId: State->DeviceReplacementUUIDs) { + *response->Record.AddDeviceReplacementUUIDs() = deviceId; + } + if (bytes) { response->Record.MutableError()->CopyFrom( MakeError(E_BS_OUT_OF_SPACE, "not enough available devices") @@ -285,11 +292,7 @@ class TDiskRegistryProxyMock final response->Record.SetMuteIOErrors(disk.MuteIOErrors); } - for (const auto& deviceId: State->DeviceReplacementUUIDs) { - *response->Record.AddDeviceReplacementUUIDs() = deviceId; - } - - NCloud::Reply(ctx, *ev, std::move(response)); + return response; } void HandleDeallocateDisk( @@ -902,12 +905,44 @@ class TDiskRegistryProxyMock final const TEvDiskRegistry::TEvAllocateCheckpointRequest::TPtr& ev, const NActors::TActorContext& ctx) { - auto response = - std::make_unique(); - response->Record.SetCheckpointDiskId( - ev->Get()->Record.GetSourceDiskId() + - ev->Get()->Record.GetCheckpointId()); - NCloud::Reply(ctx, *ev, std::move(response)); + auto reply = + [&](const NProto::TError& error, const TString& shadowDiskId) + { + auto response = std::make_unique< + TEvDiskRegistry::TEvAllocateCheckpointResponse>(error); + + if (!HasError(response->GetError())) { + response->Record.SetCheckpointDiskId(shadowDiskId); + } + + NCloud::Reply(ctx, *ev, std::move(response)); + }; + + auto& record = ev->Get()->Record; + const auto* srcDisk = State->Disks.FindPtr(record.GetSourceDiskId()); + if (!srcDisk) { + reply(MakeError(E_NOT_FOUND, "Src disk not found"), TString()); + return; + } + + auto shadowDiskId = record.GetSourceDiskId() + record.GetCheckpointId(); + + auto allocateDiskRequest = + std::make_unique(); + allocateDiskRequest->Record.SetDiskId(shadowDiskId); + allocateDiskRequest->Record.SetBlockSize(srcDisk->BlockSize); + ui64 srcDiskBytes = srcDisk->Devices.size() * + srcDisk->Devices[0].GetBlocksCount() * + srcDisk->Devices[0].GetBlockSize(); + allocateDiskRequest->Record.SetBlocksCount( + srcDiskBytes / srcDisk->BlockSize); + allocateDiskRequest->Record.SetPoolName(srcDisk->PoolName); + allocateDiskRequest->Record.SetStorageMediaKind(srcDisk->MediaKind); + + auto allocateShadowDiskResponse = + DoHandleAllocateDisk(allocateDiskRequest.get()); + + reply(allocateShadowDiskResponse->GetError(), shadowDiskId); } void HandleDeallocateCheckpoint( @@ -920,6 +955,17 @@ class TDiskRegistryProxyMock final std::make_unique< TEvDiskRegistry::TEvDeallocateCheckpointResponse>()); } + + void HandleCmsAction( + const TEvService::TEvCmsActionRequest::TPtr& ev, + const NActors::TActorContext& ctx) + { + NCloud::Reply( + ctx, + *ev, + std::make_unique< + TEvService::TEvCmsActionResponse>()); + } }; } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.cpp b/cloud/blockstore/libs/storage/volume/volume_actor.cpp index f7cf2aa016e..670adf76ee3 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor.cpp @@ -368,6 +368,13 @@ TVolumeActor::EStatus TVolumeActor::GetVolumeStatus() const return TVolumeActor::STATUS_OFFLINE; } +NRdma::IClientPtr TVolumeActor::GetRdmaClient() const +{ + return (Config->GetUseNonreplicatedRdmaActor() && State->GetUseRdma()) + ? RdmaClient + : NRdma::IClientPtr{}; +} + ui64 TVolumeActor::GetBlocksCount() const { return State ? State->GetBlocksCount() : 0; diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.h b/cloud/blockstore/libs/storage/volume/volume_actor.h index 72e21507e6a..84fb638e7df 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor.h +++ b/cloud/blockstore/libs/storage/volume/volume_actor.h @@ -454,6 +454,7 @@ class TVolumeActor final static TString GetVolumeStatusString(EStatus status); EStatus GetVolumeStatus() const; + NRdma::IClientPtr GetRdmaClient() const; ui64 GetBlocksCount() const; void ProcessNextPendingClientRequest(const NActors::TActorContext& ctx); @@ -774,11 +775,7 @@ class TVolumeActor final ui64 traceTs); template - NProto::TError ProcessAndValidateReadRequest( - typename TMethod::TRequest::ProtoRecordType& record) const; - - template - NProto::TError ProcessAndValidateRequest( + NProto::TError ProcessAndValidateReadFromCheckpoint( typename TMethod::TRequest::ProtoRecordType& record) const; template diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp index 2c64c24ff15..ecd7123401a 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_forward.cpp @@ -238,7 +238,6 @@ void TVolumeActor::SendRequestToPartition( auto* msg = ev->Get(); - auto selfId = SelfId(); auto callContext = msg->CallContext; msg->CallContext = MakeIntrusive(callContext->RequestId); @@ -246,6 +245,7 @@ void TVolumeActor::SendRequestToPartition( LWTRACK(ForkFailed, callContext->LWOrbit, TMethod::Name, callContext->RequestId); } + auto selfId = SelfId(); auto event = std::make_unique( partActorId, selfId, @@ -297,7 +297,7 @@ void TVolumeActor::FillResponse( //////////////////////////////////////////////////////////////////////////////// template -NProto::TError TVolumeActor::ProcessAndValidateReadRequest( +NProto::TError TVolumeActor::ProcessAndValidateReadFromCheckpoint( typename TMethod::TRequest::ProtoRecordType& record) const { if (!IsDiskRegistryMediaKind(State->GetConfig().GetStorageMediaKind())) { @@ -309,59 +309,49 @@ NProto::TError TVolumeActor::ProcessAndValidateReadRequest( return {}; } - const auto checkpointType = - State->GetCheckpointStore().GetCheckpointType(checkpointId); + const auto checkpointInfo = + State->GetCheckpointStore().GetCheckpoint(checkpointId); - if (!checkpointType) { + auto makeError = [](TString message) + { ui32 flags = 0; SetProtoFlag(flags, NProto::EF_SILENT); - return MakeError( - E_NOT_FOUND, + return MakeError(E_NOT_FOUND, std::move(message), flags); + }; + + if (!checkpointInfo) { + return makeError( TStringBuilder() - << "Checkpoint id=" << checkpointId.Quote() << " not found", - flags); + << "Checkpoint id=" << checkpointId.Quote() << " not found"); } - const bool checkpointValid = - *checkpointType == ECheckpointType::Light || - State->GetCheckpointStore().DoesCheckpointHaveData(checkpointId); - - if (checkpointValid) { + // For light checkpoints read from the disk itself. + if (checkpointInfo->Type == ECheckpointType::Light) { record.ClearCheckpointId(); return {}; } - ui32 flags = 0; - SetProtoFlag(flags, NProto::EF_SILENT); - return MakeError( - E_NOT_FOUND, - TStringBuilder() << "Not found data for checkpoint id=" - << checkpointId.Quote(), flags); -} - -template -NProto::TError TVolumeActor::ProcessAndValidateRequest( - typename TMethod::TRequest::ProtoRecordType& record) const -{ - Y_UNUSED(record); + switch (checkpointInfo->Data) { + case ECheckpointData::DataDeleted: { + // Return error when reading from deleted checkpoint. + return makeError( + TStringBuilder() << "Data for checkpoint id=" + << checkpointId.Quote() << " deleted"); + } break; + case ECheckpointData::DataPresent: { + if (checkpointInfo->ShadowDiskId.empty()) { + // For write blocking checkpoint clear the checkpoint ID to + // read data from the disk itself. + record.ClearCheckpointId(); + return {}; + } + } break; + } + // Will read checkpoint data from shadow disk. return {}; } -template <> -NProto::TError TVolumeActor::ProcessAndValidateRequest( - typename TEvService::TReadBlocksMethod::TRequest::ProtoRecordType& record) const -{ - return ProcessAndValidateReadRequest(record); -} - -template <> -NProto::TError TVolumeActor::ProcessAndValidateRequest( - typename TEvService::TReadBlocksLocalMethod::TRequest::ProtoRecordType& record) const -{ - return ProcessAndValidateReadRequest(record); -} - //////////////////////////////////////////////////////////////////////////////// template @@ -692,13 +682,16 @@ void TVolumeActor::ForwardRequest( } /* - * Other validation. + * Read from checkpoint processing and validation. */ - if (auto error = ProcessAndValidateRequest(msg->Record); + if constexpr (IsReadMethod) { + if (auto error = + ProcessAndValidateReadFromCheckpoint(msg->Record); HasError(error)) - { - replyError(std::move(error)); - return; + { + replyError(std::move(error)); + return; + } } /* diff --git a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp index 46abc43e7cd..9f4f07a106d 100644 --- a/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_actor_startstop.cpp @@ -93,9 +93,10 @@ void TVolumeActor::OnStarted(const TActorContext& ctx) StartCompletionTimestamp = ctx.Now(); LOG_INFO(ctx, TBlockStoreComponents::VOLUME, - "[%lu] Volume started. MountSeqNumber: %lu, generation: %lu, " + "[%lu] Volume %s started. MountSeqNumber: %lu, generation: %lu, " "time: %lu", TabletID(), + State->GetDiskId().Quote().c_str(), State->GetMountSeqNumber(), Executor()->Generation(), GetStartTime().MicroSeconds()); @@ -181,11 +182,6 @@ void TVolumeActor::SetupDiskRegistryBasedPartitions(const TActorContext& ctx) TActorId nonreplicatedActorId; - NRdma::IClientPtr rdmaClient; - if (Config->GetUseNonreplicatedRdmaActor() && State->GetUseRdma()) { - rdmaClient = RdmaClient; - } - const auto& migrations = State->GetMeta().GetMigrations(); const auto& metaReplicas = State->GetMeta().GetReplicas(); @@ -198,7 +194,7 @@ void TVolumeActor::SetupDiskRegistryBasedPartitions(const TActorContext& ctx) Config, nonreplicatedConfig, SelfId(), - std::move(rdmaClient))); + GetRdmaClient())); } else { // nonreplicated disk in migration state nonreplicatedActorId = NCloud::Register( @@ -211,7 +207,7 @@ void TVolumeActor::SetupDiskRegistryBasedPartitions(const TActorContext& ctx) State->GetReadWriteAccessClientId(), nonreplicatedConfig, migrations, - std::move(rdmaClient), + GetRdmaClient(), SelfId())); } } else { @@ -234,7 +230,7 @@ void TVolumeActor::SetupDiskRegistryBasedPartitions(const TActorContext& ctx) nonreplicatedConfig, migrations, std::move(replicas), - std::move(rdmaClient), + GetRdmaClient(), SelfId(), State->GetMeta().GetResyncIndex())); } else { @@ -249,7 +245,7 @@ void TVolumeActor::SetupDiskRegistryBasedPartitions(const TActorContext& ctx) nonreplicatedConfig, migrations, std::move(replicas), - std::move(rdmaClient), + GetRdmaClient(), SelfId())); } } diff --git a/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp b/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp index 1e4a0cbf7ac..b7a51c2d17f 100644 --- a/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp +++ b/cloud/blockstore/libs/storage/volume/volume_ut_checkpoint.cpp @@ -767,7 +767,7 @@ Y_UNIT_TEST_SUITE(TVolumeCheckpointTest) auto response = volume.RecvReadBlocksResponse(); UNIT_ASSERT_VALUES_EQUAL(E_NOT_FOUND, response->GetStatus()); UNIT_ASSERT_VALUES_EQUAL( - "Not found data for checkpoint id=\"c1\"", + "Data for checkpoint id=\"c1\" deleted", response->GetError().GetMessage()); UNIT_ASSERT(HasProtoFlag( response->GetError().GetFlags(), diff --git a/cloud/blockstore/tools/.gitignore b/cloud/blockstore/tools/.gitignore index 2e6caddffa2..5a41990419e 100644 --- a/cloud/blockstore/tools/.gitignore +++ b/cloud/blockstore/tools/.gitignore @@ -1,9 +1,12 @@ analytics/block-data-dump/block-data-dump analytics/calculate-perf-settings/blockstore-calculate-perf-settings +analytics/compaction-sim/blockstore-compaction-sim analytics/dump-event-log/blockstore-dump-event-log analytics/event-log-disk-usage/blockstore-event-log-disk-usage analytics/event-log-stats/blockstore-event-log-stats analytics/event-log-stats/tests/tools-analytics-event-log-stats-tests +analytics/event-log-suffer/blockstore-event-log-suffer +analytics/find-block-accesses/blockstore-find-block-accesses analytics/visualize-event-log/blockstore-visualize-event-log analytics/visualize-trace/blockstore-visualize-trace ci/check_emptiness_test/tests/blockstore-tools-ci-check_emptiness_test-tests @@ -24,9 +27,12 @@ cms/patcher/tests/cloud-blockstore-tools-cms-patcher-tests debug/featureconfig/blockstore-debug-featureconfig debug/formatter/blockstore-debug-formatter debug/readbdev/blockstore-readbdev +fs/cleanup-ext4-meta/cleanup-ext4-meta fs/cleanup-ext4-meta/ut/cloud-blockstore-tools-fs-cleanup-ext4-meta-ut fs/cleanup-fs/cleanup-fs +fs/cleanup-xfs/cleanup-xfs fs/cleanup-xfs/ut/cloud-blockstore-tools-fs-cleanup-xfs-ut +fs/copy_dev/copy_dev fs/detect-raid/detect-raid fs/dump_disk_info/dump_disk_info fs/gen_used_blocks_map/gen_used_blocks_map @@ -36,20 +42,27 @@ http_proxy/blockstore-http-proxy http_proxy/tests/cloud-blockstore-tools-http_proxy-tests http_proxy/ut/blockstore-http-proxy nbd/blockstore-nbd +testing/build-fresh-blob/build-fresh-blob testing/chaos-monkey/chaos-monkey testing/chaos-monkey/tests/tests +testing/eternal_tests/checkpoint-validator/bin/checkpoint-validator testing/eternal_tests/checkpoint-validator/lib/ut/lib-ut testing/eternal_tests/eternal-load/bin/eternal-load testing/eternal_tests/eternal-load/bin/tests/eternal-load-bin-tests +testing/eternal_tests/range-validator/range-validator testing/eternal_tests/test_runner/tests/eternal_tests-test_runner-tests testing/eternal_tests/test_runner/yc-nbs-run-eternal-load-tests testing/fake-conductor/blockstore-fake-conductor testing/fake-nbs/blockstore-fake-nbs +testing/fake-vhost-server/fake-vhost-server testing/generate-agents/generate-agents testing/infra-client/blockstore-infra-client testing/loadtest/bin/blockstore-loadtest testing/loadtest/lib/ut/cloud-blockstore-tools-testing-loadtest-lib-ut +testing/nbd-test/nbd-test testing/notify-mock/notify-mock +testing/pd-metadata-bench/bin/pd-metadata-bench testing/plugintest/blockstore-plugintest testing/rdma-test/rdma-test testing/stable-plugin/yandex-cloud-blockstore-plugin_*.deb +testing/verify-test/verify-test diff --git a/cloud/blockstore/tools/testing/notify-mock/__main__.py b/cloud/blockstore/tools/testing/notify-mock/__main__.py index e85157e461a..d20f436d1e8 100644 --- a/cloud/blockstore/tools/testing/notify-mock/__main__.py +++ b/cloud/blockstore/tools/testing/notify-mock/__main__.py @@ -64,10 +64,10 @@ def main(args, file): server = HTTPServer(('localhost', args.port), create_api(file)) if args.ssl_cert_file is not None or args.ssl_key_file is not None: - server.socket = ssl.wrap_socket( + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(args.ssl_cert_file, args.ssl_key_file) + server.socket = context.wrap_socket( server.socket, - certfile=args.ssl_cert_file, - keyfile=args.ssl_key_file, server_side=True) def signal_handler(signum, frame): diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache.go b/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache.go index a26d19a95e0..7a629d35f1a 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache.go @@ -1,6 +1,7 @@ package cache import ( + "context" "sort" "sync" ) @@ -25,7 +26,11 @@ func (c *chunk) read(start uint64, data []byte) uint64 { //////////////////////////////////////////////////////////////////////////////// +type readFunc = func(context.Context, uint64, []byte) error + type Cache struct { + readOnCacheMiss readFunc + chunkPool sync.Pool chunks map[uint64]*chunk chunksMutex sync.Mutex @@ -33,10 +38,11 @@ type Cache struct { chunkSize uint64 } -func NewCache() *Cache { +func NewCache(readOnCacheMiss readFunc) *Cache { // 4 MiB. chunkSize := 4 * 1024 * 1024 return &Cache{ + readOnCacheMiss: readOnCacheMiss, chunkPool: sync.Pool{ New: func() interface{} { return &chunk{ @@ -70,9 +76,9 @@ func (c *Cache) readChunk( } func (c *Cache) Read( + ctx context.Context, start uint64, data []byte, - readOnCacheMiss func(start uint64, data []byte) error, ) error { end := start + uint64(len(data)) @@ -86,7 +92,7 @@ func (c *Cache) Read( // two chunks). retrievedChunk := c.chunkPool.Get().(*chunk) retrievedChunk.start = chunkStart - err := readOnCacheMiss(retrievedChunk.start, retrievedChunk.data) + err := c.readOnCacheMiss(ctx, retrievedChunk.start, retrievedChunk.data) if err != nil { return err } @@ -102,6 +108,8 @@ func (c *Cache) Read( return nil } +//////////////////////////////////////////////////////////////////////////////// + // Not thread-safe. func (c *Cache) size() uint64 { return uint64(len(c.chunks)) * c.chunkSize diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache_test.go b/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache_test.go index 1d1b10fcadd..4c96cfde17f 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache_test.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/common/cache/cache_test.go @@ -1,6 +1,7 @@ package cache import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -15,13 +16,7 @@ func goCacheRead(cache *Cache, start uint64, wg *errgroup.Group) { data := make([]byte, cache.chunkSize) for i := 0; i < 10000; i++ { - err := cache.Read( - start, - data, - func(start uint64, data []byte) error { - return nil - }, - ) + err := cache.Read(context.Background(), start, data) if err != nil { return err } @@ -35,7 +30,11 @@ func goCacheRead(cache *Cache, start uint64, wg *errgroup.Group) { //////////////////////////////////////////////////////////////////////////////// func TestCacheReadConcurrently(t *testing.T) { - cache := NewCache() + cache := NewCache( + func(context.Context, uint64, []byte) error { + return nil + }, + ) cache.maxCacheSize = cache.chunkSize wg := errgroup.Group{} for i := uint64(0); i <= 3; i++ { diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/common/errors.go b/cloud/disk_manager/internal/pkg/dataplane/url/common/errors.go index 1ec499ba6cb..a9c83c05a49 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/common/errors.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/common/errors.go @@ -30,3 +30,14 @@ func NewSourceInvalidError(format string, args ...interface{}) error { }, ) } + +func NewSourceForbiddenError(format string, args ...interface{}) error { + return errors.NewDetailedError( + fmt.Errorf(format, args...), + &errors.ErrorDetails{ + Code: error_codes.BadSource, + Message: "url source forbidden", + Internal: false, + }, + ) +} diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/common/http_client.go b/cloud/disk_manager/internal/pkg/dataplane/url/common/http_client.go index bb08637f8fd..776a517fb60 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/common/http_client.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/common/http_client.go @@ -167,18 +167,16 @@ func (c *httpClient) head( return nil, errors.NewRetriableError(err) } + if resp.StatusCode == 403 { + return nil, NewSourceForbiddenError("http code %d", resp.StatusCode) + } + if resp.StatusCode >= 400 && resp.StatusCode <= 499 { - return nil, NewSourceNotFoundError( - "url source not found: code %d", - resp.StatusCode, - ) + return nil, NewSourceNotFoundError("http code %d", resp.StatusCode) } if resp.StatusCode >= 500 && resp.StatusCode <= 599 { - return nil, errors.NewRetriableErrorf( - "server error while http range request: code %d", - resp.StatusCode, - ) + return nil, errors.NewRetriableErrorf("http code %d", resp.StatusCode) } return resp, nil @@ -233,7 +231,7 @@ func (c *httpClient) body( resp.StatusCode == http.StatusLocked || resp.StatusCode == http.StatusRequestTimeout { return nil, errors.NewRetriableErrorf( - "range [%v:%v), code %v", + "range [%v:%v), http code %v", start, end, resp.StatusCode, @@ -248,9 +246,13 @@ func (c *httpClient) body( ) } + if resp.StatusCode == 403 { + return nil, NewSourceForbiddenError("http code %d", resp.StatusCode) + } + if resp.StatusCode >= 400 && resp.StatusCode <= 499 { return nil, NewSourceNotFoundError( - "url source not found: range [%v:%v), code %v", + "range [%v:%v), http code %v", start, end, resp.StatusCode, @@ -259,7 +261,7 @@ func (c *httpClient) body( if resp.StatusCode >= 500 && resp.StatusCode <= 599 { return nil, errors.NewRetriableErrorf( - "range [%v:%v), code %v", + "range [%v:%v), http code %v", start, end, resp.StatusCode, diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/common/reader.go b/cloud/disk_manager/internal/pkg/dataplane/url/common/reader.go index 1dac07c915a..a0d1762ce7e 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/common/reader.go +++ b/cloud/disk_manager/internal/pkg/dataplane/url/common/reader.go @@ -1,6 +1,7 @@ package common import ( + "bytes" "context" "encoding/binary" "io" @@ -84,7 +85,7 @@ func NewURLReader( func (r *urlReader) EnableCache() { if r.cache == nil { - r.cache = cache.NewCache() + r.cache = cache.NewCache(r.read) } } @@ -134,12 +135,7 @@ func (r *urlReader) read( return err } - reader, err := r.httpClient.Body( - ctx, - start, - end, - r.etag, - ) + reader, err := r.httpClient.Body(ctx, start, end, r.etag) if err != nil { return err } @@ -171,13 +167,7 @@ func (r *urlReader) Read( return size, nil } - err := r.cache.Read( - start, - data, - func(start uint64, data []byte) error { - return r.read(ctx, start, data) - }, - ) + err := r.cache.Read(ctx, start, data) if err != nil { return 0, err } @@ -200,18 +190,14 @@ func (r *urlReader) ReadBinary( return err } - reader, err := r.httpClient.Body( - ctx, - start, - end, - r.etag, - ) + byteData := make([]byte, size) + + _, err = r.Read(ctx, start, byteData) if err != nil { return err } - defer reader.Close() - err = binary.Read(reader, byteOrder, data) + err = binary.Read(bytes.NewReader(byteData), byteOrder, data) // NBS-3324: interpret all errors as retriable. if err != nil { return errors.NewRetriableError(err) diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go new file mode 100644 index 00000000000..2afeeb0d92b --- /dev/null +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/footer.go @@ -0,0 +1,43 @@ +package vhd + +import "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/common" + +//////////////////////////////////////////////////////////////////////////////// + +const ( + footerCookie = "conectix" +) + +//////////////////////////////////////////////////////////////////////////////// + +// https://learn.microsoft.com/en-us/windows/win32/vstor/about-vhd +type footer struct { + Cookie [8]byte + Features uint32 + FileFormatVersion uint32 + DataOffset uint64 + Timestamp uint32 + CreatorApplication uint32 + CreatorVersion uint32 + CreatorHostOS uint32 + OriginalSize uint64 // Stores the size of the hard disk in bytes at creation time. + CurrentSize uint64 // Stores the current size of the hard disk in bytes. + DiskGeometry uint32 + DiskType uint32 + Checksum uint32 + UniqueID [16]byte + SavedState byte + Reserved [427]byte +} + +func (f footer) validate() error { + if string(f.Cookie[:]) != footerCookie { + return common.NewSourceInvalidError( + "Failed to check vhd footer cookie: expected - %s, actual - %s", + footerCookie, + f.Cookie, + ) + } + + return nil // TODO: Implement. +} diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go new file mode 100644 index 00000000000..1aaffbd370d --- /dev/null +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/header.go @@ -0,0 +1,41 @@ +package vhd + +import "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/common" + +//////////////////////////////////////////////////////////////////////////////// + +const ( + // The dynamic disk header should appear on a sector (512-byte) boundary. + headerOffset = uint64(512) + + headerCookie = "cxsparse" +) + +// https://learn.microsoft.com/en-us/windows/win32/vstor/about-vhd +type header struct { + Cookie [8]byte + DataOffset uint64 + TableOffset uint64 + HeaderVersion uint32 + MaxTableEntries uint32 + BlockSize uint32 + Checksum uint32 + ParentUniqueID [16]byte + ParentTimeStamp uint32 + Reserved1 uint32 + ParentUnicodeName [512]byte + ParentLocatorEntries [192]byte + Reserved2 [256]byte +} + +func (h header) validate() error { + if string(h.Cookie[:]) != headerCookie { + return common.NewSourceInvalidError( + "Failed to check vhd header cookie: expected - %s, actual - %s", + headerCookie, + h.Cookie, + ) + } + + return nil // TODO: Implement. +} diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go new file mode 100644 index 00000000000..e3ac52673f2 --- /dev/null +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/image_map_reader.go @@ -0,0 +1,97 @@ +package vhd + +import ( + "context" + "encoding/binary" + "unsafe" + + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/dataplane/url/common" +) + +// Implementation is based on main source of information: +// https://learn.microsoft.com/en-us/windows/win32/vstor/about-vhd + +//////////////////////////////////////////////////////////////////////////////// + +func NewImageMapReader(reader common.Reader) *ImageMapReader { + return &ImageMapReader{ + reader: reader, + } +} + +type ImageMapReader struct { + reader common.Reader + header header + footer footer +} + +func (r *ImageMapReader) Size() uint64 { + return r.footer.CurrentSize +} + +func (r *ImageMapReader) Read(ctx context.Context) ([]common.ImageMapItem, error) { + // Because the hard disk footer is a crucial part of the hard disk image, + // the footer is mirrored as a header at the front of the file for purposes + // of redundancy. + err := r.readFooter(ctx) + if err != nil { + return []common.ImageMapItem{}, err + } + + err = r.readHeader(ctx) + if err != nil { + return []common.ImageMapItem{}, err + } + + return []common.ImageMapItem{}, nil // TODO: Implement. +} + +//////////////////////////////////////////////////////////////////////////////// + +func (r *ImageMapReader) readFooter(ctx context.Context) error { + footerSize := uint64(unsafe.Sizeof(r.footer)) + if r.reader.Size() < footerSize { + return common.NewSourceInvalidError( + "image size %v is less than footer size %v", + r.reader.Size(), + footerSize, + ) + } + + err := r.reader.ReadBinary( + ctx, + 0, // start + footerSize, + binary.LittleEndian, + &r.footer, + ) + if err != nil { + return err + } + + return r.footer.validate() +} + +func (r *ImageMapReader) readHeader(ctx context.Context) error { + headerSize := uint64(unsafe.Sizeof(r.header)) + if r.reader.Size() < headerOffset+headerSize { + return common.NewSourceInvalidError( + "image size %v is less than header offset and header size %v", + r.reader.Size(), + headerOffset+headerSize, + ) + } + + err := r.reader.ReadBinary( + ctx, + headerOffset, + headerSize, + binary.LittleEndian, + &r.header, + ) + if err != nil { + return err + } + + return r.header.validate() +} diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/vhd/ya.make b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/ya.make new file mode 100644 index 00000000000..737ae24987f --- /dev/null +++ b/cloud/disk_manager/internal/pkg/dataplane/url/vhd/ya.make @@ -0,0 +1,9 @@ +GO_LIBRARY() + +SRCS( + footer.go + header.go + image_map_reader.go +) + +END() diff --git a/cloud/disk_manager/internal/pkg/dataplane/url/ya.make b/cloud/disk_manager/internal/pkg/dataplane/url/ya.make index 6bbdd50dd3a..dd19be1ea1b 100644 --- a/cloud/disk_manager/internal/pkg/dataplane/url/ya.make +++ b/cloud/disk_manager/internal/pkg/dataplane/url/ya.make @@ -13,6 +13,7 @@ END() RECURSE( common qcow2 + vhd vmdk ) diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_nemesis_test/ya.make b/cloud/disk_manager/internal/pkg/facade/disk_service_nemesis_test/ya.make index b43689db0eb..bf3f4240d34 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_nemesis_test/ya.make +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_nemesis_test/ya.make @@ -5,6 +5,9 @@ SET_APPEND(RECIPE_ARGS --multiple-nbs) SET_APPEND(RECIPE_ARGS --encryption) INCLUDE(${ARCADIA_ROOT}/cloud/disk_manager/internal/pkg/facade/testcommon/common.inc) +FORK_SUBTESTS() +SPLIT_FACTOR(4) + GO_XTEST_SRCS( ../disk_service_test/disk_relocation_test.go ../disk_service_test/disk_service_test.go diff --git a/cloud/disk_manager/internal/pkg/services/disks/config/config.proto b/cloud/disk_manager/internal/pkg/services/disks/config/config.proto index 2dcac81a3b0..c9d84a0ab13 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/config/config.proto +++ b/cloud/disk_manager/internal/pkg/services/disks/config/config.proto @@ -16,5 +16,4 @@ message DisksConfig { repeated string OverlayDisksFolderIdWhitelist = 7; repeated string OverlayDisksFolderIdBlacklist = 8; optional string EndedMigrationExpirationTimeout = 9 [default="30m"]; - optional bool EnableOptimizationForOverlayDiskRelocation = 10 [default=true]; } diff --git a/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go b/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go index 39d4a8d37b9..fde52fca5a7 100644 --- a/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go +++ b/cloud/disk_manager/internal/pkg/services/disks/migrate_disk_task.go @@ -219,40 +219,48 @@ func (t *migrateDiskTask) start( } if t.state.RelocateInfo == nil { - if t.disksConfig.GetEnableOptimizationForOverlayDiskRelocation() { - - err := execCtx.SaveStateWithCallback( - ctx, - func(ctx context.Context, tx *persistence.Transaction) error { - relocateInfo, err := t.poolStorage.RelocateOverlayDiskTx( - ctx, - tx, - t.request.Disk, - t.request.DstZoneId, - ) - if err != nil { - return err - } + err := execCtx.SaveStateWithCallback( + ctx, + func(ctx context.Context, tx *persistence.Transaction) error { + relocateInfo, err := t.poolStorage.RelocateOverlayDiskTx( + ctx, + tx, + t.request.Disk, + t.request.DstZoneId, + ) + if err != nil { + return err + } - t.state.RelocateInfo = &protos.RelocateInfo{ - BaseDiskID: relocateInfo.BaseDiskID, - TargetBaseDiskID: relocateInfo.TargetBaseDiskID, - SlotGeneration: relocateInfo.SlotGeneration, - } - return nil - }, - ) - if err != nil { - return err - } + t.state.RelocateInfo = &protos.RelocateInfo{ + BaseDiskID: relocateInfo.BaseDiskID, + TargetBaseDiskID: relocateInfo.TargetBaseDiskID, + SlotGeneration: relocateInfo.SlotGeneration, + } + return nil + }, + ) + if err != nil { + return err + } - // TODO: refactor SaveStateWithCallback method to avoid one more SaveState. - err = execCtx.SaveState(ctx) - if err != nil { - return err - } - } else { - t.state.RelocateInfo = &protos.RelocateInfo{} + // TODO: refactor SaveStateWithCallback method to avoid one more SaveState. + err = execCtx.SaveState(ctx) + if err != nil { + return err + } + } else if len(t.state.RelocateInfo.TargetBaseDiskID) != 0 { + // Need to check that RelocateInfo is still actual. + // OverlayDiskRebasing should be idempotent. + err := t.poolStorage.OverlayDiskRebasing(ctx, storage.RebaseInfo{ + OverlayDisk: t.request.Disk, + BaseDiskID: t.state.RelocateInfo.BaseDiskID, + TargetZoneID: t.request.DstZoneId, + TargetBaseDiskID: t.state.RelocateInfo.TargetBaseDiskID, + SlotGeneration: t.state.RelocateInfo.SlotGeneration, + }) + if err != nil { + return err } } diff --git a/cloud/disk_manager/test/acceptance/test_runner/acceptance_test_runner.py b/cloud/disk_manager/test/acceptance/test_runner/acceptance_test_runner.py index 565a039fb3b..b3d307826b5 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/acceptance_test_runner.py +++ b/cloud/disk_manager/test/acceptance/test_runner/acceptance_test_runner.py @@ -41,7 +41,7 @@ def __init__(self, ycp: YcpWrapper, args: argparse.Namespace): class AcceptanceTestRunner(BaseAcceptanceTestRunner): _test_binary_executor_type = AcceptanceTestBinaryExecutor - _resource_cleaner = AcceptanceTestCleaner + _cleaner_type = AcceptanceTestCleaner @property def _remote_verify_test_path(self) -> str: diff --git a/cloud/disk_manager/test/acceptance/test_runner/base_acceptance_test_runner.py b/cloud/disk_manager/test/acceptance/test_runner/base_acceptance_test_runner.py index f044152c156..b7cc42c0010 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/base_acceptance_test_runner.py +++ b/cloud/disk_manager/test/acceptance/test_runner/base_acceptance_test_runner.py @@ -72,14 +72,14 @@ def __init__( self._s3_host = getattr(args, 's3_host', None) - def run(self, disk_id: str, disk_size: int, disk_blocksize: int) -> None: + def run(self, disk_id: str, disk_size: str, disk_blocksize: str) -> None: self._acceptance_test_cmd.extend( [ '--suffix', ( f'{self._entity_suffix}-' - f'{size_prettifier(disk_size * (1024 ** 3))}-' - f'{size_prettifier(disk_blocksize)}'.lower() + f'{size_prettifier(int(disk_size))}-' + f'{size_prettifier(int(disk_blocksize))}'.lower() ), ] ) @@ -199,13 +199,14 @@ def _initialize_run(self, def _perform_acceptance_test_on_single_disk(self, disk: Ycp.Disk) -> List[str]: # Execute acceptance test on test disk - cleanup_previous_acceptance_tests_results( - self._ycp, - self._args.test_type, - disk.size, - disk.block_size, - self._single_disk_test_ttl, - ) + if self._cleanup_before_tests: + cleanup_previous_acceptance_tests_results( + self._ycp, + self._args.test_type, + int(disk.size), + int(disk.block_size), + self._single_disk_test_ttl, + ) _logger.info(f'Executing acceptance test on disk ') self._setup_binary_executor() self._test_binary_executor.run(disk.id, disk.size, disk.block_size) diff --git a/cloud/disk_manager/test/acceptance/test_runner/cleanup.py b/cloud/disk_manager/test/acceptance/test_runner/cleanup.py index 19ada64d2bd..e75e1e95358 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/cleanup.py +++ b/cloud/disk_manager/test/acceptance/test_runner/cleanup.py @@ -1,7 +1,7 @@ import argparse import logging import re -from datetime import timedelta, datetime +from datetime import datetime, timedelta, timezone from typing import Callable, Any from cloud.blockstore.pylibs.ycp import YcpWrapper @@ -36,7 +36,7 @@ def _cleanup_stale_entities( if ttl is None: _logger.warning("No ttl for entity %s", entity_type) continue - should_be_older_than = datetime.now() - ttl + should_be_older_than = datetime.now(timezone.utc) - ttl if entity.created_at > should_be_older_than: continue for pattern in regexps.get(entity_type, []): @@ -64,7 +64,7 @@ def cleanup_previous_acceptance_tests_results( entity_ttl: timedelta = timedelta(days=1), ): entity_accessors = _get_entity_accessors(ycp) - disk_size = size_prettifier(disk_size * (1024 ** 3)).lower() + disk_size = size_prettifier(disk_size).lower() disk_blocksize = size_prettifier(disk_blocksize).lower() _logger.info( "Performing cleanup for %s disk size %s, disk block size %s", diff --git a/cloud/disk_manager/test/acceptance/test_runner/lib/disk_policy.py b/cloud/disk_manager/test/acceptance/test_runner/lib/disk_policy.py index ea3a9223764..84de635fdfc 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/lib/disk_policy.py +++ b/cloud/disk_manager/test/acceptance/test_runner/lib/disk_policy.py @@ -50,7 +50,8 @@ def obtain(self) -> Ycp.Disk: ) as disk: yield disk except Exception as e: - _logger.info(f'Error: {e}') + _logger.error( + "Error while creating a disk %s", self._name, exc_info=e) raise ResourceExhaustedError(f'Cannot create disk in zone' f' {self._zone_id}') @@ -64,7 +65,7 @@ def __init__(self, ycp: YcpWrapper, zone_id: str, name_regex: str) -> None: def obtain(self) -> Ycp.Disk: disks = sorted( self._ycp.list_disks(), - key=lambda: disk.created_at, + key=lambda disk: disk.created_at, reverse=True, ) for disk in disks: diff --git a/cloud/disk_manager/test/acceptance/test_runner/tests/test.py b/cloud/disk_manager/test/acceptance/test_runner/tests/test.py index 1ef61516e45..d73fbc341de 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/tests/test.py +++ b/cloud/disk_manager/test/acceptance/test_runner/tests/test.py @@ -1,6 +1,6 @@ import contextlib from collections import defaultdict -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import NamedTuple, Type from cloud.disk_manager.test.acceptance.test_runner.\ @@ -94,7 +94,7 @@ class _ArgumentNamespaceMock(NamedTuple): def test_cleanup(): - _now = datetime.now() + _now = datetime.now(timezone.utc) should_delete_inner_test_acceptance_common = [ _Resource( name='acceptance-test-snapshot-acceptance-12931283', @@ -398,21 +398,22 @@ def test_cleanup(): ], RuntimeError, ) + gib = 1024 ** 3 inner_test_cleanup_fixture = [ ( - ('acceptance', 4, 4096, timedelta(days=1)), + ('acceptance', 4 * gib, 4096, timedelta(days=1)), should_delete_inner_test_acceptance_4gib_4kib, ), ( - ('acceptance', 8 * 1024, 2048, timedelta(days=1)), + ('acceptance', 8 * 1024 * gib, 2048, timedelta(days=1)), should_delete_inner_test_acceptance_8tib_2kib, ), ( - ('eternal', 8 * 1024, 16 * 1024 ** 3, timedelta(days=5)), + ('eternal', 8 * 1024 * gib, 16 * 1024 ** 3, timedelta(days=5)), should_delete_inner_test_eternal_8tib_16gib, ), ( - ('eternal', 256, 2 * 1024 ** 2, timedelta(days=5)), + ('eternal', 256 * gib, 2 * 1024 ** 2, timedelta(days=5)), should_delete_inner_test_eternal_256gib_2mib, ), ] diff --git a/cloud/disk_manager/test/acceptance/test_runner/tests/ya.make b/cloud/disk_manager/test/acceptance/test_runner/tests/ya.make index 3b543f9a21c..e6569bd8e40 100644 --- a/cloud/disk_manager/test/acceptance/test_runner/tests/ya.make +++ b/cloud/disk_manager/test/acceptance/test_runner/tests/ya.make @@ -8,4 +8,5 @@ TEST_SRCS( test.py ) -END() \ No newline at end of file +END() + diff --git a/cloud/disk_manager/test/recipe/disk_manager_launcher.py b/cloud/disk_manager/test/recipe/disk_manager_launcher.py index b5622be0697..e364ba42148 100644 --- a/cloud/disk_manager/test/recipe/disk_manager_launcher.py +++ b/cloud/disk_manager/test/recipe/disk_manager_launcher.py @@ -299,6 +299,7 @@ def __init__(self, working_dir, disk_manager_binary_path, with_nemesis, + restart_timings_file, min_restart_period_sec: int = 5, max_restart_period_sec: int = 30): nemesis_binary_path = yatest_common.binary_path( @@ -315,6 +316,8 @@ def __init__(self, str(min_restart_period_sec), "--max-restart-period-sec", str(max_restart_period_sec), + "--restart-timings-file", + restart_timings_file, ] else: command = [disk_manager_binary_path] @@ -363,6 +366,7 @@ def __init__( ensure_path_exists(working_dir) self.__restarts_count_file = os.path.join(working_dir, 'restarts_count_{}.txt'.format(idx)) + restart_timings_file = os.path.join(working_dir, 'restart_timings_{}.txt'.format(idx)) with open(self.__restarts_count_file, 'w') as f: if idx % 2 == 0: f.write(str(idx)) @@ -438,6 +442,7 @@ def __init__( working_dir, disk_manager_binary_path, with_nemesis, + restart_timings_file, min_restart_period_sec=min_restart_period_sec, max_restart_period_sec=max_restart_period_sec, ) diff --git a/cloud/filestore/apps/client/lib/create_session.cpp b/cloud/filestore/apps/client/lib/create_session.cpp new file mode 100644 index 00000000000..575b6c8f4b3 --- /dev/null +++ b/cloud/filestore/apps/client/lib/create_session.cpp @@ -0,0 +1,59 @@ +#include "command.h" + +#include + +namespace NCloud::NFileStore::NClient { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TCreateSessionCommand final + : public TFileStoreCommand +{ +private: + TString SessionId; + ui64 SeqNo = 0; + +public: + TCreateSessionCommand() + { + Opts.AddLongOption("session-id") + .RequiredArgument("SESSION_ID") + .StoreResult(&SessionId); + + Opts.AddLongOption("client-id") + .RequiredArgument("CLIENT_ID") + .StoreResult(&ClientId); + + Opts.AddLongOption("seq-no") + .RequiredArgument("SEQ_NO") + .StoreResult(&SeqNo); + } + + bool Execute() override + { + auto request = std::make_shared(); + request->SetFileSystemId(FileSystemId); + request->MutableHeaders()->SetSessionId(SessionId); + request->MutableHeaders()->SetClientId(ClientId); + request->MutableHeaders()->SetSessionSeqNo(SeqNo); + + TCallContextPtr ctx = MakeIntrusive(); + auto response = WaitFor(Client->CreateSession(ctx, std::move(request))); + CheckResponse(response); + + return true; + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TCommandPtr NewCreateSessionCommand() +{ + return std::make_shared(); +} + +} // namespace NCloud::NFileStore::NClient diff --git a/cloud/filestore/apps/client/lib/factory.cpp b/cloud/filestore/apps/client/lib/factory.cpp index 52d80ec2d20..11bb85e9f65 100644 --- a/cloud/filestore/apps/client/lib/factory.cpp +++ b/cloud/filestore/apps/client/lib/factory.cpp @@ -29,6 +29,8 @@ TCommandPtr NewStopEndpointCommand(); TCommandPtr NewTouchCommand(); TCommandPtr NewWriteCommand(); TCommandPtr NewExecuteActionCommand(); +TCommandPtr NewCreateSessionCommand(); +TCommandPtr NewResetSessionCommand(); //////////////////////////////////////////////////////////////////////////////// @@ -56,6 +58,8 @@ static const TMap Commands = { { "touch", NewTouchCommand }, { "write", NewWriteCommand }, { "executeaction", NewExecuteActionCommand }, + { "createsession", NewCreateSessionCommand }, + { "resetsession", NewResetSessionCommand }, }; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/apps/client/lib/reset_session.cpp b/cloud/filestore/apps/client/lib/reset_session.cpp new file mode 100644 index 00000000000..7740a789341 --- /dev/null +++ b/cloud/filestore/apps/client/lib/reset_session.cpp @@ -0,0 +1,73 @@ +#include "command.h" + +#include + +#include + +namespace NCloud::NFileStore::NClient { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TResetSessionCommand final + : public TFileStoreCommand +{ +private: + TString SessionId; + ui64 SeqNo = 0; + TString SessionState; + +public: + TResetSessionCommand() + { + Opts.AddLongOption("session-id") + .RequiredArgument("SESSION_ID") + .Required() + .StoreResult(&SessionId); + + Opts.AddLongOption("client-id") + .RequiredArgument("CLIENT_ID") + .Required() + .StoreResult(&ClientId); + + Opts.AddLongOption("seq-no") + .RequiredArgument("SEQ_NO") + .StoreResult(&SeqNo); + + Opts.AddLongOption("session-state") + .RequiredArgument("BASE64_SESSION_STATE") + .StoreResult(&SessionState); + } + + bool Execute() override + { + Headers.SetSessionId(SessionId); + Headers.SetClientId(ClientId); + Headers.SetSessionSeqNo(SeqNo); + + auto request = CreateRequest(); + if (SessionState) { + request->SetSessionState(Base64Decode(SessionState)); + } + + auto response = WaitFor(Client->ResetSession( + PrepareCallContext(), + std::move(request))); + + CheckResponse(response); + + return true; + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TCommandPtr NewResetSessionCommand() +{ + return std::make_shared(); +} + +} // namespace NCloud::NFileStore::NClient diff --git a/cloud/filestore/apps/client/lib/ya.make b/cloud/filestore/apps/client/lib/ya.make index f8986e31561..805771bfe01 100644 --- a/cloud/filestore/apps/client/lib/ya.make +++ b/cloud/filestore/apps/client/lib/ya.make @@ -4,6 +4,7 @@ SRCS( add_cluster_node.cpp command.cpp create.cpp + create_session.cpp describe.cpp destroy.cpp execute_action.cpp @@ -18,6 +19,7 @@ SRCS( performance_profile_params.cpp read.cpp remove_cluster_node.cpp + reset_session.cpp resize.cpp rm.cpp start_endpoint.cpp diff --git a/cloud/filestore/libs/storage/api/tablet.h b/cloud/filestore/libs/storage/api/tablet.h index 8d219e0b379..4c5cd91a5e5 100644 --- a/cloud/filestore/libs/storage/api/tablet.h +++ b/cloud/filestore/libs/storage/api/tablet.h @@ -23,6 +23,7 @@ namespace NCloud::NFileStore::NStorage { xxx(GetStorageConfigFields, __VA_ARGS__) \ xxx(ChangeStorageConfig, __VA_ARGS__) \ xxx(DescribeData, __VA_ARGS__) \ + xxx(DescribeSessions, __VA_ARGS__) \ // FILESTORE_TABLET_REQUESTS //////////////////////////////////////////////////////////////////////////////// @@ -61,6 +62,9 @@ struct TEvIndexTablet EvDescribeDataRequest = EvBegin + 15, EvDescribeDataResponse, + EvDescribeSessionsRequest = EvBegin + 17, + EvDescribeSessionsResponse, + EvEnd }; diff --git a/cloud/filestore/libs/storage/model/block_buffer.cpp b/cloud/filestore/libs/storage/model/block_buffer.cpp index 6d2e5dc67db..44eb1422ac8 100644 --- a/cloud/filestore/libs/storage/model/block_buffer.cpp +++ b/cloud/filestore/libs/storage/model/block_buffer.cpp @@ -2,6 +2,8 @@ #include +#include + namespace NCloud::NFileStore::NStorage { namespace { @@ -57,15 +59,63 @@ class TBlockBuffer final char* ptr = const_cast(Buffer.data()) + offset; memset(ptr, 0, ByteRange.BlockSize); } +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TLazyBlockBuffer final + : public IBlockBuffer +{ +private: + const TByteRange ByteRange; + TString UnalignedHead; + TVector Blocks; + TString UnalignedTail; + +public: + explicit TLazyBlockBuffer(TByteRange byteRange) + : ByteRange(byteRange) + , Blocks(ByteRange.BlockCount()) + { + } + + TStringBuf GetUnalignedHead() override + { + if (!UnalignedHead) { + UnalignedHead.ReserveAndResize(ByteRange.UnalignedHeadLength()); + } + + return UnalignedHead; + } + + TStringBuf GetUnalignedTail() override + { + if (!UnalignedTail) { + UnalignedTail.ReserveAndResize(ByteRange.UnalignedTailLength()); + } + + return UnalignedTail; + } - TStringBuf GetContentRef() override + TStringBuf GetBlock(size_t index) override + { + auto& block = Blocks[index]; + if (!block) { + block.ReserveAndResize(ByteRange.BlockSize); + } + + return block; + } + + void SetBlock(size_t index, TStringBuf block) override { - return Buffer; + Y_ABORT_UNLESS(block.size() == ByteRange.BlockSize); + Blocks[index] = block; } - TString GetContent() override + void ClearBlock(size_t index) override { - return Buffer; + Blocks[index].clear(); } }; @@ -86,4 +136,59 @@ IBlockBufferPtr CreateBlockBuffer(TByteRange byteRange, TString buffer) return std::make_shared(byteRange, std::move(buffer)); } +IBlockBufferPtr CreateLazyBlockBuffer(TByteRange byteRange) +{ + return std::make_shared(byteRange); +} + +//////////////////////////////////////////////////////////////////////////////// + +void CopyFileData( + const TString& logTag, + const TByteRange origin, + const TByteRange aligned, + const ui64 fileSize, + IBlockBuffer& buffer, + TString* out) +{ + const auto end = Min(fileSize, origin.End()); + if (end <= origin.Offset) { + return; + } + + out->ReserveAndResize(end - origin.Offset); + char* outPtr = out->begin(); + + ui32 i = 0; + + // processing unaligned head + if (origin.Offset > aligned.Offset) { + const auto block = buffer.GetBlock(i); + const auto shift = origin.Offset - aligned.Offset; + const auto sz = Min(block.Size() - shift, out->Size()); + STORAGE_VERIFY( + outPtr + sz <= out->end(), + TWellKnownEntityTypes::FILESYSTEM, + logTag); + memcpy(outPtr, block.Data() + shift, sz); + outPtr += sz; + + ++i; + } + + while (i < aligned.BlockCount()) { + const auto block = buffer.GetBlock(i); + // Min needed to properly process unaligned tail + const auto len = Min(out->end() - outPtr, block.Size()); + if (!len) { + // origin.End() is greater than fileSize + break; + } + memcpy(outPtr, block.Data(), len); + outPtr += len; + + ++i; + } +} + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/model/block_buffer.h b/cloud/filestore/libs/storage/model/block_buffer.h index 898f2526e9f..d71a35c76e1 100644 --- a/cloud/filestore/libs/storage/model/block_buffer.h +++ b/cloud/filestore/libs/storage/model/block_buffer.h @@ -20,14 +20,22 @@ struct IBlockBuffer virtual TStringBuf GetUnalignedTail() = 0; virtual void SetBlock(size_t index, TStringBuf block) = 0; virtual void ClearBlock(size_t index) = 0; - - virtual TStringBuf GetContentRef() = 0; - virtual TString GetContent() = 0; }; //////////////////////////////////////////////////////////////////////////////// IBlockBufferPtr CreateBlockBuffer(TByteRange byteRange); IBlockBufferPtr CreateBlockBuffer(TByteRange byteRange, TString buffer); +IBlockBufferPtr CreateLazyBlockBuffer(TByteRange byteRange); + +//////////////////////////////////////////////////////////////////////////////// + +void CopyFileData( + const TString& logTag, + const TByteRange origin, + const TByteRange aligned, + const ui64 fileSize, + IBlockBuffer& buffer, + TString* out); } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/model/block_buffer_ut.cpp b/cloud/filestore/libs/storage/model/block_buffer_ut.cpp index 1c52035b916..a1595f7685b 100644 --- a/cloud/filestore/libs/storage/model/block_buffer_ut.cpp +++ b/cloud/filestore/libs/storage/model/block_buffer_ut.cpp @@ -2,15 +2,65 @@ #include +#include + namespace NCloud::NFileStore::NStorage { //////////////////////////////////////////////////////////////////////////////// Y_UNIT_TEST_SUITE(TBlockBufferTest) { - Y_UNIT_TEST(Test) + struct TTest + { + TByteRange Range; + TByteRange AlignedRange; + TString Data; + IBlockBufferPtr BlockBuffer; + TString Out; + + TTest( + ui64 offset, + ui64 length, + ui32 blockSize, + ui64 fileSize) + : Range(offset, length, blockSize) + , AlignedRange(Range.AlignedSuperRange()) + { + Data.ReserveAndResize(AlignedRange.Length); + for (ui32 i = 0; i < Data.Size(); ++i) { + Data[i] = 'a' + RandomNumber('z' - 'a' + 1); + } + BlockBuffer = CreateBlockBuffer(AlignedRange, Data); + CopyFileData( + "logtag", + Range, + AlignedRange, + fileSize, + *BlockBuffer, + &Out); + } + }; + + Y_UNIT_TEST(ShouldCopyFileDataAligned) + { + TTest test(100_KB, 8_KB, 4_KB, 200_KB); + UNIT_ASSERT_VALUES_EQUAL(test.Data, test.Out); + } + + Y_UNIT_TEST(ShouldCopyFileDataUnaligned) + { + TTest test(101_KB, 10_KB, 4_KB, 107_KB); + UNIT_ASSERT_VALUES_EQUAL( + test.Data.substr(1_KB, 6_KB), + test.Out); + } + + Y_UNIT_TEST(ShouldCopyFileDataUnalignedSmall) { - // TODO + TTest test(101_KB, 10_KB, 4_KB, 103_KB); + UNIT_ASSERT_VALUES_EQUAL( + test.Data.substr(1_KB, 2_KB), + test.Out); } } diff --git a/cloud/filestore/libs/storage/model/ya.make b/cloud/filestore/libs/storage/model/ya.make index 2b4afba512b..5dd8647d60f 100644 --- a/cloud/filestore/libs/storage/model/ya.make +++ b/cloud/filestore/libs/storage/model/ya.make @@ -12,4 +12,8 @@ SRCS( utils.cpp ) +PEERDIR( + cloud/storage/core/libs/common +) + END() diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h index dd6b4a54f89..9e293287aeb 100644 --- a/cloud/filestore/libs/storage/service/service_actor.h +++ b/cloud/filestore/libs/storage/service/service_actor.h @@ -146,6 +146,10 @@ class TStorageServiceActor final TRequestInfoPtr requestInfo, TString input); + NActors::IActorPtr CreateDescribeSessionsActionActor( + TRequestInfoPtr requestInfo, + TString input); + private: void RenderSessions(IOutputStream& out); void RenderLocalFileStores(IOutputStream& out); diff --git a/cloud/filestore/libs/storage/service/service_actor_actions.cpp b/cloud/filestore/libs/storage/service/service_actor_actions.cpp index e9428b1d1d0..3786aa6bdc7 100644 --- a/cloud/filestore/libs/storage/service/service_actor_actions.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_actions.cpp @@ -44,6 +44,10 @@ void TStorageServiceActor::HandleExecuteAction( "changestorageconfig", &TStorageServiceActor::CreateChangeStorageConfigActionActor }, + { + "describesessions", + &TStorageServiceActor::CreateDescribeSessionsActionActor + }, }; auto it = actions.find(action); diff --git a/cloud/filestore/libs/storage/service/service_actor_actions_change_storage_config.cpp b/cloud/filestore/libs/storage/service/service_actor_actions_change_storage_config.cpp index 9855fe1c2b7..af4a51cbd45 100644 --- a/cloud/filestore/libs/storage/service/service_actor_actions_change_storage_config.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_actions_change_storage_config.cpp @@ -41,7 +41,7 @@ class TChangeStorageConfigActionActor final private: void ReplyAndDie( const TActorContext& ctx, - NProtoPrivate::TChangeStorageConfigResponse response); + const NProtoPrivate::TChangeStorageConfigResponse& response); private: STFUNC(StateWork); @@ -85,9 +85,9 @@ void TChangeStorageConfigActionActor::Bootstrap(const TActorContext& ctx) "Start to change storage config of %s", request.GetFileSystemId().Quote().c_str()); - auto requestToTablets = + auto requestToTablet = std::make_unique(); - auto& record = requestToTablets->Record; + auto& record = requestToTablet->Record; record.SetFileSystemId(request.GetFileSystemId()); *record.MutableStorageConfig() = request.GetStorageConfig(); record.SetMergeWithStorageConfigFromTabletDB( @@ -96,20 +96,20 @@ void TChangeStorageConfigActionActor::Bootstrap(const TActorContext& ctx) NCloud::Send( ctx, MakeIndexTabletProxyServiceId(), - std::move(requestToTablets)); + std::move(requestToTablet)); Become(&TThis::StateWork); } void TChangeStorageConfigActionActor::ReplyAndDie( const TActorContext& ctx, - NProtoPrivate::TChangeStorageConfigResponse response) + const NProtoPrivate::TChangeStorageConfigResponse& response) { auto msg = std::make_unique( response.GetError()); google::protobuf::util::MessageToJsonString( - std::move(response), + response, msg->Record.MutableOutput()); NCloud::Reply(ctx, *RequestInfo, std::move(msg)); @@ -122,14 +122,14 @@ void TChangeStorageConfigActionActor::HandleChangeStorageConfigResponse( const TEvIndexTablet::TEvChangeStorageConfigResponse::TPtr& ev, const TActorContext& ctx) { - auto* msg = ev->Get(); + const auto* msg = ev->Get(); if (SUCCEEDED(msg->GetStatus())) { NCloud::Send( ctx, ev->Sender, std::make_unique()); } - ReplyAndDie(ctx, std::move(msg->Record)); + ReplyAndDie(ctx, msg->Record); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/storage/service/service_actor_actions_describe_sessions.cpp b/cloud/filestore/libs/storage/service/service_actor_actions_describe_sessions.cpp new file mode 100644 index 00000000000..da2e95bd6d6 --- /dev/null +++ b/cloud/filestore/libs/storage/service/service_actor_actions_describe_sessions.cpp @@ -0,0 +1,144 @@ +#include "service_actor.h" + +#include "helpers.h" + +#include +#include +#include +#include +#include + +#include + +#include + +namespace NCloud::NFileStore::NStorage { + +using namespace NActors; + +using namespace NKikimr; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TDescribeSessionsActionActor final + : public TActorBootstrapped +{ +private: + const TRequestInfoPtr RequestInfo; + const TString Input; + +public: + TDescribeSessionsActionActor( + TRequestInfoPtr requestInfo, + TString input); + + void Bootstrap(const TActorContext& ctx); + +private: + void ReplyAndDie( + const TActorContext& ctx, + const NProtoPrivate::TDescribeSessionsResponse& response); + +private: + STFUNC(StateWork); + + void HandleDescribeSessionsResponse( + const TEvIndexTablet::TEvDescribeSessionsResponse::TPtr& ev, + const TActorContext& ctx); +}; + +//////////////////////////////////////////////////////////////////////////////// + +TDescribeSessionsActionActor::TDescribeSessionsActionActor( + TRequestInfoPtr requestInfo, + TString input) + : RequestInfo(std::move(requestInfo)) + , Input(std::move(input)) +{ + ActivityType = TFileStoreActivities::SERVICE_WORKER; +} + +void TDescribeSessionsActionActor::Bootstrap(const TActorContext& ctx) +{ + NProtoPrivate::TDescribeSessionsRequest request; + if (!google::protobuf::util::JsonStringToMessage(Input, &request).ok()) { + ReplyAndDie( + ctx, + TErrorResponse(E_ARGUMENT, "Failed to parse input")); + return; + } + + if (!request.GetFileSystemId()) { + ReplyAndDie( + ctx, + TErrorResponse(E_ARGUMENT, "FileSystem id should be supplied")); + return; + } + + auto requestToTablet = + std::make_unique(); + + auto& record = requestToTablet->Record; + record.SetFileSystemId(request.GetFileSystemId()); + + NCloud::Send( + ctx, + MakeIndexTabletProxyServiceId(), + std::move(requestToTablet)); + + Become(&TThis::StateWork); +} + +void TDescribeSessionsActionActor::ReplyAndDie( + const TActorContext& ctx, + const NProtoPrivate::TDescribeSessionsResponse& response) +{ + auto msg = std::make_unique( + response.GetError()); + + google::protobuf::util::MessageToJsonString( + response, + msg->Record.MutableOutput()); + + NCloud::Reply(ctx, *RequestInfo, std::move(msg)); + Die(ctx); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TDescribeSessionsActionActor::HandleDescribeSessionsResponse( + const TEvIndexTablet::TEvDescribeSessionsResponse::TPtr& ev, + const TActorContext& ctx) +{ + ReplyAndDie(ctx, ev->Get()->Record); +} + +//////////////////////////////////////////////////////////////////////////////// + +STFUNC(TDescribeSessionsActionActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + HFunc( + TEvIndexTablet::TEvDescribeSessionsResponse, + HandleDescribeSessionsResponse); + + default: + HandleUnexpectedEvent(ev, TFileStoreComponents::SERVICE); + break; + } +} + +} // namespace + +IActorPtr TStorageServiceActor::CreateDescribeSessionsActionActor( + TRequestInfoPtr requestInfo, + TString input) +{ + return std::make_unique( + std::move(requestInfo), + std::move(input)); +} + +} // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp b/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp index af25dad1ac9..55a4dacee4e 100644 --- a/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_actions_get_storage_config_fields.cpp @@ -39,7 +39,7 @@ class TGetStorageConfigFieldsActionActor final private: void ReplyAndDie( const TActorContext& ctx, - NProtoPrivate::TGetStorageConfigFieldsResponse response); + const NProtoPrivate::TGetStorageConfigFieldsResponse& response); private: STFUNC(StateWork); @@ -77,30 +77,30 @@ void TGetStorageConfigFieldsActionActor::Bootstrap(const TActorContext& ctx) return; } - auto requestToTablets = + auto requestToTablet = std::make_unique(); - auto& record = requestToTablets->Record; + auto& record = requestToTablet->Record; *record.MutableStorageConfigFields() = request.GetStorageConfigFields(); record.SetFileSystemId(request.GetFileSystemId()); NCloud::Send( ctx, MakeIndexTabletProxyServiceId(), - std::move(requestToTablets)); + std::move(requestToTablet)); Become(&TThis::StateWork); } void TGetStorageConfigFieldsActionActor::ReplyAndDie( const TActorContext& ctx, - NProtoPrivate::TGetStorageConfigFieldsResponse response) + const NProtoPrivate::TGetStorageConfigFieldsResponse& response) { auto msg = std::make_unique( response.GetError()); google::protobuf::util::MessageToJsonString( - std::move(response), + response, msg->Record.MutableOutput()); NCloud::Reply(ctx, *RequestInfo, std::move(msg)); @@ -113,7 +113,7 @@ void TGetStorageConfigFieldsActionActor::HandleGetStorageConfigFieldsResponse( const TEvIndexTablet::TEvGetStorageConfigFieldsResponse::TPtr& ev, const TActorContext& ctx) { - ReplyAndDie(ctx, std::move(ev->Get()->Record)); + ReplyAndDie(ctx, ev->Get()->Record); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp index a33240c9896..2b4218bc1b3 100644 --- a/cloud/filestore/libs/storage/service/service_actor_readdata.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_readdata.cpp @@ -29,6 +29,7 @@ class TReadDataActor final: public TActorBootstrapped NProto::TReadDataRequest ReadRequest; // Filesystem-specific params + const TString LogTag; const ui32 BlockSize; // Response data @@ -43,6 +44,7 @@ class TReadDataActor final: public TActorBootstrapped TReadDataActor( TRequestInfoPtr requestInfo, NProto::TReadDataRequest readRequest, + TString logTag, ui32 blockSize); void Bootstrap(const TActorContext& ctx); @@ -66,7 +68,7 @@ class TReadDataActor final: public TActorBootstrapped const TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); - void ReadData(const TActorContext& ctx); + void ReadData(const TActorContext& ctx, const TString& fallbackReason); void HandleReadDataResponse( const TEvService::TEvReadDataResponse::TPtr& ev, @@ -81,9 +83,11 @@ class TReadDataActor final: public TActorBootstrapped TReadDataActor::TReadDataActor( TRequestInfoPtr requestInfo, NProto::TReadDataRequest readRequest, + TString logTag, ui32 blockSize) : RequestInfo(std::move(requestInfo)) , ReadRequest(std::move(readRequest)) + , LogTag(std::move(logTag)) , BlockSize(blockSize) , OriginByteRange( ReadRequest.GetOffset(), @@ -140,10 +144,25 @@ TString DescribeResponseDebugString( //////////////////////////////////////////////////////////////////////////////// +char* GetDataPtr( + ui64 offset, + TByteRange alignedByteRange, + ui32 blockSize, + IBlockBuffer& buffer) +{ + + const ui64 relOffset = offset - alignedByteRange.Offset; + const ui32 blockNo = relOffset / blockSize; + const auto block = buffer.GetBlock(blockNo); + return const_cast(block.Data()) + relOffset - blockNo * blockSize; +} + +//////////////////////////////////////////////////////////////////////////////// + void ApplyFreshDataRange( const TActorContext& ctx, const NProtoPrivate::TFreshDataRange& sourceFreshData, - const IBlockBufferPtr& targetBuffer, + IBlockBuffer& targetBuffer, TByteRange alignedTargetByteRange, ui32 blockSize, ui64 offset, @@ -169,7 +188,6 @@ void ApplyFreshDataRange( length, DescribeResponseDebugString(describeResponse).c_str()); - char* targetData = const_cast(targetBuffer->GetContentRef().data()); auto commonRange = sourceByteRange.Intersect(alignedTargetByteRange); Y_ABORT_UNLESS(sourceByteRange == commonRange); @@ -182,9 +200,14 @@ void ApplyFreshDataRange( alignedTargetByteRange.Describe().c_str()); return; } - targetData += commonRange.Offset - alignedTargetByteRange.Offset; + char* targetData = GetDataPtr( + commonRange.Offset, + alignedTargetByteRange, + blockSize, + targetBuffer); // NB: we assume that underlying target data is a continuous buffer + // TODO: don't make such an assumption - use GetBlock(i) API memcpy( targetData, sourceFreshData.GetContent().data() + @@ -194,28 +217,6 @@ void ApplyFreshDataRange( //////////////////////////////////////////////////////////////////////////////// -// TODO(debnatkh): get rid of copy-paste -void CopyFileData( - const TByteRange origin, - const TByteRange aligned, - const ui64 fileSize, - TStringBuf content, - TString* out) -{ - auto end = Min(fileSize, origin.End()); - if (end < aligned.End()) { - ui64 delta = Min(aligned.End() - end, content.size()); - content.Chop(delta); - } - - Y_ABORT_UNLESS(origin.Offset >= aligned.Offset); - content.Skip(origin.Offset - aligned.Offset); - - out->assign(content.data(), content.size()); -} - -//////////////////////////////////////////////////////////////////////////////// - void TReadDataActor::HandleDescribeDataResponse( const TEvIndexTablet::TEvDescribeDataResponse::TPtr& ev, const TActorContext& ctx) @@ -223,7 +224,7 @@ void TReadDataActor::HandleDescribeDataResponse( const auto* msg = ev->Get(); if (FAILED(msg->GetStatus())) { - ReadData(ctx); + ReadData(ctx, FormatError(msg->GetError())); return; } @@ -307,14 +308,14 @@ void TReadDataActor::HandleReadBlobResponse( const auto* msg = ev->Get(); if (msg->Status != NKikimrProto::OK) { + const auto errorReason = FormatError( + MakeError(MAKE_KIKIMR_ERROR(msg->Status), msg->ErrorReason)); LOG_WARN( ctx, TFileStoreComponents::SERVICE, "ReadBlob error: %s", - FormatError(MakeError( - MAKE_KIKIMR_ERROR(msg->Status), - msg->ErrorReason)).c_str()); - ReadData(ctx); + errorReason.c_str()); + ReadData(ctx, errorReason); return; } @@ -337,14 +338,14 @@ void TReadDataActor::HandleReadBlobResponse( const auto& blobRange = blobPiece.GetRanges(i); const auto& response = msg->Responses[i]; if (response.Status != NKikimrProto::OK) { + const auto errorReason = FormatError( + MakeError(MAKE_KIKIMR_ERROR(response.Status), "read error")); LOG_WARN( ctx, TFileStoreComponents::SERVICE, "ReadBlob error: %s", - FormatError(MakeError( - MAKE_KIKIMR_ERROR(response.Status), - "read error")).c_str()); - ReadData(ctx); + errorReason.c_str()); + ReadData(ctx, errorReason); return; } @@ -355,14 +356,14 @@ void TReadDataActor::HandleReadBlobResponse( response.Buffer.empty() || response.Buffer.size() % BlockSize != 0) { + const auto error = + FormatError(MakeError(E_FAIL, "invalid response received")); LOG_WARN( ctx, TFileStoreComponents::SERVICE, "ReadBlob error: %s", - FormatError(MakeError( - E_FAIL, - "invalid response received")).c_str()); - ReadData(ctx); + error.c_str()); + ReadData(ctx, error); return; } @@ -387,14 +388,11 @@ void TReadDataActor::HandleReadBlobResponse( response.Buffer.size()); Y_ABORT_UNLESS(blobRange.GetOffset() >= AlignedByteRange.Offset); - const TByteRange sourceByteRange( + char* targetData = GetDataPtr( blobRange.GetOffset(), - blobRange.GetLength(), - BlockSize); - - char* targetData = - const_cast(BlockBuffer->GetContentRef().data()) + - (blobRange.GetOffset() - AlignedByteRange.Offset); + AlignedByteRange, + BlockSize, + *BlockBuffer); dataIter.ExtractPlainDataAndAdvance(targetData, blobRange.GetLength()); } @@ -417,18 +415,21 @@ void TReadDataActor::HandlePoisonPill( //////////////////////////////////////////////////////////////////////////////// -void TReadDataActor::ReadData(const TActorContext& ctx) +void TReadDataActor::ReadData( + const TActorContext& ctx, + const TString& fallbackReason) { ReadDataFallbackEnabled = true; LOG_WARN( ctx, TFileStoreComponents::SERVICE, - "Falling back to ReadData for %lu, %lu, %lu, %lu", + "Falling back to ReadData for %lu, %lu, %lu, %lu. Message: %s", ReadRequest.GetNodeId(), ReadRequest.GetHandle(), ReadRequest.GetOffset(), - ReadRequest.GetLength()); + ReadRequest.GetLength(), + fallbackReason.Quote().c_str()); auto request = std::make_unique(); request->Record = std::move(ReadRequest); @@ -475,7 +476,7 @@ void TReadDataActor::ReplyAndDie(const TActorContext& ctx) ApplyFreshDataRange( ctx, freshDataRange, - BlockBuffer, + *BlockBuffer, AlignedByteRange, BlockSize, ReadRequest.GetOffset(), @@ -491,10 +492,11 @@ void TReadDataActor::ReplyAndDie(const TActorContext& ctx) } CopyFileData( + LogTag, OriginByteRange, AlignedByteRange, DescribeResponse.GetFileSize(), - BlockBuffer->GetContent(), + *BlockBuffer, response->Record.MutableBuffer()); NCloud::Reply(ctx, *RequestInfo, std::move(response)); @@ -570,8 +572,8 @@ void TStorageServiceActor::HandleReadData( auto [cookie, inflight] = CreateInFlightRequest( TRequestInfo(ev->Sender, ev->Cookie, msg->CallContext), - NProto::EStorageMediaKind::STORAGE_MEDIA_DEFAULT, - StatsRegistry->GetRequestStats(), + session->MediaKind, + session->RequestStats, ctx.Now()); InitProfileLogRequestInfo(inflight->ProfileLogRequest, msg->Record); @@ -581,6 +583,7 @@ void TStorageServiceActor::HandleReadData( auto actor = std::make_unique( std::move(requestInfo), std::move(msg->Record), + filestore.GetFileSystemId(), filestore.GetBlockSize()); NCloud::Register(ctx, std::move(actor)); diff --git a/cloud/filestore/libs/storage/service/service_ut.cpp b/cloud/filestore/libs/storage/service/service_ut.cpp index 66bfd6fc26d..024aa1a8a04 100644 --- a/cloud/filestore/libs/storage/service/service_ut.cpp +++ b/cloud/filestore/libs/storage/service/service_ut.cpp @@ -1195,16 +1195,16 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) UNIT_ASSERT(google::protobuf::util::JsonStringToMessage( jsonResponse->Record.GetOutput(), &response).ok()); - auto storageValues = response.GetStorageConfigFieldsToValues(); + const auto& storageValues = response.GetStorageConfigFieldsToValues(); UNIT_ASSERT_VALUES_EQUAL( - storageValues["SSDBoostTime"], + storageValues.at("SSDBoostTime"), "Default"); UNIT_ASSERT_VALUES_EQUAL( - storageValues["Unknown"], + storageValues.at("Unknown"), "Not found"); UNIT_ASSERT_VALUES_EQUAL( - storageValues["CompactionThreshold"], + storageValues.at("CompactionThreshold"), "1000"); } @@ -1338,6 +1338,60 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) service); } + Y_UNIT_TEST(ShouldDescribeSessions) + { + NProto::TStorageConfig config; + config.SetCompactionThreshold(1000); + TTestEnv env({}, config); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + TServiceClient service(env.GetRuntime(), nodeIdx); + service.CreateFileStore("test", 1'000); + + THeaders headers = {"test", "client", "session", 3}; + service.CreateSession( + headers, + "", // checkpointId + false, // restoreClientSession + headers.SessionSeqNo); + service.ResetSession(headers, "some_state"); + + headers = {"test", "client2", "session2", 4}; + service.CreateSession( + headers, + "", // checkpointId + false, // restoreClientSession + headers.SessionSeqNo); + service.ResetSession(headers, "some_state2"); + + NProtoPrivate::TDescribeSessionsRequest request; + request.SetFileSystemId("test"); + + TString buf; + google::protobuf::util::MessageToJsonString(request, &buf); + auto jsonResponse = service.ExecuteAction("describesessions", buf); + NProtoPrivate::TDescribeSessionsResponse response; + UNIT_ASSERT(google::protobuf::util::JsonStringToMessage( + jsonResponse->Record.GetOutput(), &response).ok()); + + const auto& sessions = response.GetSessions(); + UNIT_ASSERT_VALUES_EQUAL(2, sessions.size()); + + UNIT_ASSERT_VALUES_EQUAL("session", sessions[0].GetSessionId()); + UNIT_ASSERT_VALUES_EQUAL("client", sessions[0].GetClientId()); + UNIT_ASSERT_VALUES_EQUAL("some_state", sessions[0].GetSessionState()); + UNIT_ASSERT_VALUES_EQUAL(3, sessions[0].GetMaxSeqNo()); + UNIT_ASSERT_VALUES_EQUAL(3, sessions[0].GetMaxRwSeqNo()); + + UNIT_ASSERT_VALUES_EQUAL("session2", sessions[1].GetSessionId()); + UNIT_ASSERT_VALUES_EQUAL("client2", sessions[1].GetClientId()); + UNIT_ASSERT_VALUES_EQUAL("some_state2", sessions[1].GetSessionState()); + UNIT_ASSERT_VALUES_EQUAL(4, sessions[1].GetMaxSeqNo()); + UNIT_ASSERT_VALUES_EQUAL(4, sessions[1].GetMaxRwSeqNo()); + } + Y_UNIT_TEST(ShouldValidateBlockSize) { TTestEnv env; @@ -1545,8 +1599,9 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) service.ReadData(headers, fs, nodeId, handle, 0, data.Size()); UNIT_ASSERT_VALUES_EQUAL(readDataResult->Record.GetBuffer(), data); - // fresh blocks - data = TString(4_KB, 'a'); + // fresh blocks - adding multiple adjacent blocks is important here to + // catch some subtle bugs + data = TString(8_KB, 'a'); service.WriteData(headers, fs, nodeId, handle, 0, data); readDataResult = service.ReadData(headers, fs, nodeId, handle, 0, data.Size()); @@ -1567,6 +1622,16 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) service.ReadData(headers, fs, nodeId, handle, 0, data.Size()); memcpy(data.begin() + patchOffset, patch.Data(), patch.Size()); UNIT_ASSERT_VALUES_EQUAL(readDataResult->Record.GetBuffer(), data); + + auto counters = env.GetCounters() + ->FindSubgroup("component", "service_fs") + ->FindSubgroup("host", "cluster") + ->FindSubgroup("filesystem", fs) + ->FindSubgroup("client", "client") + ->FindSubgroup("request", "ReadData"); + UNIT_ASSERT(counters); + + UNIT_ASSERT_EQUAL(4, counters->GetCounter("Count")->GetAtomic()); } Y_UNIT_TEST(ShouldFallbackToReadDataIfDescribeDataFails) diff --git a/cloud/filestore/libs/storage/service/ya.make b/cloud/filestore/libs/storage/service/ya.make index 59428aa7e3e..fbca807c47e 100644 --- a/cloud/filestore/libs/storage/service/ya.make +++ b/cloud/filestore/libs/storage/service/ya.make @@ -5,6 +5,7 @@ SRCS( service.cpp service_actor.cpp service_actor_actions_change_storage_config.cpp + service_actor_actions_describe_sessions.cpp service_actor_actions_drain_tablets.cpp service_actor_actions_get_storage_config_fields.cpp service_actor_actions.cpp diff --git a/cloud/filestore/libs/storage/tablet/session.h b/cloud/filestore/libs/storage/tablet/session.h index b6ccea4a237..62a60877634 100644 --- a/cloud/filestore/libs/storage/tablet/session.h +++ b/cloud/filestore/libs/storage/tablet/session.h @@ -271,4 +271,10 @@ using TSessionOwnerMap = THashMap; using TSessionClientMap = THashMap; using TSessionHistoryList = TDeque; +struct TSessionsStats +{ + ui32 StatefulSessionsCount = 0; + ui32 StatelessSessionsCount = 0; +}; + } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp index b1bc1e83243..910aa085732 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp @@ -300,6 +300,8 @@ void TIndexTabletActor::HandleSessionDisconnected( OrphanSession(ev->Sender, ctx.Now()); } +//////////////////////////////////////////////////////////////////////////////// + void TIndexTabletActor::HandleGetFileSystemConfig( const TEvIndexTablet::TEvGetFileSystemConfigRequest::TPtr& ev, const TActorContext& ctx) @@ -339,6 +341,23 @@ void TIndexTabletActor::HandleGetStorageConfigFields( NCloud::Reply(ctx, *ev, std::move(response)); } +void TIndexTabletActor::HandleDescribeSessions( + const TEvIndexTablet::TEvDescribeSessionsRequest::TPtr& ev, + const TActorContext& ctx) +{ + auto response = + std::make_unique(); + + auto sessionInfos = DescribeSessions(); + for (auto& si: sessionInfos) { + *response->Record.AddSessions() = std::move(si); + } + + NCloud::Reply(ctx, *ev, std::move(response)); +} + +//////////////////////////////////////////////////////////////////////////////// + bool TIndexTabletActor::HandleRequests(STFUNC_SIG) { switch (ev->GetTypeRewrite()) { diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index 1578554a573..403662ecf22 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -82,6 +82,9 @@ class TIndexTabletActor final std::atomic UsedSessionsCount{0}; std::atomic UsedHandlesCount{0}; std::atomic UsedLocksCount{0}; + std::atomic StatefulSessionsCount{0}; + std::atomic StatelessSessionsCount{0}; + std::atomic SessionTimeouts{0}; std::atomic AllocatedCompactionRangesCount{0}; std::atomic UsedCompactionRangesCount{0}; @@ -146,7 +149,8 @@ class TIndexTabletActor final const NProto::TFileSystem& fileSystem, const NProto::TFileSystemStats& stats, const NProto::TFileStorePerformanceProfile& performanceProfile, - const TCompactionMapStats& compactionStats); + const TCompactionMapStats& compactionStats, + const TSessionsStats& sessionsStats); } Metrics; const IProfileLogPtr ProfileLog; diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp index d9ff523a1fb..09ec0c053ac 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_cleanupsessions.cpp @@ -185,6 +185,10 @@ void TIndexTabletActor::HandleCleanupSessions( return; } + Metrics.SessionTimeouts.fetch_add( + sessions.size(), + std::memory_order_relaxed); + TVector list(sessions.size()); for (size_t i = 0; i < sessions.size(); ++i) { list[i].CopyFrom(*sessions[i]); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 77d073dccf8..31eb8ca68fe 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -92,9 +92,9 @@ void TIndexTabletActor::TMetrics::Register( REGISTER_AGGREGATABLE_SUM(UsedSessionsCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(UsedHandlesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(UsedLocksCount, EMetricType::MT_ABSOLUTE); - - REGISTER_AGGREGATABLE_SUM(AllocatedCompactionRangesCount, EMetricType::MT_ABSOLUTE); - REGISTER_AGGREGATABLE_SUM(UsedCompactionRangesCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(StatefulSessionsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(StatelessSessionsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM(SessionTimeouts, EMetricType::MT_DERIVATIVE); REGISTER_AGGREGATABLE_SUM(FreshBytesCount, EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM(MixedBytesCount, EMetricType::MT_ABSOLUTE); @@ -217,7 +217,8 @@ void TIndexTabletActor::TMetrics::Update( const NProto::TFileSystem& fileSystem, const NProto::TFileSystemStats& stats, const NProto::TFileStorePerformanceProfile& performanceProfile, - const TCompactionMapStats& compactionStats) + const TCompactionMapStats& compactionStats, + const TSessionsStats& sessionsStats) { const ui32 blockSize = fileSystem.GetBlockSize(); @@ -264,6 +265,9 @@ void TIndexTabletActor::TMetrics::Update( .Stats.DeletionsCount); } + Store(StatefulSessionsCount, sessionsStats.StatefulSessionsCount); + Store(StatelessSessionsCount, sessionsStats.StatelessSessionsCount); + BusyIdleCalc.OnUpdateStats(); } @@ -300,14 +304,15 @@ void TIndexTabletActor::RegisterStatCounters() TABLET_VERIFY(!storageMediaKind.empty()); // Update should be called before Register, because we want to write - // correct values to solomon. If we reorder this two actions, we can + // correct values to solomon. If we reorder these two actions, we can // aggregate zero values, in the middle of the registration (or right after // registration, before update). Metrics.Update( fs, GetFileSystemStats(), GetPerformanceProfile(), - GetCompactionMapStats(1)); + GetCompactionMapStats(1), + CalculateSessionsStats()); Metrics.Register(fsId, storageMediaKind); } @@ -338,7 +343,8 @@ void TIndexTabletActor::HandleUpdateCounters( GetFileSystem(), GetFileSystemStats(), GetPerformanceProfile(), - GetCompactionMapStats(1)); + GetCompactionMapStats(1), + CalculateSessionsStats()); UpdateCountersScheduled = false; ScheduleUpdateCounters(ctx); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp index 03cf7bae37b..777bf8b5e20 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_createsession.cpp @@ -201,12 +201,12 @@ void TIndexTabletActor::ExecuteTx_CreateSession( ctx); return; - } else { - LOG_INFO(ctx, TFileStoreComponents::TABLET, - "%s CreateSession: no session available for client c: %s", - LogTag.c_str(), - clientId.c_str()); } + + LOG_INFO(ctx, TFileStoreComponents::TABLET, + "%s CreateSession: no session available for client c: %s", + LogTag.c_str(), + clientId.c_str()); } if (!sessionId) { diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp index df08bc038ce..a8b7f5a6b97 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_readdata.cpp @@ -36,28 +36,6 @@ bool ShouldReadBlobs(const TVector& blocks) //////////////////////////////////////////////////////////////////////////////// -void CopyFileData( - const TString& LogTag, - const TByteRange origin, - const TByteRange aligned, - const ui64 fileSize, - TStringBuf content, - TString* out) -{ - auto end = Min(fileSize, origin.End()); - if (end < aligned.End()) { - ui64 delta = Min(aligned.End() - end, content.size()); - content.Chop(delta); - } - - TABLET_VERIFY(origin.Offset >= aligned.Offset); - content.Skip(origin.Offset - aligned.Offset); - - out->assign(content.data(), content.size()); -} - -//////////////////////////////////////////////////////////////////////////////// - void ApplyBytes(const TBlockBytes& bytes, TStringBuf blockData) { for (const auto& interval: bytes.Intervals) { @@ -479,7 +457,7 @@ void TReadDataActor::ReplyAndDie( OriginByteRange, AlignedByteRange, TotalSize, - Buffer->GetContentRef(), + *Buffer, response->Record.MutableBuffer()); } @@ -624,9 +602,7 @@ void TIndexTabletActor::HandleDescribeData( requestInfo->StartedTs = ctx.Now(); TByteRange alignedByteRange = byteRange.AlignedSuperRange(); - // TODO: implement a block buffer with lazy block allocation and use it - // here - auto blockBuffer = CreateBlockBuffer(alignedByteRange); + auto blockBuffer = CreateLazyBlockBuffer(alignedByteRange); ExecuteTx( ctx, @@ -830,7 +806,7 @@ void TIndexTabletActor::CompleteTx_ReadData( args.OriginByteRange, args.AlignedByteRange, args.Node->Attrs.GetSize(), - args.Buffer->GetContentRef(), + *args.Buffer, response->Record.MutableBuffer()); CompleteResponse( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp index 97009484e74..4813d94c535 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp @@ -113,6 +113,7 @@ template void TIndexTabletActor::CompleteResponse( FILESTORE_SERVICE(FILESTORE_GENERATE_IMPL, TEvService) FILESTORE_GENERATE_IMPL(DescribeData, TEvIndexTablet) +FILESTORE_GENERATE_IMPL(DescribeSessions, TEvIndexTablet) #undef FILESTORE_GENERATE_IMPL diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp index 94b8b44cac4..6ffe354d499 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_writedata.cpp @@ -316,7 +316,8 @@ void TIndexTabletActor::HandleWriteData( } if (!IsWriteAllowed(BuildBackpressureThresholds())) { - if (++BackpressureErrorCount >= + if (CompactionStateLoadStatus.Finished + && ++BackpressureErrorCount >= Config->GetMaxBackpressureErrorsBeforeSuicide()) { LOG_WARN(ctx, TFileStoreComponents::TABLET_WORKER, diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h index 179d6568834..05680f941f9 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state.h @@ -440,6 +440,7 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_COUNTER) TVector GetTimeoutedSessions(TInstant now) const; TVector GetSessionsToNotify(const NProto::TSessionEvent& event) const; + TVector DescribeSessions() const; const TSessionHistoryList& GetSessionHistoryList() const; void AddSessionHistoryEntry( @@ -447,6 +448,7 @@ FILESTORE_FILESYSTEM_STATS(FILESTORE_DECLARE_COUNTER) const TSessionHistoryEntry& entry, size_t maxEntryCount); TVector GetActiveSessions() const; + TSessionsStats CalculateSessionsStats() const; private: TSession* CreateSession( diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_sessions.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_sessions.cpp index d552800279a..94dee193441 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_sessions.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_sessions.cpp @@ -1,5 +1,7 @@ #include "tablet_state_impl.h" +#include + #include #include @@ -377,6 +379,21 @@ TVector TIndexTabletState::GetSessionsToNotify( return result; } +TVector TIndexTabletState::DescribeSessions() const +{ + TVector sessionInfos; + for (const auto& session: Impl->Sessions) { + NProtoPrivate::TTabletSessionInfo sessionInfo; + sessionInfo.SetSessionId(session.GetSessionId()); + sessionInfo.SetClientId(session.GetClientId()); + sessionInfo.SetSessionState(session.GetSessionState()); + sessionInfo.SetMaxSeqNo(session.GetMaxSeqNo()); + sessionInfo.SetMaxRwSeqNo(session.GetMaxRwSeqNo()); + sessionInfos.push_back(std::move(sessionInfo)); + } + return sessionInfos; +} + const TSessionHistoryList& TIndexTabletState::GetSessionHistoryList() const { return Impl->SessionHistoryList; @@ -412,6 +429,24 @@ TVector TIndexTabletState::GetActiveSessions() const return sessions; } +TSessionsStats TIndexTabletState::CalculateSessionsStats() const +{ + TSessionsStats stats; + + // recalculating these stats on purpose to be able to perform basic + // validation of the counters (UsedSessionsCount should be equal to the sum + // of Stateless and Stateful SessionsCount) + for (const auto& s: Impl->Sessions) { + if (s.GetSessionState()) { + ++stats.StatefulSessionsCount; + } else { + ++stats.StatelessSessionsCount; + } + } + + return stats; +} + //////////////////////////////////////////////////////////////////////////////// // Handles diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp index 15699960706..55583aeaab5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_counters.cpp @@ -378,7 +378,10 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Counters) registry->Visit(TInstant::Zero(), Visitor); Visitor.ValidateExpectedCounters({ - {{{"sensor", "UsedSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "UsedSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "StatefulSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "StatelessSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "SessionTimeouts"}, {"filesystem", "test"}}, 0}, }); Tablet->InitSession("client", "session"); @@ -389,9 +392,27 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Counters) registry->Visit(TInstant::Zero(), Visitor); Visitor.ValidateExpectedCounters({ {{{"sensor", "UsedSessionsCount"}, {"filesystem", "test"}}, 1}, + {{{"sensor", "StatefulSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "StatelessSessionsCount"}, {"filesystem", "test"}}, 1}, + {{{"sensor", "SessionTimeouts"}, {"filesystem", "test"}}, 0}, + }); + + Tablet->ResetSession("client", "session", 0, "state"); + + Tablet->AdvanceTime(TDuration::Seconds(15)); + Env.GetRuntime().DispatchEvents({}, TDuration::Seconds(5)); + + registry->Visit(TInstant::Zero(), Visitor); + Visitor.ValidateExpectedCounters({ + {{{"sensor", "UsedSessionsCount"}, {"filesystem", "test"}}, 1}, + {{{"sensor", "StatefulSessionsCount"}, {"filesystem", "test"}}, 1}, + {{{"sensor", "StatelessSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "SessionTimeouts"}, {"filesystem", "test"}}, 0}, }); - Tablet->DestroySession(); + Tablet->RebootTablet(); + Tablet->AdvanceTime(TDuration::Minutes(1)); + Tablet->CleanupSessions(); Tablet->AdvanceTime(TDuration::Seconds(15)); Env.GetRuntime().DispatchEvents({}, TDuration::Seconds(5)); @@ -399,6 +420,9 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Counters) registry->Visit(TInstant::Zero(), Visitor); Visitor.ValidateExpectedCounters({ {{{"sensor", "UsedSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "StatefulSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "StatelessSessionsCount"}, {"filesystem", "test"}}, 0}, + {{{"sensor", "SessionTimeouts"}, {"filesystem", "test"}}, 1}, }); } diff --git a/cloud/filestore/libs/storage/testlib/tablet_client.h b/cloud/filestore/libs/storage/testlib/tablet_client.h index bac5f2eb3d8..9c60fb51f5c 100644 --- a/cloud/filestore/libs/storage/testlib/tablet_client.h +++ b/cloud/filestore/libs/storage/testlib/tablet_client.h @@ -358,6 +358,11 @@ class TIndexTabletClient rangeId); } + auto CreateCleanupSessionsRequest() + { + return std::make_unique(); + } + // // TEvService // diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index 749c3745ac2..35d8efc8960 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -337,3 +337,34 @@ message TDescribeDataResponse // Optional response headers. NProto.TResponseHeaders Headers = 1000; } + +//////////////////////////////////////////////////////////////////////////////// +// DescribeSessions request/response. + +message TDescribeSessionsRequest +{ + // Optional request headers. + NProto.THeaders Headers = 1; + + // FileSystem identifier. + string FileSystemId = 2; +} + +message TTabletSessionInfo +{ + // A subset of the params stored in TSession in the IndexTablet's DB. + string SessionId = 1; + string ClientId = 2; + bytes SessionState = 3; + uint64 MaxSeqNo = 4; + uint64 MaxRwSeqNo = 5; +} + +message TDescribeSessionsResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; + + // All tablet sessions. + repeated TTabletSessionInfo Sessions = 2; +} diff --git a/cloud/filestore/tests/client/canondata/result.json b/cloud/filestore/tests/client/canondata/result.json index 43e9549ffdf..35d31afeae7 100644 --- a/cloud/filestore/tests/client/canondata/result.json +++ b/cloud/filestore/tests/client/canondata/result.json @@ -8,6 +8,9 @@ "test.test_create_mkdir_ls_write_destroy": { "uri": "file://test.test_create_mkdir_ls_write_destroy/results.txt" }, + "test.test_describe_sessions": { + "uri": "file://test.test_describe_sessions/results.txt" + }, "test.test_list_filestores": { "uri": "file://test.test_list_filestores/results.txt" } diff --git a/cloud/filestore/tests/client/canondata/test.test_describe_sessions/results.txt b/cloud/filestore/tests/client/canondata/test.test_describe_sessions/results.txt new file mode 100644 index 00000000000..1085cbbbceb --- /dev/null +++ b/cloud/filestore/tests/client/canondata/test.test_describe_sessions/results.txt @@ -0,0 +1,14 @@ +{ + "Sessions": [ + { + "SessionId": "session0", + "ClientId": "client0", + "SessionState": "c29tZSBzZXNzaW9uIHN0YXRl" + }, + { + "SessionId": "session1", + "ClientId": "client1", + "SessionState": "YW5vdGhlciBzZXNzaW9uIHN0YXRl" + } + ] +} \ No newline at end of file diff --git a/cloud/filestore/tests/client/test.py b/cloud/filestore/tests/client/test.py index 4e3751ad42b..26246caf44f 100644 --- a/cloud/filestore/tests/client/test.py +++ b/cloud/filestore/tests/client/test.py @@ -98,3 +98,32 @@ def test_list_filestores(): ret = common.canonical_file(results_path, local=True) return ret + + +def test_describe_sessions(): + client, results_path = __init_test() + + client.create("fs0", "test_cloud", "test_folder", BLOCK_SIZE, BLOCKS_COUNT) + + # creating a bunch of sessions + client.create_session("fs0", "session0", "client0") + client.create_session("fs0", "session1", "client1") + client.reset_session( + "fs0", + "session0", + "client0", + "some session state".encode("utf-8")) + client.reset_session( + "fs0", + "session1", + "client1", + "another session state".encode("utf-8")) + + out = client.execute_action("describesessions", {"FileSystemId": "fs0"}) + sessions = json.loads(out) + + with open(results_path, "w") as results_file: + json.dump(sessions, results_file, indent=4) + + ret = common.canonical_file(results_path, local=True) + return ret diff --git a/cloud/filestore/tests/python/lib/client.py b/cloud/filestore/tests/python/lib/client.py index 4a753415908..515f23eee74 100644 --- a/cloud/filestore/tests/python/lib/client.py +++ b/cloud/filestore/tests/python/lib/client.py @@ -1,6 +1,9 @@ +import base64 +import json import logging import os import signal +import tempfile import uuid import yatest.common as common @@ -23,7 +26,7 @@ def __init__(self, binary_path, port, vhost_port=None, verbose=False, cwd=".", t self.__cwd = cwd self.__timeout = timeout - def create(self, fs, cloud, folder, blk_size=4096, blk_count=100*1024*1024*1024): + def create(self, fs, cloud, folder, blk_size=4096, blk_count=100 * 1024 * 1024 * 1024): cmd = [ self.__binary_path, "create", "--filesystem", fs, @@ -123,6 +126,42 @@ def kick_endpoint(self, keyring_id): return common.execute(cmd) + def create_session(self, fs, session_id, client_id): + cmd = [ + self.__binary_path, "createsession", + "--filesystem", fs, + "--session-id", session_id, + "--client-id", client_id, + ] + self.__cmd_opts() + + return common.execute(cmd).stdout + + def reset_session(self, fs, session_id, client_id, session_state): + cmd = [ + self.__binary_path, "resetsession", + "--filesystem", fs, + "--session-id", session_id, + "--client-id", client_id, + "--session-state", base64.b64encode(session_state).decode("utf-8"), + ] + self.__cmd_opts() + + return common.execute(cmd).stdout + + def execute_action(self, action, request): + request_file = tempfile.NamedTemporaryFile(mode="w", delete=False) + json.dump(request, request_file) + request_file.close() + cmd = [ + self.__binary_path, "executeaction", + "--action", action, + "--input-file", request_file.name, + ] + self.__cmd_opts() + print(cmd) + + res = common.execute(cmd) + os.unlink(request_file.name) + return res.stdout + def __cmd_opts(self, vhost=False): opts = [ "--server-address", "localhost", diff --git a/cloud/storage/core/libs/common/history.cpp b/cloud/storage/core/libs/common/history.cpp new file mode 100644 index 00000000000..54d0e7587fd --- /dev/null +++ b/cloud/storage/core/libs/common/history.cpp @@ -0,0 +1 @@ +#include "history.h" diff --git a/cloud/storage/core/libs/common/history.h b/cloud/storage/core/libs/common/history.h new file mode 100644 index 00000000000..3b5e86ed601 --- /dev/null +++ b/cloud/storage/core/libs/common/history.h @@ -0,0 +1,38 @@ +#pragma once + +#include "ring_buffer.h" + +namespace NCloud { + +//////////////////////////////////////////////////////////////////////////////// + +template +class THistory +{ +private: + TRingBuffer Ring; + THashMap References; + +public: + THistory(size_t capacity) + : Ring(capacity) + {} + + void Put(T item) + { + References[item]++; + if (auto oldest = Ring.PushBack(std::move(item))) { + auto it = References.find(*oldest); + if (--it->second == 0) { + References.erase(it); + } + } + } + + bool Contains(const T& item) const + { + return References.find(item) != References.end(); + } +}; + +} // namespace NCloud diff --git a/cloud/storage/core/libs/common/history_ut.cpp b/cloud/storage/core/libs/common/history_ut.cpp new file mode 100644 index 00000000000..bdb371dfbae --- /dev/null +++ b/cloud/storage/core/libs/common/history_ut.cpp @@ -0,0 +1,51 @@ +#include "history.h" + +#include + +namespace NCloud { + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(THistoryTest) +{ + Y_UNIT_TEST(ShouldReplaceItems) + { + { + constexpr int count = 10; + auto history = THistory(count); + + for (int i = 0; i < count; i++) { + history.Put(i); + UNIT_ASSERT_VALUES_EQUAL(history.Contains(i), true); + } + for (int i = count; i < count + count; i++) { + history.Put(i); + UNIT_ASSERT_VALUES_EQUAL(history.Contains(i), true); + UNIT_ASSERT_VALUES_EQUAL(history.Contains(i - count), false); + } + } + } + + Y_UNIT_TEST(ShouldHandleDuplicates) + { + { + auto history = THistory(2); + + history.Put(0); + history.Put(0); + + history.Put(0); // replaces 0 with 0 + UNIT_ASSERT_VALUES_EQUAL(history.Contains(0), true); + + history.Put(1); // replaces 0 with 1 + UNIT_ASSERT_VALUES_EQUAL(history.Contains(0), true); + UNIT_ASSERT_VALUES_EQUAL(history.Contains(1), true); + + history.Put(1); // replaces last 0 with 1 + UNIT_ASSERT_VALUES_EQUAL(history.Contains(0), false); + UNIT_ASSERT_VALUES_EQUAL(history.Contains(1), true); + } + } +} + +} // namespace NCloud diff --git a/cloud/storage/core/libs/common/ut/ya.make b/cloud/storage/core/libs/common/ut/ya.make index 92721dc07a8..381b320fbcc 100644 --- a/cloud/storage/core/libs/common/ut/ya.make +++ b/cloud/storage/core/libs/common/ut/ya.make @@ -23,6 +23,7 @@ SRCS( error_ut.cpp file_io_service_ut.cpp guarded_sglist_ut.cpp + history_ut.cpp ring_buffer_ut.cpp scheduler_ut.cpp scoped_handle_ut.cpp diff --git a/cloud/storage/core/libs/common/ya.make b/cloud/storage/core/libs/common/ya.make index e97a641188a..cb12f48f918 100644 --- a/cloud/storage/core/libs/common/ya.make +++ b/cloud/storage/core/libs/common/ya.make @@ -17,6 +17,7 @@ SRCS( format.cpp guarded_sglist.cpp helpers.cpp + history.cpp media.cpp proto_helpers.cpp random.cpp diff --git a/cloud/storage/core/tools/ci/runner/disk_manager_acceptance_common.sh b/cloud/storage/core/tools/ci/runner/disk_manager_acceptance_common.sh index a6a438acc3a..d45f86062be 100644 --- a/cloud/storage/core/tools/ci/runner/disk_manager_acceptance_common.sh +++ b/cloud/storage/core/tools/ci/runner/disk_manager_acceptance_common.sh @@ -25,6 +25,7 @@ function base_shell_args () { "--instance-ram" "${instance_ram:?Variable instance_ram not present}" "--chunk-storage-type" "${chunk_storage_type:=ydb}" "--instance-platform-ids" "standard-v2" "standard-v3" + "--cleanup-before-tests" "--acceptance-test" "$dm/acceptance-test" "--results-path" "${results_path}" "--verbose" diff --git a/cloud/storage/core/tools/ci/runner/run_all.sh b/cloud/storage/core/tools/ci/runner/run_all.sh index 56435f51a82..c00c0770e65 100755 --- a/cloud/storage/core/tools/ci/runner/run_all.sh +++ b/cloud/storage/core/tools/ci/runner/run_all.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash + + logs_root="/var/www/build/logs" +logs_dir="${logs_root}/run_$(date +%y_%m_%d__%H)" && +rm -rf "$logs_dir" && +mkdir -p "$logs_dir" + exec 3>&1 4>&2 trap 'exec 2>&4 1>&3' 0 1 2 3 -exec 1>"${logs_root}/run_all.out" 2>&1 +exec 1>"${logs_dir}/run_all.out" 2>&1 d="/root" scripts="${d}/runner" @@ -25,9 +31,6 @@ git pull "${nbspath}/ya" gc cache -logs_dir="${logs_root}/run_$(date +%y_%m_%d__%H)" && -rm -rf "$logs_dir" && -mkdir -p "$logs_dir" find "${logs_root}" -maxdepth 1 -mtime +7 -type d -exec rm -rf {} \; lineArr=() diff --git a/cloud/tasks/test/nemesis/main.go b/cloud/tasks/test/nemesis/main.go index 348ecb24715..87c512a87a6 100644 --- a/cloud/tasks/test/nemesis/main.go +++ b/cloud/tasks/test/nemesis/main.go @@ -49,6 +49,7 @@ func waitIteration( errors chan error, minRestartPeriodSec uint32, maxRestartPeriodSec uint32, + restartTimingsFile string, ) error { restartPeriod := common.RandomDuration( @@ -60,6 +61,24 @@ func waitIteration( case <-time.After(restartPeriod): logging.Info(ctx, "Cancel iteration") cancel() + + if len(restartTimingsFile) != 0 { + f, err := os.OpenFile( + restartTimingsFile, + os.O_APPEND|os.O_WRONLY|os.O_CREATE, + 0644, + ) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(time.Now().String()) + if err != nil { + return err + } + } + // Wait for iteration to stop. <-errors return nil @@ -73,6 +92,7 @@ func run( cmdString string, minRestartPeriodSec uint32, maxRestartPeriodSec uint32, + restartTimingsFile string, ) error { cmdString = strings.TrimSpace(cmdString) @@ -99,6 +119,7 @@ func run( errors, minRestartPeriodSec, maxRestartPeriodSec, + restartTimingsFile, ) if err != nil { return err @@ -112,10 +133,16 @@ func main() { var cmdString string var minRestartPeriodSec uint32 var maxRestartPeriodSec uint32 + var restartTimingsFile string rootCmd := &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { - return run(cmdString, minRestartPeriodSec, maxRestartPeriodSec) + return run( + cmdString, + minRestartPeriodSec, + maxRestartPeriodSec, + restartTimingsFile, + ) }, } @@ -138,6 +165,13 @@ func main() { "maximum time (in seconds) between two consecutive restarts", ) + rootCmd.Flags().StringVar( + &restartTimingsFile, + "restart-timings-file", + "", + "file where to store restart timings", + ) + if err := rootCmd.Execute(); err != nil { log.Fatalf("Failed to execute: %v", err) } diff --git a/go.mod b/go.mod index 84a0520ba2f..26de7fba143 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/ydb-platform/nbs -go 1.21.6 +go 1.21.3 diff --git a/vscode_generate_workspace.sh b/vscode_generate_workspace.sh new file mode 100755 index 00000000000..8a46f159d0e --- /dev/null +++ b/vscode_generate_workspace.sh @@ -0,0 +1,11 @@ +./ya ide vscode-clangd --allow-project-inside-arc --setup-tidy -P=. \ + build \ + cloud/blockstore \ + cloud/filestore \ + cloud/storage \ + cloud/vm \ + cloud/contrib \ + cloud/disk_manager \ + contrib/ydb/library/actors \ + example \ + library \