From c94c81076bd768002500bc23e82b701d615e99bd Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:25:05 +0000 Subject: [PATCH 01/17] temporary fix for SA tablet local db schema (#7459) --- .github/config/muted_ya.txt | 1 + .../run/kikimr_services_initializers.cpp | 2 +- ydb/core/driver_lib/run/ya.make | 2 +- ydb/core/kqp/gateway/kqp_metadata_loader.cpp | 2 +- ydb/core/kqp/gateway/ya.make | 1 + .../counters_statistics_aggregator.proto | 10 +- ydb/core/protos/statistics.proto | 124 ++- .../statistics/aggregator/aggregator_impl.cpp | 291 ++++--- .../statistics/aggregator/aggregator_impl.h | 156 ++-- ydb/core/statistics/aggregator/schema.h | 50 +- .../statistics/aggregator/tx_ack_timeout.cpp | 32 + .../aggregator/tx_aggr_stat_response.cpp | 139 ++++ .../aggregator/tx_analyze_table.cpp | 85 ++ ...nse.cpp => tx_datashard_scan_response.cpp} | 26 +- .../aggregator/tx_delete_query_response.cpp | 39 - .../aggregator/tx_finish_trasersal.cpp | 63 ++ ydb/core/statistics/aggregator/tx_init.cpp | 186 +++-- .../statistics/aggregator/tx_init_schema.cpp | 4 +- .../statistics/aggregator/tx_navigate.cpp | 15 +- ydb/core/statistics/aggregator/tx_resolve.cpp | 44 +- .../tx_response_tablet_distribution.cpp | 109 +++ .../aggregator/tx_save_query_response.cpp | 39 - .../statistics/aggregator/tx_scan_table.cpp | 74 -- .../aggregator/tx_schedule_scan.cpp | 41 - .../aggregator/tx_schedule_traversal.cpp | 41 + .../aggregator/tx_schemeshard_stats.cpp | 45 +- .../aggregator/ut/ut_analyze_columnshard.cpp | 72 ++ .../aggregator/ut/ut_analyze_datashard.cpp | 104 +++ .../aggregator/ut/ut_traverse_columnshard.cpp | 406 +++++++++ .../aggregator/ut/ut_traverse_datashard.cpp | 161 ++++ ydb/core/statistics/aggregator/ut/ya.make | 30 + ydb/core/statistics/aggregator/ya.make | 17 +- .../database.cpp} | 5 +- .../database.h} | 0 .../ut/ut_database.cpp} | 4 +- ydb/core/statistics/{ => database}/ut/ya.make | 10 +- ydb/core/statistics/database/ya.make | 23 + ydb/core/statistics/events.h | 91 +- ydb/core/statistics/service/http_request.cpp | 163 ++++ ydb/core/statistics/service/http_request.h | 65 ++ ydb/core/statistics/service/service.cpp | 50 ++ ydb/core/statistics/service/service.h | 47 ++ .../service_impl.cpp} | 785 ++++++++++++++---- .../ut/ut_basic_statistics.cpp} | 6 +- ydb/core/statistics/service/ut/ut_service.cpp | 391 +++++++++ ydb/core/statistics/service/ut/ya.make | 29 + ydb/core/statistics/service/ya.make | 28 + ydb/core/statistics/stat_service.h | 16 - ydb/core/statistics/ut/ut_aggregator.cpp | 304 ------- ydb/core/statistics/ut/ut_common.cpp | 144 ---- ydb/core/statistics/ut_common/ut_common.cpp | 370 +++++++++ .../statistics/{ut => ut_common}/ut_common.h | 31 +- ydb/core/statistics/ut_common/ya.make | 15 + ydb/core/statistics/ya.make | 12 +- ydb/core/testlib/basics/services.cpp | 2 +- ydb/core/testlib/basics/ya.make | 1 + .../columnshard/columnshard__statistics.cpp | 26 +- ydb/core/tx/columnshard/columnshard_impl.h | 2 + .../tx/datashard/datashard__column_stats.cpp | 14 +- .../datashard/datashard_ut_column_stats.cpp | 6 +- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 4 +- 61 files changed, 3839 insertions(+), 1216 deletions(-) create mode 100644 ydb/core/statistics/aggregator/tx_ack_timeout.cpp create mode 100644 ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp create mode 100644 ydb/core/statistics/aggregator/tx_analyze_table.cpp rename ydb/core/statistics/aggregator/{tx_statistics_scan_response.cpp => tx_datashard_scan_response.cpp} (69%) delete mode 100644 ydb/core/statistics/aggregator/tx_delete_query_response.cpp create mode 100644 ydb/core/statistics/aggregator/tx_finish_trasersal.cpp create mode 100644 ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp delete mode 100644 ydb/core/statistics/aggregator/tx_save_query_response.cpp delete mode 100644 ydb/core/statistics/aggregator/tx_scan_table.cpp delete mode 100644 ydb/core/statistics/aggregator/tx_schedule_scan.cpp create mode 100644 ydb/core/statistics/aggregator/tx_schedule_traversal.cpp create mode 100644 ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp create mode 100644 ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp create mode 100644 ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp create mode 100644 ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp create mode 100644 ydb/core/statistics/aggregator/ut/ya.make rename ydb/core/statistics/{save_load_stats.cpp => database/database.cpp} (99%) rename ydb/core/statistics/{save_load_stats.h => database/database.h} (100%) rename ydb/core/statistics/{ut/ut_save_load_stats.cpp => database/ut/ut_database.cpp} (97%) rename ydb/core/statistics/{ => database}/ut/ya.make (64%) create mode 100644 ydb/core/statistics/database/ya.make create mode 100644 ydb/core/statistics/service/http_request.cpp create mode 100644 ydb/core/statistics/service/http_request.h create mode 100644 ydb/core/statistics/service/service.cpp create mode 100644 ydb/core/statistics/service/service.h rename ydb/core/statistics/{stat_service.cpp => service/service_impl.cpp} (52%) rename ydb/core/statistics/{ut/ut_statistics.cpp => service/ut/ut_basic_statistics.cpp} (98%) create mode 100644 ydb/core/statistics/service/ut/ut_service.cpp create mode 100644 ydb/core/statistics/service/ut/ya.make create mode 100644 ydb/core/statistics/service/ya.make delete mode 100644 ydb/core/statistics/stat_service.h delete mode 100644 ydb/core/statistics/ut/ut_aggregator.cpp delete mode 100644 ydb/core/statistics/ut/ut_common.cpp create mode 100644 ydb/core/statistics/ut_common/ut_common.cpp rename ydb/core/statistics/{ut => ut_common}/ut_common.h (55%) create mode 100644 ydb/core/statistics/ut_common/ya.make diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 98ba4fbd71d9..0e4a99412c4f 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -107,3 +107,4 @@ ydb/tests/functional/tenants test_tenants.py.* ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset* ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout ydb/tests/functional/rename [test_rename.py */10] chunk chunk +ydb/tests/functional/suite_tests test_postgres.py.TestPGSQL.test_sql_suite[plan-jointest/join2.test] diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 53a4ee27983c..4863864e8a67 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -123,7 +123,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 2433ec8d5b2d..0f2a0e70dd7f 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -98,8 +98,8 @@ PEERDIR( ydb/core/scheme_types ydb/core/security ydb/core/security/ldap_auth_provider - ydb/core/statistics ydb/core/statistics/aggregator + ydb/core/statistics/service ydb/core/sys_view/processor ydb/core/sys_view/service ydb/core/tablet diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index 23ea354852a9..7a93a1860898 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/gateway/ya.make b/ydb/core/kqp/gateway/ya.make index 51767a6992a1..3f7788f2b640 100644 --- a/ydb/core/kqp/gateway/ya.make +++ b/ydb/core/kqp/gateway/ya.make @@ -20,6 +20,7 @@ PEERDIR( ydb/core/kqp/gateway/behaviour/resource_pool ydb/core/kqp/gateway/behaviour/view ydb/core/kqp/gateway/utils + ydb/core/statistics/service ydb/library/yql/providers/result/expr_nodes ) diff --git a/ydb/core/protos/counters_statistics_aggregator.proto b/ydb/core/protos/counters_statistics_aggregator.proto index d85810cce9b6..48fdd0a4bfaf 100644 --- a/ydb/core/protos/counters_statistics_aggregator.proto +++ b/ydb/core/protos/counters_statistics_aggregator.proto @@ -11,11 +11,13 @@ enum ETxTypes { TXTYPE_INIT = 1 [(TxTypeOpts) = {Name: "TxInit"}]; TXTYPE_CONFIGURE = 2 [(TxTypeOpts) = {Name: "TxConfigure"}]; TXTYPE_SCHEMESHARD_STATS = 3 [(TxTypeOpts) = {Name: "TxSchemeShardStats"}]; - TXTYPE_SCAN_TABLE = 4 [(TxTypeOpts) = {Name: "TxScanTable"}]; + TXTYPE_ANALYZE_TABLE = 4 [(TxTypeOpts) = {Name: "TxAnalyzeTable"}]; TXTYPE_NAVIGATE = 5 [(TxTypeOpts) = {Name: "TxNavigate"}]; TXTYPE_RESOLVE = 6 [(TxTypeOpts) = {Name: "TxResolve"}]; TXTYPE_SCAN_RESPONSE = 7 [(TxTypeOpts) = {Name: "TxScanResponse"}]; - TXTYPE_SAVE_QUERY_RESPONSE = 8 [(TxTypeOpts) = {Name: "TxSaveQueryResponse"}]; - TXTYPE_SCHEDULE_SCAN = 9 [(TxTypeOpts) = {Name: "TxScheduleScan"}]; - TXTYPE_DELETE_QUERY_RESPONSE = 10 [(TxTypeOpts) = {Name: "TxDeleteQueryResponse"}]; + TXTYPE_FINISH_TRAVERSAL = 8 [(TxTypeOpts) = {Name: "TxFinishTraversak"}]; + TXTYPE_SCHEDULE_TRAVERSAL = 9 [(TxTypeOpts) = {Name: "TxScheduleTraversal"}]; + TXTYPE_AGGR_STAT_RESPONSE = 10 [(TxTypeOpts) = {Name: "TxAggregateStatisticsResponse"}]; + TXTYPE_RESPONSE_TABLET_DISTRIBUTION = 11 [(TxTypeOpts) = {Name: "TxResponseTabletDistribution"}]; + TXTYPE_ACK_TIMEOUT = 12 [(TxTypeOpts) = {Name: "TxAckTimeout"}]; } diff --git a/ydb/core/protos/statistics.proto b/ydb/core/protos/statistics.proto index d6ab30157c03..4e28439d41db 100644 --- a/ydb/core/protos/statistics.proto +++ b/ydb/core/protos/statistics.proto @@ -13,6 +13,7 @@ message TPathEntry { optional NKikimrProto.TPathID PathId = 1; optional uint64 RowCount = 2; optional uint64 BytesSize = 3; + optional bool IsColumnTable = 4; } message TSchemeShardStats { @@ -67,56 +68,121 @@ message TEvPropagateStatisticsResponse { message TEvStatisticsIsDisabled { } -message TEvScanTable { - optional NKikimrProto.TPathID PathId = 1; +enum EColumnStatisticType { + TYPE_UNSPECIFIED = 0; + TYPE_COUNT_MIN_SKETCH = 1; +} + +// table to gather statistics from +message TTable { + optional NKikimrProto.TPathID PathId = 1; // table path + repeated uint32 ColumnTags = 2; // list of columns to gather statistics from. Empty means asking for every column. } -message TEvScanTableAccepted { - optional uint64 OperationId = 1; +// KQP -> SA +message TEvAnalyze { + optional uint64 Cookie = 1; // request cookie to match response item + repeated TTable Tables = 2; // list of analyzed tables and columns + repeated EColumnStatisticType Types = 3; // list of statistics types requested. Empty means asking for all available. } -message TEvScanTableResponse { +// SA -> KQP +message TEvAnalyzeResponse { + optional uint64 Cookie = 1; } -message TEvGetScanStatus { +// KQP -> SA +message TEvAnalyzeStatus { optional NKikimrProto.TPathID PathId = 1; } -message TEvGetScanStatusResponse { +// SA -> KQP +message TEvAnalyzeStatusResponse { + optional NKikimrProto.TPathID PathId = 1; + enum EStatus { - NO_OPERATION = 0; - ENQUEUED = 1; - IN_PROGRESS = 2; + STATUS_UNSPECIFIED = 0; + STATUS_NO_OPERATION = 1; + STATUS_ENQUEUED = 2; + STATUS_IN_PROGRESS = 3; } - optional EStatus Status = 1; + optional EStatus Status = 2; +} + +// SA -> Shard +message TEvAnalyzeTable { + optional TTable Table = 1; // analyzed table + repeated EColumnStatisticType Types = 2; // list of statistics types requested. Empty means asking for all available. } +// Shard -> SA +message TEvAnalyzeTableResponse { + optional NKikimrProto.TPathID PathId = 1; +} + + message TEvStatisticsRequest { - optional NKikimrDataEvents.TTableId TableId = 1; + optional TTable Table = 1; + optional bytes StartKey = 2; - // list of columns to gather statistics from. Empty means asking for every column. - repeated uint32 ColumnTags = 3; - // list of statistics types requested. Empty means asking for all available. - repeated uint32 Types = 4; + + repeated EColumnStatisticType Types = 3; +} + +message TStatistic { + optional uint32 Type = 1; + optional bytes Data = 2; +} + +message TColumnStatistics { + optional uint32 Tag = 1; + repeated TStatistic Statistics = 2; } message TEvStatisticsResponse { - message TStatistic { - optional uint32 Type = 1; - optional bytes Data = 2; - } - message TColumn { - optional uint32 Tag = 1; - repeated TStatistic Statistics = 2; - } - repeated TColumn Columns = 1; + repeated TColumnStatistics Columns = 1; enum EStatus { - SUCCESS = 1; - ABORTED = 2; - ERROR = 3; + STATUS_UNSPECIFIED = 0; + STATUS_SUCCESS = 1; + STATUS_ABORTED = 2; + STATUS_ERROR = 3; } optional EStatus Status = 2; - optional fixed64 ShardTabletId = 3; } + +message TEvAggregateStatistics { + optional uint64 Round = 1; + message TNodeMap { + optional uint32 NodeId = 1; + repeated fixed64 TabletIds = 2; + } + repeated TNodeMap Nodes = 2; + optional NKikimrProto.TPathID PathId = 3; + repeated uint32 ColumnTags = 4; +} + +message TEvAggregateKeepAlive { + optional uint64 Round = 1; +} + +message TEvAggregateKeepAliveAck { + optional uint64 Round = 1; +} + +message TEvAggregateStatisticsResponse { + optional uint64 Round = 1; + repeated TColumnStatistics Columns = 2; + enum EErrorType { + TYPE_UNSPECIFIED = 0; + TYPE_UNAVAILABLE_NODE = 1; + TYPE_NON_LOCAL_TABLET = 2; + } + message TFailedTablet { + optional EErrorType Error = 1; + optional fixed64 TabletId = 2; + optional uint32 NodeId = 3; + } + repeated TFailedTablet FailedTablets = 3; +} diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 734070165b9e..6a92bf8c2534 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -1,8 +1,9 @@ #include "aggregator_impl.h" +#include +#include + #include -#include -#include #include #include #include @@ -217,7 +218,7 @@ void TStatisticsAggregator::Handle(TEvPrivate::TEvFastPropagateCheck::TPtr&) { } void TStatisticsAggregator::Handle(TEvPrivate::TEvPropagate::TPtr&) { - SA_LOG_D("[" << TabletID() << "] EvPropagate"); + SA_LOG_T("[" << TabletID() << "] EvPropagate"); if (EnableStatistics) { PropagateStatistics(); @@ -308,14 +309,15 @@ void TStatisticsAggregator::SendStatisticsToNode(TNodeId nodeId, const std::vect } void TStatisticsAggregator::PropagateStatistics() { - SA_LOG_D("[" << TabletID() << "] PropagateStatistics()" - << ", node count = " << Nodes.size() - << ", schemeshard count = " << RequestedSchemeShards.size()); - if (Nodes.empty() || RequestedSchemeShards.empty()) { + SA_LOG_T("[" << TabletID() << "] PropagateStatistics() No data"); return; } + SA_LOG_D("[" << TabletID() << "] PropagateStatistics()" + << ", node count = " << Nodes.size() + << ", schemeshard count = " << RequestedSchemeShards.size()); + std::vector nodeIds; nodeIds.reserve(Nodes.size()); for (const auto& [nodeId, _] : Nodes) { @@ -381,8 +383,8 @@ size_t TStatisticsAggregator::PropagatePart(const std::vector& nodeIds, auto ssId = ssIds[index]; auto* entry = record->AddEntries(); entry->SetSchemeShardId(ssId); - auto itStats = BaseStats.find(ssId); - if (itStats != BaseStats.end()) { + auto itStats = BaseStatistics.find(ssId); + if (itStats != BaseStatistics.end()) { entry->SetStats(itStats->second); size += itStats->second.size(); } else { @@ -396,15 +398,26 @@ size_t TStatisticsAggregator::PropagatePart(const std::vector& nodeIds, } void TStatisticsAggregator::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - auto tabletId = ev->Get()->TabletId; - if (ShardRanges.empty()) { + if (!TraversalTableId.PathId) { return; } - auto& range = ShardRanges.front(); - if (tabletId != range.DataShardId) { - return; + auto tabletId = ev->Get()->TabletId; + if (TraversalIsColumnTable) { + if (tabletId == HiveId) { + Schedule(HiveRetryInterval, new TEvPrivate::TEvRequestDistribution); + } else { + SA_LOG_CRIT("[" << TabletID() << "] TEvDeliveryProblem with unexpected tablet " << tabletId); + } + } else { + if (DatashardRanges.empty()) { + return; + } + auto& range = DatashardRanges.front(); + if (tabletId != range.DataShardId) { + return; + } + Resolve(); } - Resolve(); } void TStatisticsAggregator::Handle(TEvStatistics::TEvStatTableCreationResponse::TPtr&) { @@ -419,26 +432,50 @@ void TStatisticsAggregator::Handle(TEvStatistics::TEvStatTableCreationResponse:: } } -void TStatisticsAggregator::Handle(TEvStatistics::TEvGetScanStatus::TPtr& ev) { +void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyzeStatus::TPtr& ev) { auto& inRecord = ev->Get()->Record; auto pathId = PathIdFromPathId(inRecord.GetPathId()); - auto response = std::make_unique(); + auto response = std::make_unique(); auto& outRecord = response->Record; - if (ScanTableId.PathId == pathId) { - outRecord.SetStatus(NKikimrStat::TEvGetScanStatusResponse::IN_PROGRESS); + if (TraversalTableId.PathId == pathId) { + outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS); } else { - auto it = ScanOperationsByPathId.find(pathId); - if (it != ScanOperationsByPathId.end()) { - outRecord.SetStatus(NKikimrStat::TEvGetScanStatusResponse::ENQUEUED); + if (std::any_of(ForceTraversals.begin(), ForceTraversals.end(), + [&pathId](const TForceTraversal& elem) { return elem.PathId == pathId;})) { + outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); } else { - outRecord.SetStatus(NKikimrStat::TEvGetScanStatusResponse::NO_OPERATION); + outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); } } Send(ev->Sender, response.release(), 0, ev->Cookie); } +void TStatisticsAggregator::Handle(TEvPrivate::TEvResolve::TPtr&) { + Resolve(); +} + +void TStatisticsAggregator::Handle(TEvPrivate::TEvRequestDistribution::TPtr&) { + ++HiveRequestRound; + + auto reqDistribution = std::make_unique(); + reqDistribution->Record.MutableTabletIds()->Reserve(TabletsForReqDistribution.size()); + for (auto& tablet : TabletsForReqDistribution) { + reqDistribution->Record.AddTabletIds(tablet); + } + + Send(MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(reqDistribution.release(), HiveId, true)); +} + +void TStatisticsAggregator::Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev) { + auto ack = std::make_unique(); + ack->Record.SetRound(ev->Get()->Record.GetRound()); + Send(ev->Sender, ack.release()); + Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++KeepAliveSeqNo)); +} + void TStatisticsAggregator::InitializeStatisticsTable() { if (!EnableColumnStatistics) { return; @@ -449,7 +486,7 @@ void TStatisticsAggregator::InitializeStatisticsTable() { void TStatisticsAggregator::Navigate() { using TNavigate = NSchemeCache::TSchemeCacheNavigate; TNavigate::TEntry entry; - entry.TableId = ScanTableId; + entry.TableId = TraversalTableId; entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; entry.Operation = TNavigate::OpTable; @@ -460,10 +497,12 @@ void TStatisticsAggregator::Navigate() { } void TStatisticsAggregator::Resolve() { + ++ResolveRound; + TVector plusInf; - TTableRange range(StartKey.GetCells(), true, plusInf, true, false); + TTableRange range(TraversalStartKey.GetCells(), true, plusInf, true, false); auto keyDesc = MakeHolder( - ScanTableId, range, TKeyDesc::ERowOperation::Read, KeyColumnTypes, Columns); + TraversalTableId, range, TKeyDesc::ERowOperation::Read, KeyColumnTypes, Columns); auto request = std::make_unique(); request->ResultSet.emplace_back(std::move(keyDesc)); @@ -471,18 +510,19 @@ void TStatisticsAggregator::Resolve() { Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvResolveKeySet(request.release())); } -void TStatisticsAggregator::NextRange() { - if (ShardRanges.empty()) { +void TStatisticsAggregator::ScanNextDatashardRange() { + if (DatashardRanges.empty()) { SaveStatisticsToTable(); return; } - auto& range = ShardRanges.front(); + auto& range = DatashardRanges.front(); auto request = std::make_unique(); auto& record = request->Record; - record.MutableTableId()->SetOwnerId(ScanTableId.PathId.OwnerId); - record.MutableTableId()->SetTableId(ScanTableId.PathId.LocalPathId); - record.SetStartKey(StartKey.GetBuffer()); + auto* path = record.MutableTable()->MutablePathId(); + path->SetOwnerId(TraversalTableId.PathId.OwnerId); + path->SetLocalId(TraversalTableId.PathId.LocalPathId); + record.SetStartKey(TraversalStartKey.GetBuffer()); Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(request.release(), range.DataShardId, true), @@ -500,6 +540,9 @@ void TStatisticsAggregator::SaveStatisticsToTable() { std::vector columnTags; std::vector data; auto count = CountMinSketches.size(); + if (count == 0) { + return; + } columnTags.reserve(count); data.reserve(count); @@ -513,7 +556,7 @@ void TStatisticsAggregator::SaveStatisticsToTable() { data.push_back(strSketch); } - Register(CreateSaveStatisticsQuery(ScanTableId.PathId, EStatType::COUNT_MIN_SKETCH, + Register(CreateSaveStatisticsQuery(TraversalTableId.PathId, EStatType::COUNT_MIN_SKETCH, std::move(columnTags), std::move(data))); } @@ -525,60 +568,96 @@ void TStatisticsAggregator::DeleteStatisticsFromTable() { PendingDeleteStatistics = false; - Register(CreateDeleteStatisticsQuery(ScanTableId.PathId)); + Register(CreateDeleteStatisticsQuery(TraversalTableId.PathId)); } -void TStatisticsAggregator::ScheduleNextScan(NIceDb::TNiceDb& db) { - if (!ScanOperations.Empty()) { - auto* operation = ScanOperations.Front(); - ReplyToActorIds.swap(operation->ReplyToActorIds); - - StartScan(db, operation->PathId); - - db.Table().Key(operation->OperationId).Delete(); - ScanOperations.PopFront(); - ScanOperationsByPathId.erase(operation->PathId); +void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { + if (!IsSchemeshardSeen) { + SA_LOG_T("[" << TabletID() << "] No info from schemeshard"); return; } - if (ScanTablesByTime.Empty()) { - return; + + TPathId pathId; + + if (!ForceTraversals.empty() && !LastTraversalWasForce) { + LastTraversalWasForce = true; + + TForceTraversal& operation = ForceTraversals.front(); + pathId = operation.PathId; + + ForceTraversalOperationId = operation.OperationId; + ForceTraversalCookie = operation.Cookie; + ForceTraversalColumnTags = operation.ColumnTags; + ForceTraversalTypes = operation.Types; + ForceTraversalReplyToActorId = operation.ReplyToActorId; + + PersistForceTraversal(db); + +// db.Table().Key(operation.OperationId, operation.PathId.OwnerId, operation.PathId.LocalPathId).Delete(); + ForceTraversals.pop_front(); + } else if (!ScheduleTraversalsByTime.Empty()){ + LastTraversalWasForce = false; + + auto* oldestTable = ScheduleTraversalsByTime.Top(); + if (TInstant::Now() < oldestTable->LastUpdateTime + ScheduleTraversalPeriod) { + SA_LOG_T("[" << TabletID() << "] A schedule traversal is skiped. " + << "The oldest table " << oldestTable->PathId << " update time " << oldestTable->LastUpdateTime << " is too fresh."); + return; + } + + pathId = oldestTable->PathId; + } else { + SA_LOG_E("[" << TabletID() << "] No schedule traversal from schemeshard."); + return; } - auto* topTable = ScanTablesByTime.Top(); - auto now = TInstant::Now(); - auto updateTime = topTable->LastUpdateTime; - if (now - updateTime < ScanIntervalTime) { + + auto itPath = ScheduleTraversals.find(pathId); + if (itPath != ScheduleTraversals.end()) { + TraversalIsColumnTable = itPath->second.IsColumnTable; + } else { + SA_LOG_E("[" << TabletID() << "] traversal path " << pathId << " is not known to schemeshard"); return; } - StartScan(db, topTable->PathId); + + TraversalTableId.PathId = pathId; + + SA_LOG_D("[" << TabletID() << "] Start " + << LastTraversalWasForceString() + << " traversal for path " << pathId); + + StartTraversal(db); } -void TStatisticsAggregator::StartScan(NIceDb::TNiceDb& db, TPathId pathId) { - ScanTableId.PathId = pathId; - ScanStartTime = TInstant::Now(); - PersistCurrentScan(db); +void TStatisticsAggregator::StartTraversal(NIceDb::TNiceDb& db) { + TraversalStartTime = TInstant::Now(); + PersistTraversal(db); - StartKey = TSerializedCellVec(); + TraversalStartKey = TSerializedCellVec(); PersistStartKey(db); Navigate(); } -void TStatisticsAggregator::FinishScan(NIceDb::TNiceDb& db) { - auto pathId = ScanTableId.PathId; +void TStatisticsAggregator::FinishTraversal(NIceDb::TNiceDb& db) { + auto pathId = TraversalTableId.PathId; - auto pathIt = ScanTables.find(pathId); - if (pathIt != ScanTables.end()) { - auto& scanTable = pathIt->second; - scanTable.LastUpdateTime = ScanStartTime; - db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Update( - NIceDb::TUpdate(ScanStartTime.MicroSeconds())); + auto pathIt = ScheduleTraversals.find(pathId); + if (pathIt != ScheduleTraversals.end()) { + auto& traversalTable = pathIt->second; + traversalTable.LastUpdateTime = TraversalStartTime; + db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate(TraversalStartTime.MicroSeconds())); - if (ScanTablesByTime.Has(&scanTable)) { - ScanTablesByTime.Update(&scanTable); + if (ScheduleTraversalsByTime.Has(&traversalTable)) { + ScheduleTraversalsByTime.Update(&traversalTable); } } - ResetScanState(db); + ResetTraversalState(db); +} + +TString TStatisticsAggregator::LastTraversalWasForceString() const { + return LastTraversalWasForce ? "force" : "schedule"; } void TStatisticsAggregator::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value) { @@ -586,40 +665,62 @@ void TStatisticsAggregator::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const NIceDb::TUpdate(value)); } -void TStatisticsAggregator::PersistCurrentScan(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_ScanTableOwnerId, ToString(ScanTableId.PathId.OwnerId)); - PersistSysParam(db, Schema::SysParam_ScanTableLocalPathId, ToString(ScanTableId.PathId.LocalPathId)); - PersistSysParam(db, Schema::SysParam_ScanStartTime, ToString(ScanStartTime.MicroSeconds())); +void TStatisticsAggregator::PersistTraversal(NIceDb::TNiceDb& db) { + PersistSysParam(db, Schema::SysParam_TraversalTableOwnerId, ToString(TraversalTableId.PathId.OwnerId)); + PersistSysParam(db, Schema::SysParam_TraversalTableLocalPathId, ToString(TraversalTableId.PathId.LocalPathId)); + PersistSysParam(db, Schema::SysParam_TraversalStartTime, ToString(TraversalStartTime.MicroSeconds())); + PersistSysParam(db, Schema::SysParam_TraversalIsColumnTable, ToString(TraversalIsColumnTable)); } void TStatisticsAggregator::PersistStartKey(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_StartKey, StartKey.GetBuffer()); + PersistSysParam(db, Schema::SysParam_TraversalStartKey, TraversalStartKey.GetBuffer()); +} + +void TStatisticsAggregator::PersistForceTraversal(NIceDb::TNiceDb& db) { + PersistSysParam(db, Schema::SysParam_ForceTraversalOperationId, ToString(ForceTraversalOperationId)); + PersistSysParam(db, Schema::SysParam_ForceTraversalCookie, ToString(ForceTraversalCookie)); + PersistSysParam(db, Schema::SysParam_ForceTraversalColumnTags, ToString(ForceTraversalColumnTags)); + PersistSysParam(db, Schema::SysParam_ForceTraversalTypes, ToString(ForceTraversalTypes)); } -void TStatisticsAggregator::PersistLastScanOperationId(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_LastScanOperationId, ToString(LastScanOperationId)); +void TStatisticsAggregator::PersistNextForceTraversalOperationId(NIceDb::TNiceDb& db) { + PersistSysParam(db, Schema::SysParam_NextForceTraversalOperationId, ToString(NextForceTraversalOperationId)); } -void TStatisticsAggregator::ResetScanState(NIceDb::TNiceDb& db) { - ScanTableId.PathId = TPathId(); - ScanStartTime = TInstant::MicroSeconds(0); - PersistCurrentScan(db); +void TStatisticsAggregator::PersistGlobalTraversalRound(NIceDb::TNiceDb& db) { + PersistSysParam(db, Schema::SysParam_GlobalTraversalRound, ToString(GlobalTraversalRound)); +} - StartKey = TSerializedCellVec(); +void TStatisticsAggregator::ResetTraversalState(NIceDb::TNiceDb& db) { + ForceTraversalOperationId = 0; + ForceTraversalCookie = 0; + TraversalTableId.PathId = TPathId(); + ForceTraversalColumnTags.clear(); + ForceTraversalTypes.clear(); + TraversalStartTime = TInstant::MicroSeconds(0); + PersistTraversal(db); + + TraversalStartKey = TSerializedCellVec(); PersistStartKey(db); - ReplyToActorIds.clear(); + ForceTraversalReplyToActorId = {}; for (auto& [tag, _] : CountMinSketches) { - db.Table().Key(tag).Delete(); + db.Table().Key(tag).Delete(); } CountMinSketches.clear(); - ShardRanges.clear(); + DatashardRanges.clear(); KeyColumnTypes.clear(); Columns.clear(); ColumnNames.clear(); + + TabletsForReqDistribution.clear(); + + ResolveRound = 0; + HiveRequestRound = 0; + TraversalRound = 0; } template @@ -658,7 +759,7 @@ bool TStatisticsAggregator::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev PRE() { str << "---- StatisticsAggregator ----" << Endl << Endl; str << "Database: " << Database << Endl; - str << "BaseStats: " << BaseStats.size() << Endl; + str << "BaseStatistics: " << BaseStatistics.size() << Endl; str << "SchemeShards: " << SchemeShards.size() << Endl; { std::function&)> extr = @@ -703,24 +804,24 @@ bool TStatisticsAggregator::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev str << "PendingRequests: " << PendingRequests.size() << Endl; str << "ProcessUrgentInFlight: " << ProcessUrgentInFlight << Endl << Endl; - str << "ScanTableId: " << ScanTableId << Endl; + str << "TraversalTableId: " << TraversalTableId << Endl; str << "Columns: " << Columns.size() << Endl; - str << "ShardRanges: " << ShardRanges.size() << Endl; + str << "DatashardRanges: " << DatashardRanges.size() << Endl; str << "CountMinSketches: " << CountMinSketches.size() << Endl << Endl; - str << "ScanTablesByTime: " << ScanTablesByTime.Size() << Endl; - if (!ScanTablesByTime.Empty()) { - auto* scanTable = ScanTablesByTime.Top(); - str << " top: " << scanTable->PathId - << ", last update time: " << scanTable->LastUpdateTime << Endl; + str << "ScheduleTraversalsByTime: " << ScheduleTraversalsByTime.Size() << Endl; + if (!ScheduleTraversalsByTime.Empty()) { + auto* oldestTable = ScheduleTraversalsByTime.Top(); + str << " oldest table: " << oldestTable->PathId + << ", ordest table update time: " << oldestTable->LastUpdateTime << Endl; } - str << "ScanTablesBySchemeShard: " << ScanTablesBySchemeShard.size() << Endl; - if (!ScanTablesBySchemeShard.empty()) { - str << " " << ScanTablesBySchemeShard.begin()->first << Endl; + str << "ScheduleTraversalsBySchemeShard: " << ScheduleTraversalsBySchemeShard.size() << Endl; + if (!ScheduleTraversalsBySchemeShard.empty()) { + str << " " << ScheduleTraversalsBySchemeShard.begin()->first << Endl; std::function extr = [](const auto& x) { return x; }; - PrintContainerStart(ScanTablesBySchemeShard.begin()->second, 2, str, extr); + PrintContainerStart(ScheduleTraversalsBySchemeShard.begin()->second, 2, str, extr); } - str << "ScanStartTime: " << ScanStartTime << Endl; + str << "TraversalStartTime: " << TraversalStartTime << Endl; } } diff --git a/ydb/core/statistics/aggregator/aggregator_impl.h b/ydb/core/statistics/aggregator/aggregator_impl.h index 314a7051a446..ca8872dcb4e5 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.h +++ b/ydb/core/statistics/aggregator/aggregator_impl.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -44,13 +45,15 @@ class TStatisticsAggregator : public TActor, public NTabl struct TTxInit; struct TTxConfigure; struct TTxSchemeShardStats; - struct TTxScanTable; + struct TTxAnalyzeTable; struct TTxNavigate; struct TTxResolve; - struct TTxStatisticsScanResponse; - struct TTxSaveQueryResponse; - struct TTxScheduleScan; - struct TTxDeleteQueryResponse; + struct TTxDatashardScanResponse; + struct TTxFinishTraversal; + struct TTxScheduleTrasersal; + struct TTxAggregateStatisticsResponse; + struct TTxResponseTabletDistribution; + struct TTxAckTimeout; struct TEvPrivate { enum EEv { @@ -58,7 +61,10 @@ class TStatisticsAggregator : public TActor, public NTabl EvFastPropagateCheck, EvProcessUrgent, EvPropagateTimeout, - EvScheduleScan, + EvScheduleTraversal, + EvRequestDistribution, + EvResolve, + EvAckTimeout, EvEnd }; @@ -67,7 +73,16 @@ class TStatisticsAggregator : public TActor, public NTabl struct TEvFastPropagateCheck : public TEventLocal {}; struct TEvProcessUrgent : public TEventLocal {}; struct TEvPropagateTimeout : public TEventLocal {}; - struct TEvScheduleScan : public TEventLocal {}; + struct TEvScheduleTraversal : public TEventLocal {}; + struct TEvRequestDistribution : public TEventLocal {}; + struct TEvResolve : public TEventLocal {}; + + struct TEvAckTimeout : public TEventLocal { + size_t SeqNo = 0; + explicit TEvAckTimeout(size_t seqNo) { + SeqNo = seqNo; + } + }; }; private: @@ -104,7 +119,7 @@ class TStatisticsAggregator : public TActor, public NTabl size_t PropagatePart(const std::vector& nodeIds, const std::vector& ssIds, size_t lastSSIndex, bool useSizeLimit); - void Handle(TEvStatistics::TEvScanTable::TPtr& ev); + void Handle(TEvStatistics::TEvAnalyze::TPtr& ev); void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev); void Handle(NStat::TEvStatistics::TEvStatisticsResponse::TPtr& ev); @@ -112,26 +127,35 @@ class TStatisticsAggregator : public TActor, public NTabl void Handle(TEvStatistics::TEvStatTableCreationResponse::TPtr& ev); void Handle(TEvStatistics::TEvSaveStatisticsQueryResponse::TPtr& ev); void Handle(TEvStatistics::TEvDeleteStatisticsQueryResponse::TPtr& ev); - void Handle(TEvPrivate::TEvScheduleScan::TPtr& ev); - void Handle(TEvStatistics::TEvGetScanStatus::TPtr& ev); + void Handle(TEvPrivate::TEvScheduleTraversal::TPtr& ev); + void Handle(TEvStatistics::TEvAnalyzeStatus::TPtr& ev); + void Handle(TEvHive::TEvResponseTabletDistribution::TPtr& ev); + void Handle(TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev); + void Handle(TEvPrivate::TEvResolve::TPtr& ev); + void Handle(TEvPrivate::TEvRequestDistribution::TPtr& ev); + void Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev); + void Handle(TEvPrivate::TEvAckTimeout::TPtr& ev); void InitializeStatisticsTable(); void Navigate(); void Resolve(); - void NextRange(); + void ScanNextDatashardRange(); void SaveStatisticsToTable(); void DeleteStatisticsFromTable(); void PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value); - void PersistCurrentScan(NIceDb::TNiceDb& db); + void PersistTraversal(NIceDb::TNiceDb& db); + void PersistForceTraversal(NIceDb::TNiceDb& db); void PersistStartKey(NIceDb::TNiceDb& db); - void PersistLastScanOperationId(NIceDb::TNiceDb& db); + void PersistNextForceTraversalOperationId(NIceDb::TNiceDb& db); + void PersistGlobalTraversalRound(NIceDb::TNiceDb& db); - void ResetScanState(NIceDb::TNiceDb& db); - void ScheduleNextScan(NIceDb::TNiceDb& db); - void StartScan(NIceDb::TNiceDb& db, TPathId pathId); - void FinishScan(NIceDb::TNiceDb& db); + void ResetTraversalState(NIceDb::TNiceDb& db); + void ScheduleNextTraversal(NIceDb::TNiceDb& db); + void StartTraversal(NIceDb::TNiceDb& db); + void FinishTraversal(NIceDb::TNiceDb& db); + TString LastTraversalWasForceString() const; STFUNC(StateInit) { StateInitImpl(ev, SelfId()); @@ -154,7 +178,7 @@ class TStatisticsAggregator : public TActor, public NTabl hFunc(TEvPrivate::TEvProcessUrgent, Handle); hFunc(TEvPrivate::TEvPropagateTimeout, Handle); - hFunc(TEvStatistics::TEvScanTable, Handle); + hFunc(TEvStatistics::TEvAnalyze, Handle); hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); hFunc(TEvTxProxySchemeCache::TEvResolveKeySetResult, Handle); hFunc(NStat::TEvStatistics::TEvStatisticsResponse, Handle); @@ -162,8 +186,14 @@ class TStatisticsAggregator : public TActor, public NTabl hFunc(TEvStatistics::TEvStatTableCreationResponse, Handle); hFunc(TEvStatistics::TEvSaveStatisticsQueryResponse, Handle); hFunc(TEvStatistics::TEvDeleteStatisticsQueryResponse, Handle); - hFunc(TEvPrivate::TEvScheduleScan, Handle); - hFunc(TEvStatistics::TEvGetScanStatus, Handle); + hFunc(TEvPrivate::TEvScheduleTraversal, Handle); + hFunc(TEvStatistics::TEvAnalyzeStatus, Handle); + hFunc(TEvHive::TEvResponseTabletDistribution, Handle); + hFunc(TEvStatistics::TEvAggregateStatisticsResponse, Handle); + hFunc(TEvPrivate::TEvResolve, Handle); + hFunc(TEvPrivate::TEvRequestDistribution, Handle); + hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); + hFunc(TEvPrivate::TEvAckTimeout, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -188,7 +218,7 @@ class TStatisticsAggregator : public TActor, public NTabl TDuration PropagateTimeout; static constexpr TDuration FastCheckInterval = TDuration::MilliSeconds(50); - std::unordered_map BaseStats; // schemeshard id -> serialized stats for all paths + std::unordered_map BaseStatistics; // schemeshard id -> serialized stats for all paths std::unordered_map SchemeShards; // all connected schemeshards std::unordered_map SchemeShardPipes; // schemeshard pipe servers @@ -211,11 +241,9 @@ class TStatisticsAggregator : public TActor, public NTabl std::queue PendingRequests; bool ProcessUrgentInFlight = false; - // - - TTableId ScanTableId; // stored in local db - std::unordered_set ReplyToActorIds; + TActorId ForceTraversalReplyToActorId = {}; + bool IsSchemeshardSeen = false; bool IsStatisticsTableCreated = false; bool PendingSaveStatistics = false; bool PendingDeleteStatistics = false; @@ -228,53 +256,85 @@ class TStatisticsAggregator : public TActor, public NTabl TSerializedCellVec EndKey; ui64 DataShardId = 0; }; - std::deque ShardRanges; - - TSerializedCellVec StartKey; // stored in local db + std::deque DatashardRanges; - std::unordered_map> CountMinSketches; // stored in local db + // period for both force and schedule traversals + static constexpr TDuration TraversalPeriod = TDuration::Seconds(1); + // if table traverse time is older, than traserse it on schedule + static constexpr TDuration ScheduleTraversalPeriod = TDuration::Hours(24); - static constexpr TDuration ScanIntervalTime = TDuration::Hours(24); - static constexpr TDuration ScheduleScanIntervalTime = TDuration::Seconds(1); - - struct TScanTable { + struct TScheduleTraversal { TPathId PathId; ui64 SchemeShardId = 0; TInstant LastUpdateTime; + bool IsColumnTable = false; size_t HeapIndexByTime = -1; struct THeapIndexByTime { - size_t& operator()(TScanTable& value) const { + size_t& operator()(TScheduleTraversal& value) const { return value.HeapIndexByTime; } }; struct TLessByTime { - bool operator()(const TScanTable& l, const TScanTable& r) const { + bool operator()(const TScheduleTraversal& l, const TScheduleTraversal& r) const { return l.LastUpdateTime < r.LastUpdateTime; } }; }; - std::unordered_map ScanTables; // stored in local db - std::unordered_map> ScanTablesBySchemeShard; + size_t ResolveRound = 0; + static constexpr size_t MaxResolveRoundCount = 5; + static constexpr TDuration ResolveRetryInterval = TDuration::Seconds(1); + + ui64 HiveId = 0; + std::unordered_set TabletsForReqDistribution; + + size_t HiveRequestRound = 0; + static constexpr size_t MaxHiveRequestRoundCount = 5; + static constexpr TDuration HiveRetryInterval = TDuration::Seconds(1); + + size_t TraversalRound = 0; + static constexpr size_t MaxTraversalRoundCount = 5; - typedef TIntrusiveHeap - TScanTableQueueByTime; - TScanTableQueueByTime ScanTablesByTime; + size_t KeepAliveSeqNo = 0; + static constexpr TDuration KeepAliveTimeout = TDuration::Seconds(3); - struct TScanOperation : public TIntrusiveListItem { + // alternate between forced and scheduled traversals + bool LastTraversalWasForce = false; + +private: // stored in local db + + ui64 ForceTraversalOperationId = 0; + ui64 ForceTraversalCookie = 0; + TString ForceTraversalColumnTags; + TString ForceTraversalTypes; + TTableId TraversalTableId; + bool TraversalIsColumnTable = false; + TSerializedCellVec TraversalStartKey; + TInstant TraversalStartTime; + ui64 NextForceTraversalOperationId = 0; + + size_t GlobalTraversalRound = 1; + + std::unordered_map> CountMinSketches; + + std::unordered_map ScheduleTraversals; + std::unordered_map> ScheduleTraversalsBySchemeShard; + typedef TIntrusiveHeap + TTraversalsByTime; + TTraversalsByTime ScheduleTraversalsByTime; + + struct TForceTraversal { ui64 OperationId = 0; + ui64 Cookie = 0; TPathId PathId; - std::unordered_set ReplyToActorIds; + TString ColumnTags; + TString Types; + TActorId ReplyToActorId; }; - TIntrusiveList ScanOperations; // stored in local db - std::unordered_map ScanOperationsByPathId; - - ui64 LastScanOperationId = 0; // stored in local db - - TInstant ScanStartTime; + std::list ForceTraversals; }; } // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/schema.h b/ydb/core/statistics/aggregator/schema.h index fea591a932c4..dc843a425cc7 100644 --- a/ydb/core/statistics/aggregator/schema.h +++ b/ydb/core/statistics/aggregator/schema.h @@ -13,7 +13,7 @@ struct TAggregatorSchema : NIceDb::Schema { using TColumns = TableColumns; }; - struct BaseStats : Table<2> { + struct BaseStatistics : Table<2> { struct SchemeShardId : Column<1, NScheme::NTypeIds::Uint64> {}; struct Stats : Column<2, NScheme::NTypeIds::String> {}; @@ -21,7 +21,7 @@ struct TAggregatorSchema : NIceDb::Schema { using TColumns = TableColumns; }; - struct Statistics : Table<3> { + struct ColumnStatistics : Table<3> { struct ColumnTag : Column<1, NScheme::NTypeIds::Uint32> {}; struct CountMinSketch : Column<2, NScheme::NTypeIds::String> {}; @@ -29,40 +29,48 @@ struct TAggregatorSchema : NIceDb::Schema { using TColumns = TableColumns; }; - struct ScanTables : Table<4> { + struct ScheduleTraversals : Table<4> { struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; struct LocalPathId : Column<2, NScheme::NTypeIds::Uint64> {}; struct LastUpdateTime : Column<3, NScheme::NTypeIds::Timestamp> {}; struct SchemeShardId : Column<4, NScheme::NTypeIds::Uint64> {}; + struct IsColumnTable : Column<5, NScheme::NTypeIds::Bool> {}; using TKey = TableKey; using TColumns = TableColumns< OwnerId, LocalPathId, LastUpdateTime, - SchemeShardId + SchemeShardId, + IsColumnTable >; }; - - struct ScanOperations : Table<5> { +/* + struct ForceTraversals : Table<5> { struct OperationId : Column<1, NScheme::NTypeIds::Uint64> {}; struct OwnerId : Column<2, NScheme::NTypeIds::Uint64> {}; struct LocalPathId : Column<3, NScheme::NTypeIds::Uint64> {}; + struct Cookie : Column<4, NScheme::NTypeIds::Uint64> {}; + struct ColumnTags : Column<5, NScheme::NTypeIds::String> {}; + struct Types : Column<6, NScheme::NTypeIds::String> {}; - using TKey = TableKey; + using TKey = TableKey; using TColumns = TableColumns< OperationId, OwnerId, - LocalPathId + LocalPathId, + Cookie, + ColumnTags, + Types >; }; - +*/ using TTables = SchemaTables< SysParams, - BaseStats, - Statistics, - ScanTables, - ScanOperations + BaseStatistics, + ColumnStatistics, + ScheduleTraversals +// ForceTraversals >; using TSettings = SchemaSettings< @@ -71,11 +79,17 @@ struct TAggregatorSchema : NIceDb::Schema { >; static constexpr ui64 SysParam_Database = 1; - static constexpr ui64 SysParam_StartKey = 2; - static constexpr ui64 SysParam_ScanTableOwnerId = 3; - static constexpr ui64 SysParam_ScanTableLocalPathId = 4; - static constexpr ui64 SysParam_ScanStartTime = 5; - static constexpr ui64 SysParam_LastScanOperationId = 6; + static constexpr ui64 SysParam_TraversalStartKey = 2; + static constexpr ui64 SysParam_ForceTraversalOperationId = 3; + static constexpr ui64 SysParam_TraversalTableOwnerId = 4; + static constexpr ui64 SysParam_TraversalTableLocalPathId = 5; + static constexpr ui64 SysParam_ForceTraversalCookie = 6; + static constexpr ui64 SysParam_ForceTraversalColumnTags = 7; + static constexpr ui64 SysParam_ForceTraversalTypes = 8; + static constexpr ui64 SysParam_TraversalStartTime = 9; + static constexpr ui64 SysParam_NextForceTraversalOperationId = 10; + static constexpr ui64 SysParam_TraversalIsColumnTable = 11; + static constexpr ui64 SysParam_GlobalTraversalRound = 12; }; } // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_ack_timeout.cpp b/ydb/core/statistics/aggregator/tx_ack_timeout.cpp new file mode 100644 index 000000000000..6afa1bd86afe --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_ack_timeout.cpp @@ -0,0 +1,32 @@ +#include "aggregator_impl.h" + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAckTimeout : public TTxBase { + explicit TTxAckTimeout(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_ACK_TIMEOUT; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAckTimeout::Execute"); + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAckTimeout::Complete"); + + ctx.Send(Self->SelfId(), new TEvPrivate::TEvRequestDistribution); + } +}; + +void TStatisticsAggregator::Handle(TEvPrivate::TEvAckTimeout::TPtr& ev) { + if (ev->Get()->SeqNo < KeepAliveSeqNo) { + return; + } + // timeout + Execute(new TTxAckTimeout(this), TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp b/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp new file mode 100644 index 000000000000..89f01dc55763 --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp @@ -0,0 +1,139 @@ +#include "aggregator_impl.h" + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAggregateStatisticsResponse : public TTxBase { + NKikimrStat::TEvAggregateStatisticsResponse Record; + + enum class EAction : ui8 { + None, + SendReqDistribution, + SendAggregate, + }; + EAction Action = EAction::None; + + std::unique_ptr Request; + + TTxAggregateStatisticsResponse(TSelf* self, NKikimrStat::TEvAggregateStatisticsResponse&& record) + : TTxBase(self) + , Record(std::move(record)) + {} + + TTxType GetTxType() const override { return TXTYPE_AGGR_STAT_RESPONSE; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAggregateStatisticsResponse::Execute"); + + ++Self->KeepAliveSeqNo; // cancel timeout events + + NIceDb::TNiceDb db(txc.DB); + + for (auto& column : Record.GetColumns()) { + auto tag = column.GetTag(); + for (auto& statistic : column.GetStatistics()) { + if (statistic.GetType() == NKikimr::NStat::COUNT_MIN_SKETCH) { + if (!Self->ColumnNames.contains(tag)) { + continue; + } + + auto [currentIt, emplaced] = Self->CountMinSketches.try_emplace(tag); + if (emplaced) { + currentIt->second.reset(TCountMinSketch::Create()); + } + + auto* data = statistic.GetData().Data(); + auto* sketch = reinterpret_cast(data); + *(currentIt->second) += *sketch; + } + } + } + + if (Record.FailedTabletsSize() == 0 || + Self->TraversalRound >= Self->MaxTraversalRoundCount) + { + Self->SaveStatisticsToTable(); + return true; + } + + std::unordered_map> nonLocalTablets; + Self->TabletsForReqDistribution.clear(); + + for (auto& tablet : Record.GetFailedTablets()) { + auto error = tablet.GetError(); + switch (error) { + case NKikimrStat::TEvAggregateStatisticsResponse::TYPE_UNSPECIFIED: + SA_LOG_CRIT("[" << Self->TabletID() << "] Unspecified TEvAggregateStatisticsResponse status"); + return false; + + case NKikimrStat::TEvAggregateStatisticsResponse::TYPE_UNAVAILABLE_NODE: + Self->TabletsForReqDistribution.insert(tablet.GetTabletId()); + Action = EAction::SendReqDistribution; + break; + + case NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET: + auto nodeId = tablet.GetNodeId(); + if (nodeId == 0) { + // we cannot reach this tablet + Self->TabletsForReqDistribution.insert(tablet.GetTabletId()); + Action = EAction::SendReqDistribution; + + } else if (Action != EAction::SendReqDistribution) { + nonLocalTablets[nodeId].push_back(tablet.GetTabletId()); + } + break; + } + } + + if (Action == EAction::SendReqDistribution) { + return true; + } + + Request = std::make_unique(); + auto& outRecord = Request->Record; + + for (auto& [nodeId, tabletIds] : nonLocalTablets) { + auto& outNode = *outRecord.AddNodes(); + outNode.SetNodeId(nodeId); + outNode.MutableTabletIds()->Reserve(tabletIds.size()); + for (auto tabletId : tabletIds) { + outNode.AddTabletIds(tabletId); + } + } + + ++Self->TraversalRound; + ++Self->GlobalTraversalRound; + Self->PersistGlobalTraversalRound(db); + outRecord.SetRound(Self->GlobalTraversalRound); + Action = EAction::SendAggregate; + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAggregateStatisticsResponse::Complete"); + + switch (Action) { + case EAction::SendReqDistribution: + ctx.Send(Self->SelfId(), new TEvPrivate::TEvRequestDistribution); + break; + + case EAction::SendAggregate: + ctx.Send(MakeStatServiceID(Self->SelfId().NodeId()), Request.release()); + ctx.Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++Self->KeepAliveSeqNo)); + break; + + default: + break; + } + } +}; + +void TStatisticsAggregator::Handle(TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + Execute(new TTxAggregateStatisticsResponse(this, std::move(record)), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_analyze_table.cpp b/ydb/core/statistics/aggregator/tx_analyze_table.cpp new file mode 100644 index 000000000000..be97dae4cc31 --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze_table.cpp @@ -0,0 +1,85 @@ +#include "aggregator_impl.h" + +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyzeTable : public TTxBase { + const NKikimrStat::TEvAnalyze& Record; + TActorId ReplyToActorId; + + TTxAnalyzeTable(TSelf* self, const NKikimrStat::TEvAnalyze& record, TActorId replyToActorId) + : TTxBase(self) + , Record(record) + , ReplyToActorId(replyToActorId) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute"); + + if (!Self->EnableColumnStatistics) { + return true; + } + + NIceDb::TNiceDb db(txc.DB); + + const ui64 cookie = Record.GetCookie(); + const TString types = JoinVectorIntoString(TVector(Record.GetTypes().begin(), Record.GetTypes().end()), ","); + + for (const auto& table : Record.GetTables()) { + const TPathId pathId = PathIdFromPathId(table.GetPathId()); + const TString columnTags = JoinVectorIntoString(TVector{table.GetColumnTags().begin(),table.GetColumnTags().end()},","); + + // drop request with the same cookie and path from this sender + if (std::any_of(Self->ForceTraversals.begin(), Self->ForceTraversals.end(), + [this, &pathId, &cookie](const TForceTraversal& elem) { + return elem.PathId == pathId + && elem.Cookie == cookie + && elem.ReplyToActorId == ReplyToActorId + ;})) { + return true; + } + + // create new force trasersal + TForceTraversal operation { + .OperationId = Self->NextForceTraversalOperationId, + .Cookie = cookie, + .PathId = pathId, + .ColumnTags = columnTags, + .Types = types, + .ReplyToActorId = ReplyToActorId + }; + Self->ForceTraversals.emplace_back(operation); +/* + db.Table().Key(Self->NextForceTraversalOperationId, pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate(Self->NextForceTraversalOperationId), + NIceDb::TUpdate(pathId.OwnerId), + NIceDb::TUpdate(pathId.LocalPathId), + NIceDb::TUpdate(cookie), + NIceDb::TUpdate(columnTags), + NIceDb::TUpdate(types) + ); +*/ + } + + Self->PersistNextForceTraversalOperationId(db); + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Complete"); + } +}; + +void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyze::TPtr& ev) { + ++NextForceTraversalOperationId; + + Execute(new TTxAnalyzeTable(this, ev->Get()->Record, ev->Sender), TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_statistics_scan_response.cpp b/ydb/core/statistics/aggregator/tx_datashard_scan_response.cpp similarity index 69% rename from ydb/core/statistics/aggregator/tx_statistics_scan_response.cpp rename to ydb/core/statistics/aggregator/tx_datashard_scan_response.cpp index f32a78450ef3..79ea60b5bbc7 100644 --- a/ydb/core/statistics/aggregator/tx_statistics_scan_response.cpp +++ b/ydb/core/statistics/aggregator/tx_datashard_scan_response.cpp @@ -4,11 +4,11 @@ namespace NKikimr::NStat { -struct TStatisticsAggregator::TTxStatisticsScanResponse : public TTxBase { +struct TStatisticsAggregator::TTxDatashardScanResponse : public TTxBase { NKikimrStat::TEvStatisticsResponse Record; bool IsCorrectShardId = false; - TTxStatisticsScanResponse(TSelf* self, NKikimrStat::TEvStatisticsResponse&& record) + TTxDatashardScanResponse(TSelf* self, NKikimrStat::TEvStatisticsResponse&& record) : TTxBase(self) , Record(std::move(record)) {} @@ -16,17 +16,17 @@ struct TStatisticsAggregator::TTxStatisticsScanResponse : public TTxBase { TTxType GetTxType() const override { return TXTYPE_SCAN_RESPONSE; } bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxStatisticsScanResponse::Execute"); + SA_LOG_D("[" << Self->TabletID() << "] TTxDatashardScanResponse::Execute"); NIceDb::TNiceDb db(txc.DB); // TODO: handle scan errors - if (Self->ShardRanges.empty()) { + if (Self->DatashardRanges.empty()) { return true; } - auto& range = Self->ShardRanges.front(); + auto& range = Self->DatashardRanges.front(); auto replyShardId = Record.GetShardTabletId(); if (replyShardId != range.DataShardId) { @@ -53,31 +53,31 @@ struct TStatisticsAggregator::TTxStatisticsScanResponse : public TTxBase { *current += *sketch; auto currentStr = TString(current->AsStringBuf()); - db.Table().Key(tag).Update( - NIceDb::TUpdate(currentStr)); + db.Table().Key(tag).Update( + NIceDb::TUpdate(currentStr)); } } } - Self->StartKey = range.EndKey; + Self->TraversalStartKey = range.EndKey; Self->PersistStartKey(db); return true; } void Complete(const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxStatisticsScanResponse::Complete"); + SA_LOG_D("[" << Self->TabletID() << "] TTxDatashardScanResponse::Complete"); - if (IsCorrectShardId && !Self->ShardRanges.empty()) { - Self->ShardRanges.pop_front(); - Self->NextRange(); + if (IsCorrectShardId && !Self->DatashardRanges.empty()) { + Self->DatashardRanges.pop_front(); + Self->ScanNextDatashardRange(); } } }; void TStatisticsAggregator::Handle(NStat::TEvStatistics::TEvStatisticsResponse::TPtr& ev) { auto& record = ev->Get()->Record; - Execute(new TTxStatisticsScanResponse(this, std::move(record)), + Execute(new TTxDatashardScanResponse(this, std::move(record)), TActivationContext::AsActorContext()); } diff --git a/ydb/core/statistics/aggregator/tx_delete_query_response.cpp b/ydb/core/statistics/aggregator/tx_delete_query_response.cpp deleted file mode 100644 index 2c0487846a8f..000000000000 --- a/ydb/core/statistics/aggregator/tx_delete_query_response.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "aggregator_impl.h" - -#include - -namespace NKikimr::NStat { - -struct TStatisticsAggregator::TTxDeleteQueryResponse : public TTxBase { - std::unordered_set ReplyToActorIds; - - TTxDeleteQueryResponse(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_DELETE_QUERY_RESPONSE; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxDeleteQueryResponse::Execute"); - - ReplyToActorIds.swap(Self->ReplyToActorIds); - - NIceDb::TNiceDb db(txc.DB); - Self->FinishScan(db); - - return true; - } - - void Complete(const TActorContext& ctx) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxDeleteQueryResponse::Complete"); - - for (auto& id : ReplyToActorIds) { - ctx.Send(id, new TEvStatistics::TEvScanTableResponse); - } - } -}; -void TStatisticsAggregator::Handle(TEvStatistics::TEvDeleteStatisticsQueryResponse::TPtr&) { - Execute(new TTxDeleteQueryResponse(this), TActivationContext::AsActorContext()); -} - -} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp new file mode 100644 index 000000000000..6d9d165def1c --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp @@ -0,0 +1,63 @@ +#include "aggregator_impl.h" + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxFinishTraversal : public TTxBase { + ui64 OperationId; + ui64 Cookie; + TPathId PathId; + TActorId ReplyToActorId; + + TTxFinishTraversal(TSelf* self) + : TTxBase(self) + , OperationId(self->ForceTraversalOperationId) + , Cookie(self->ForceTraversalCookie) + , PathId(self->TraversalTableId.PathId) + , ReplyToActorId(self->ForceTraversalReplyToActorId) + {} + + TTxType GetTxType() const override { return TXTYPE_FINISH_TRAVERSAL; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Execute"); + + NIceDb::TNiceDb db(txc.DB); + Self->FinishTraversal(db); + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete " << + Self->LastTraversalWasForceString() << " traversal for path " << PathId); + + if (!ReplyToActorId) { + SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete. No ActorId to send reply."); + return; + } + + bool operationsRemain = std::any_of(Self->ForceTraversals.begin(), Self->ForceTraversals.end(), + [this](const TForceTraversal& elem) { return elem.OperationId == OperationId;}); + + if (operationsRemain) { + SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete. Don't send TEvAnalyzeResponse. " << + "There are pending operations, Cookie " << Cookie << " , ActorId=" << ReplyToActorId); + } else { + SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete. " << + "Send TEvAnalyzeResponse, Cookie=" << Cookie << ", ActorId=" << ReplyToActorId); + auto response = std::make_unique(); + response->Record.SetCookie(Cookie); + ctx.Send(ReplyToActorId, response.release()); + } + } +}; +void TStatisticsAggregator::Handle(TEvStatistics::TEvSaveStatisticsQueryResponse::TPtr&) { + Execute(new TTxFinishTraversal(this), TActivationContext::AsActorContext()); +} +void TStatisticsAggregator::Handle(TEvStatistics::TEvDeleteStatisticsQueryResponse::TPtr&) { + Execute(new TTxFinishTraversal(this), TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index 4dbd2295c015..f50ea80d2705 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -19,16 +19,16 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { { // precharge auto sysParamsRowset = db.Table().Range().Select(); - auto baseStatsRowset = db.Table().Range().Select(); - auto statisticsRowset = db.Table().Range().Select(); - auto scanTablesRowset = db.Table().Range().Select(); - auto scanOperationsRowset = db.Table().Range().Select(); + auto baseStatisticsRowset = db.Table().Range().Select(); + auto statisticsRowset = db.Table().Range().Select(); + auto scheduleTraversalRowset = db.Table().Range().Select(); +// auto forceTraversalRowset = db.Table().Range().Select(); if (!sysParamsRowset.IsReady() || - !baseStatsRowset.IsReady() || + !baseStatisticsRowset.IsReady() || !statisticsRowset.IsReady() || - !scanTablesRowset.IsReady() || - !scanOperationsRowset.IsReady()) + !scheduleTraversalRowset.IsReady()) +// !forceTraversalRowset.IsReady()) { return false; } @@ -48,35 +48,63 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { switch (id) { case Schema::SysParam_Database: Self->Database = value; - SA_LOG_D("[" << Self->TabletID() << "] Loading database: " << Self->Database); + SA_LOG_D("[" << Self->TabletID() << "] Loaded database: " << Self->Database); break; - case Schema::SysParam_StartKey: - Self->StartKey = TSerializedCellVec(value); - SA_LOG_D("[" << Self->TabletID() << "] Loading start key"); + case Schema::SysParam_TraversalStartKey: + Self->TraversalStartKey = TSerializedCellVec(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal start key"); break; - case Schema::SysParam_ScanTableOwnerId: - Self->ScanTableId.PathId.OwnerId = FromString(value); - SA_LOG_D("[" << Self->TabletID() << "] Loading scan table owner id: " - << Self->ScanTableId.PathId.OwnerId); + case Schema::SysParam_ForceTraversalOperationId: { + Self->ForceTraversalOperationId = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal operation id: " << value); break; - case Schema::SysParam_ScanTableLocalPathId: - Self->ScanTableId.PathId.LocalPathId = FromString(value); - SA_LOG_D("[" << Self->TabletID() << "] Loading scan table local path id: " - << Self->ScanTableId.PathId.LocalPathId); + } + case Schema::SysParam_ForceTraversalCookie: { + Self->ForceTraversalCookie = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal cookie: " << value); break; - case Schema::SysParam_ScanStartTime: { + } + case Schema::SysParam_TraversalTableOwnerId: + Self->TraversalTableId.PathId.OwnerId = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal table owner id: " + << Self->TraversalTableId.PathId.OwnerId); + break; + case Schema::SysParam_TraversalTableLocalPathId: + Self->TraversalTableId.PathId.LocalPathId = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal table local path id: " + << Self->TraversalTableId.PathId.LocalPathId); + break; + case Schema::SysParam_ForceTraversalColumnTags: { + Self->ForceTraversalColumnTags = value; + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal columns tags: " << value); + break; + } + case Schema::SysParam_ForceTraversalTypes: { + Self->ForceTraversalTypes = value; + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal types: " << value); + break; + } + case Schema::SysParam_TraversalStartTime: { auto us = FromString(value); - Self->ScanStartTime = TInstant::MicroSeconds(us); - SA_LOG_D("[" << Self->TabletID() << "] Loading scan start time: " << us); + Self->TraversalStartTime = TInstant::MicroSeconds(us); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal start time: " << us); break; } - case Schema::SysParam_LastScanOperationId: { - auto id = FromString(value); - Self->LastScanOperationId = id; - SA_LOG_D("[" << Self->TabletID() << "] Loading last scan operation id: " << id); + case Schema::SysParam_NextForceTraversalOperationId: { + Self->NextForceTraversalOperationId = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded next traversal operation id: " << value); + break; + } + case Schema::SysParam_TraversalIsColumnTable: { + Self->TraversalIsColumnTable = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal IsColumnTable: " << value); + break; + } + case Schema::SysParam_GlobalTraversalRound: { + Self->GlobalTraversalRound = FromString(value); + SA_LOG_D("[" << Self->TabletID() << "] Loaded global traversal round: " << value); break; } - default: SA_LOG_CRIT("[" << Self->TabletID() << "] Unexpected SysParam id: " << id); } @@ -87,42 +115,42 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { } } - // BaseStats + // BaseStatistics { - Self->BaseStats.clear(); + Self->BaseStatistics.clear(); - auto rowset = db.Table().Range().Select(); + auto rowset = db.Table().Range().Select(); if (!rowset.IsReady()) { return false; } while (!rowset.EndOfSet()) { - ui64 schemeShardId = rowset.GetValue(); - TString stats = rowset.GetValue(); + ui64 schemeShardId = rowset.GetValue(); + TString stats = rowset.GetValue(); - Self->BaseStats[schemeShardId] = stats; + Self->BaseStatistics[schemeShardId] = stats; if (!rowset.Next()) { return false; } } - SA_LOG_D("[" << Self->TabletID() << "] Loading base stats: " - << "schemeshard count# " << Self->BaseStats.size()); + SA_LOG_D("[" << Self->TabletID() << "] Loaded BaseStatistics: " + << "schemeshard count# " << Self->BaseStatistics.size()); } - // Statistics + // ColumnStatistics { Self->CountMinSketches.clear(); - auto rowset = db.Table().Range().Select(); + auto rowset = db.Table().Range().Select(); if (!rowset.IsReady()) { return false; } while (!rowset.EndOfSet()) { - ui32 columnTag = rowset.GetValue(); - TString sketch = rowset.GetValue(); + ui32 columnTag = rowset.GetValue(); + TString sketch = rowset.GetValue(); Self->CountMinSketches[columnTag].reset( TCountMinSketch::FromString(sketch.data(), sketch.size())); @@ -132,78 +160,88 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { } } - SA_LOG_D("[" << Self->TabletID() << "] Loading statistics: " + SA_LOG_D("[" << Self->TabletID() << "] Loaded ColumnStatistics: " << "column count# " << Self->CountMinSketches.size()); } - // ScanTables + // ScheduleTraversals { - Self->ScanTablesByTime.Clear(); - Self->ScanTablesBySchemeShard.clear(); - Self->ScanTables.clear(); + Self->ScheduleTraversalsByTime.Clear(); + Self->ScheduleTraversalsBySchemeShard.clear(); + Self->ScheduleTraversals.clear(); - auto rowset = db.Table().Range().Select(); + auto rowset = db.Table().Range().Select(); if (!rowset.IsReady()) { return false; } while (!rowset.EndOfSet()) { - ui64 ownerId = rowset.GetValue(); - ui64 localPathId = rowset.GetValue(); - ui64 lastUpdateTime = rowset.GetValue(); - ui64 schemeShardId = rowset.GetValue(); + ui64 ownerId = rowset.GetValue(); + ui64 localPathId = rowset.GetValue(); + ui64 lastUpdateTime = rowset.GetValue(); + ui64 schemeShardId = rowset.GetValue(); + bool isColumnTable = rowset.GetValue(); auto pathId = TPathId(ownerId, localPathId); - TScanTable scanTable; - scanTable.PathId = pathId; - scanTable.SchemeShardId = schemeShardId; - scanTable.LastUpdateTime = TInstant::MicroSeconds(lastUpdateTime); + TScheduleTraversal scheduleTraversal; + scheduleTraversal.PathId = pathId; + scheduleTraversal.SchemeShardId = schemeShardId; + scheduleTraversal.LastUpdateTime = TInstant::MicroSeconds(lastUpdateTime); + scheduleTraversal.IsColumnTable = isColumnTable; - auto [it, _] = Self->ScanTables.emplace(pathId, scanTable); - Self->ScanTablesByTime.Add(&it->second); - Self->ScanTablesBySchemeShard[schemeShardId].insert(pathId); + auto [it, _] = Self->ScheduleTraversals.emplace(pathId, scheduleTraversal); + Self->ScheduleTraversalsByTime.Add(&it->second); + Self->ScheduleTraversalsBySchemeShard[schemeShardId].insert(pathId); if (!rowset.Next()) { return false; } } - SA_LOG_D("[" << Self->TabletID() << "] Loading scan tables: " - << "table count# " << Self->ScanTables.size()); + SA_LOG_D("[" << Self->TabletID() << "] Loaded ScheduleTraversals: " + << "table count# " << Self->ScheduleTraversals.size()); } - // ScanOperations + // ForceTraversals +/* { - Self->ScanOperations.Clear(); - Self->ScanOperationsByPathId.clear(); + Self->ForceTraversals.clear(); - auto rowset = db.Table().Range().Select(); + auto rowset = db.Table().Range().Select(); if (!rowset.IsReady()) { return false; } while (!rowset.EndOfSet()) { - ui64 operationId = rowset.GetValue(); - ui64 ownerId = rowset.GetValue(); - ui64 localPathId = rowset.GetValue(); + ui64 operationId = rowset.GetValue(); + ui64 ownerId = rowset.GetValue(); + ui64 localPathId = rowset.GetValue(); + ui64 cookie = rowset.GetValue(); + TString columnTags = rowset.GetValue(); + TString types = rowset.GetValue(); auto pathId = TPathId(ownerId, localPathId); - TScanOperation& operation = Self->ScanOperationsByPathId[pathId]; - operation.PathId = pathId; - operation.OperationId = operationId; - Self->ScanOperations.PushBack(&operation); + TForceTraversal operation { + .OperationId = operationId, + .Cookie = cookie, + .PathId = pathId, + .ColumnTags = columnTags, + .Types = types, + .ReplyToActorId = {} + }; + Self->ForceTraversals.emplace_back(operation); if (!rowset.Next()) { return false; } } - SA_LOG_D("[" << Self->TabletID() << "] Loading scan operations: " - << "table count# " << Self->ScanOperationsByPathId.size()); + SA_LOG_D("[" << Self->TabletID() << "] Loaded ForceTraversals: " + << "table count# " << Self->ForceTraversals.size()); } - +*/ return true; } @@ -217,11 +255,11 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { Self->SubscribeForConfigChanges(ctx); Self->Schedule(Self->PropagateInterval, new TEvPrivate::TEvPropagate()); - Self->Schedule(Self->ScheduleScanIntervalTime, new TEvPrivate::TEvScheduleScan()); + Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); Self->InitializeStatisticsTable(); - if (Self->ScanTableId.PathId) { + if (Self->TraversalTableId.PathId) { Self->Navigate(); } diff --git a/ydb/core/statistics/aggregator/tx_init_schema.cpp b/ydb/core/statistics/aggregator/tx_init_schema.cpp index 4c77a9b5e9cb..352786245589 100644 --- a/ydb/core/statistics/aggregator/tx_init_schema.cpp +++ b/ydb/core/statistics/aggregator/tx_init_schema.cpp @@ -15,7 +15,9 @@ struct TStatisticsAggregator::TTxInitSchema : public TTxBase { NIceDb::TNiceDb(txc.DB).Materialize(); static constexpr NIceDb::TTableId bigTableIds[] = { - Schema::BaseStats::TableId, + Schema::BaseStatistics::TableId, + Schema::ColumnStatistics::TableId, + Schema::ScheduleTraversals::TableId }; for (auto id : bigTableIds) { diff --git a/ydb/core/statistics/aggregator/tx_navigate.cpp b/ydb/core/statistics/aggregator/tx_navigate.cpp index 95abd04cecbb..35534d042b06 100644 --- a/ydb/core/statistics/aggregator/tx_navigate.cpp +++ b/ydb/core/statistics/aggregator/tx_navigate.cpp @@ -29,7 +29,7 @@ struct TStatisticsAggregator::TTxNavigate : public TTxBase { if (entry.Status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) { Self->DeleteStatisticsFromTable(); } else { - Self->FinishScan(db); + Self->FinishTraversal(db); } return true; } @@ -52,12 +52,21 @@ struct TStatisticsAggregator::TTxNavigate : public TTxBase { Self->KeyColumnTypes[col.second.KeyOrder] = col.second.PType; } - if (Self->StartKey.GetCells().empty()) { + if (Self->TraversalStartKey.GetCells().empty()) { TVector minusInf(Self->KeyColumnTypes.size()); - Self->StartKey = TSerializedCellVec(minusInf); + Self->TraversalStartKey = TSerializedCellVec(minusInf); Self->PersistStartKey(db); } + if (Self->TraversalIsColumnTable) { + // TODO: serverless case + if (entry.DomainInfo->Params.HasHive()) { + Self->HiveId = entry.DomainInfo->Params.GetHive(); + } else { + Self->HiveId = AppData()->DomainsInfo->GetHive(); + } + } + return true; } diff --git a/ydb/core/statistics/aggregator/tx_resolve.cpp b/ydb/core/statistics/aggregator/tx_resolve.cpp index 36c9360e6f43..3215cde186b5 100644 --- a/ydb/core/statistics/aggregator/tx_resolve.cpp +++ b/ydb/core/statistics/aggregator/tx_resolve.cpp @@ -1,5 +1,6 @@ #include "aggregator_impl.h" +#include #include namespace NKikimr::NStat { @@ -7,6 +8,7 @@ namespace NKikimr::NStat { struct TStatisticsAggregator::TTxResolve : public TTxBase { std::unique_ptr Request; bool Cancelled = false; + bool DoSend = true; TTxResolve(TSelf* self, NSchemeCache::TSchemeCacheRequest* request) : TTxBase(self) @@ -29,34 +31,56 @@ struct TStatisticsAggregator::TTxResolve : public TTxBase { if (entry.Status == NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist) { Self->DeleteStatisticsFromTable(); } else { - Self->FinishScan(db); + Self->FinishTraversal(db); } return true; } - Self->ShardRanges.clear(); + auto& partitioning = entry.KeyDescription->GetPartitions(); - auto& partitioning = entry.KeyDescription->Partitioning; - for (auto& part : *partitioning) { + if (Self->TraversalIsColumnTable) { + Self->TabletsForReqDistribution.clear(); + Self->CountMinSketches.clear(); + } else { + Self->DatashardRanges.clear(); + } + + for (auto& part : partitioning) { if (!part.Range) { continue; } - TRange range; - range.EndKey = part.Range->EndKeyPrefix; - range.DataShardId = part.ShardId; - Self->ShardRanges.push_back(range); + if (Self->TraversalIsColumnTable) { + Self->TabletsForReqDistribution.insert(part.ShardId); + } else { + TRange range; + range.EndKey = part.Range->EndKeyPrefix; + range.DataShardId = part.ShardId; + Self->DatashardRanges.push_back(range); + } } + + if (Self->TraversalIsColumnTable && Self->TabletsForReqDistribution.empty()) { + Self->FinishTraversal(db); + DoSend = false; + } + return true; } - void Complete(const TActorContext&) override { + void Complete(const TActorContext& ctx) override { SA_LOG_D("[" << Self->TabletID() << "] TTxResolve::Complete"); if (Cancelled) { return; } - Self->NextRange(); + if (Self->TraversalIsColumnTable) { + if (DoSend) { + ctx.Send(Self->SelfId(), new TEvPrivate::TEvRequestDistribution); + } + } else { + Self->ScanNextDatashardRange(); + } } }; diff --git a/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp b/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp new file mode 100644 index 000000000000..341b31964907 --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp @@ -0,0 +1,109 @@ +#include "aggregator_impl.h" + +#include +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { + NKikimrHive::TEvResponseTabletDistribution Record; + + enum class EAction : ui8 { + None, + SendAggregate, + ScheduleResolve, + ScheduleReqDistribution, + }; + EAction Action = EAction::None; + + std::unique_ptr Request; + + TTxResponseTabletDistribution(TSelf* self, NKikimrHive::TEvResponseTabletDistribution&& record) + : TTxBase(self) + , Record(std::move(record)) + {} + + TTxType GetTxType() const override { return TXTYPE_RESPONSE_TABLET_DISTRIBUTION; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxResponseTabletDistribution::Execute"); + + NIceDb::TNiceDb db(txc.DB); + + Request = std::make_unique(); + auto& outRecord = Request->Record; + + PathIdFromPathId(Self->TraversalTableId.PathId, outRecord.MutablePathId()); + + TVector columnTags = Scan(SplitString(Self->ForceTraversalColumnTags, ",")); + outRecord.MutableColumnTags()->Add(columnTags.begin(), columnTags.end()); + + auto distribution = Self->TabletsForReqDistribution; + for (auto& inNode : Record.GetNodes()) { + if (inNode.GetNodeId() == 0) { + // these tablets are probably in Hive boot queue + if (Self->HiveRequestRound < Self->MaxHiveRequestRoundCount) { + Action = EAction::ScheduleReqDistribution; + } + continue; + } + auto& outNode = *outRecord.AddNodes(); + outNode.SetNodeId(inNode.GetNodeId()); + outNode.MutableTabletIds()->Reserve(inNode.TabletIdsSize()); + for (auto tabletId : inNode.GetTabletIds()) { + outNode.AddTabletIds(tabletId); + distribution.erase(tabletId); + } + } + + if (Action == EAction::ScheduleReqDistribution) { + return true; + } + + if (!distribution.empty() && Self->ResolveRound < Self->MaxResolveRoundCount) { + // these tablets do not exist in Hive anymore + Action = EAction::ScheduleResolve; + return true; + } + + ++Self->TraversalRound; + ++Self->GlobalTraversalRound; + Self->PersistGlobalTraversalRound(db); + outRecord.SetRound(Self->GlobalTraversalRound); + Action = EAction::SendAggregate; + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxResponseTabletDistribution::Complete"); + + switch (Action) { + case EAction::ScheduleResolve: + ctx.Schedule(ResolveRetryInterval, new TEvPrivate::TEvResolve()); + break; + + case EAction::ScheduleReqDistribution: + ctx.Schedule(HiveRetryInterval, new TEvPrivate::TEvRequestDistribution()); + break; + + case EAction::SendAggregate: + ctx.Send(MakeStatServiceID(Self->SelfId().NodeId()), Request.release()); + ctx.Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++Self->KeepAliveSeqNo)); + break; + + default: + break; + } + } +}; + +void TStatisticsAggregator::Handle(TEvHive::TEvResponseTabletDistribution::TPtr& ev) { + auto& record = ev->Get()->Record; + Execute(new TTxResponseTabletDistribution(this, std::move(record)), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_save_query_response.cpp b/ydb/core/statistics/aggregator/tx_save_query_response.cpp deleted file mode 100644 index 3c8a24357af8..000000000000 --- a/ydb/core/statistics/aggregator/tx_save_query_response.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "aggregator_impl.h" - -#include - -namespace NKikimr::NStat { - -struct TStatisticsAggregator::TTxSaveQueryResponse : public TTxBase { - std::unordered_set ReplyToActorIds; - - TTxSaveQueryResponse(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_SAVE_QUERY_RESPONSE; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxSaveQueryResponse::Execute"); - - ReplyToActorIds.swap(Self->ReplyToActorIds); - - NIceDb::TNiceDb db(txc.DB); - Self->FinishScan(db); - - return true; - } - - void Complete(const TActorContext& ctx) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxSaveQueryResponse::Complete"); - - for (auto& id : ReplyToActorIds) { - ctx.Send(id, new TEvStatistics::TEvScanTableResponse); - } - } -}; -void TStatisticsAggregator::Handle(TEvStatistics::TEvSaveStatisticsQueryResponse::TPtr&) { - Execute(new TTxSaveQueryResponse(this), TActivationContext::AsActorContext()); -} - -} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_scan_table.cpp b/ydb/core/statistics/aggregator/tx_scan_table.cpp deleted file mode 100644 index ae687ccca616..000000000000 --- a/ydb/core/statistics/aggregator/tx_scan_table.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include "aggregator_impl.h" - -#include - -namespace NKikimr::NStat { - -struct TStatisticsAggregator::TTxScanTable : public TTxBase { - NKikimrStat::TEvScanTable Record; - TActorId ReplyToActorId; - ui64 OperationId = 0; - - TTxScanTable(TSelf* self, NKikimrStat::TEvScanTable&& record, TActorId replyToActorId) - : TTxBase(self) - , Record(std::move(record)) - , ReplyToActorId(replyToActorId) - {} - - TTxType GetTxType() const override { return TXTYPE_SCAN_TABLE; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxScanTable::Execute"); - - if (!Self->EnableColumnStatistics) { - return true; - } - - auto pathId = PathIdFromPathId(Record.GetPathId()); - - auto itOp = Self->ScanOperationsByPathId.find(pathId); - if (itOp != Self->ScanOperationsByPathId.end()) { - itOp->second.ReplyToActorIds.insert(ReplyToActorId); - OperationId = itOp->second.OperationId; - return true; - } - - NIceDb::TNiceDb db(txc.DB); - - TScanOperation& operation = Self->ScanOperationsByPathId[pathId]; - operation.PathId = pathId; - operation.OperationId = ++Self->LastScanOperationId; - operation.ReplyToActorIds.insert(ReplyToActorId); - Self->ScanOperations.PushBack(&operation); - - Self->PersistLastScanOperationId(db); - - db.Table().Key(operation.OperationId).Update( - NIceDb::TUpdate(pathId.OwnerId), - NIceDb::TUpdate(pathId.LocalPathId)); - - OperationId = operation.OperationId; - - return true; - } - - void Complete(const TActorContext& ctx) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxScanTable::Complete"); - - if (!Self->EnableColumnStatistics) { - return; - } - - auto accepted = std::make_unique(); - accepted->Record.SetOperationId(OperationId); - ctx.Send(ReplyToActorId, accepted.release()); - } -}; - -void TStatisticsAggregator::Handle(TEvStatistics::TEvScanTable::TPtr& ev) { - auto& record = ev->Get()->Record; - Execute(new TTxScanTable(this, std::move(record), ev->Sender), - TActivationContext::AsActorContext()); -} - -} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_schedule_scan.cpp b/ydb/core/statistics/aggregator/tx_schedule_scan.cpp deleted file mode 100644 index a1bbd9438915..000000000000 --- a/ydb/core/statistics/aggregator/tx_schedule_scan.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "aggregator_impl.h" - -#include - -namespace NKikimr::NStat { - -struct TStatisticsAggregator::TTxScheduleScan : public TTxBase { - TTxScheduleScan(TSelf* self) - : TTxBase(self) - {} - - TTxType GetTxType() const override { return TXTYPE_SCHEDULE_SCAN; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxScheduleScan::Execute"); - - Self->Schedule(Self->ScheduleScanIntervalTime, new TEvPrivate::TEvScheduleScan()); - - if (!Self->EnableColumnStatistics) { - return true; - } - - if (Self->ScanTableId.PathId) { - return true; // scan is in progress - } - - NIceDb::TNiceDb db(txc.DB); - Self->ScheduleNextScan(db); - return true; - } - - void Complete(const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxScheduleScan::Complete"); - } -}; - -void TStatisticsAggregator::Handle(TEvPrivate::TEvScheduleScan::TPtr&) { - Execute(new TTxScheduleScan(this), TActivationContext::AsActorContext()); -} - -} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp new file mode 100644 index 000000000000..0eb09bbd67af --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp @@ -0,0 +1,41 @@ +#include "aggregator_impl.h" + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxScheduleTrasersal : public TTxBase { + TTxScheduleTrasersal(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_SCHEDULE_TRAVERSAL; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Execute"); + + Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); + + if (!Self->EnableColumnStatistics) { + return true; + } + + if (Self->TraversalTableId.PathId) { + return true; // traverse is in progress + } + + NIceDb::TNiceDb db(txc.DB); + Self->ScheduleNextTraversal(db); + return true; + } + + void Complete(const TActorContext&) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Complete"); + } +}; + +void TStatisticsAggregator::Handle(TEvPrivate::TEvScheduleTraversal::TPtr&) { + Execute(new TTxScheduleTrasersal(this), TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp b/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp index ead6e8642fc0..f81a99b4ac83 100644 --- a/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp +++ b/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp @@ -21,10 +21,10 @@ struct TStatisticsAggregator::TTxSchemeShardStats : public TTxBase { << ", stats size# " << stats.size()); NIceDb::TNiceDb db(txc.DB); - db.Table().Key(schemeShardId).Update( - NIceDb::TUpdate(stats)); + db.Table().Key(schemeShardId).Update( + NIceDb::TUpdate(stats)); - Self->BaseStats[schemeShardId] = stats; + Self->BaseStatistics[schemeShardId] = stats; if (!Self->EnableColumnStatistics) { return true; @@ -33,36 +33,39 @@ struct TStatisticsAggregator::TTxSchemeShardStats : public TTxBase { NKikimrStat::TSchemeShardStats statRecord; Y_PROTOBUF_SUPPRESS_NODISCARD statRecord.ParseFromString(stats); - auto& oldPathIds = Self->ScanTablesBySchemeShard[schemeShardId]; + auto& oldPathIds = Self->ScheduleTraversalsBySchemeShard[schemeShardId]; std::unordered_set newPathIds; for (auto& entry : statRecord.GetEntries()) { auto pathId = PathIdFromPathId(entry.GetPathId()); newPathIds.insert(pathId); if (oldPathIds.find(pathId) == oldPathIds.end()) { - TStatisticsAggregator::TScanTable scanTable; - scanTable.PathId = pathId; - scanTable.SchemeShardId = schemeShardId; - scanTable.LastUpdateTime = TInstant::MicroSeconds(0); - auto [it, _] = Self->ScanTables.emplace(pathId, scanTable); - Self->ScanTablesByTime.Add(&it->second); - - db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Update( - NIceDb::TUpdate(schemeShardId), - NIceDb::TUpdate(0)); + TStatisticsAggregator::TScheduleTraversal traversalTable; + traversalTable.PathId = pathId; + traversalTable.SchemeShardId = schemeShardId; + traversalTable.LastUpdateTime = TInstant::MicroSeconds(0); + traversalTable.IsColumnTable = entry.GetIsColumnTable(); + auto [it, _] = Self->ScheduleTraversals.emplace(pathId, traversalTable); + if (!Self->ScheduleTraversalsByTime.Has(&it->second)) { + Self->ScheduleTraversalsByTime.Add(&it->second); + } + db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate(schemeShardId), + NIceDb::TUpdate(0), + NIceDb::TUpdate(entry.GetIsColumnTable())); } } for (auto& pathId : oldPathIds) { if (newPathIds.find(pathId) == newPathIds.end()) { - auto it = Self->ScanTables.find(pathId); - if (it != Self->ScanTables.end()) { - if (Self->ScanTablesByTime.Has(&it->second)) { - Self->ScanTablesByTime.Remove(&it->second); + auto it = Self->ScheduleTraversals.find(pathId); + if (it != Self->ScheduleTraversals.end()) { + if (Self->ScheduleTraversalsByTime.Has(&it->second)) { + Self->ScheduleTraversalsByTime.Remove(&it->second); } - Self->ScanTables.erase(it); + Self->ScheduleTraversals.erase(it); } - db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); + db.Table().Key(pathId.OwnerId, pathId.LocalPathId).Delete(); } } @@ -73,6 +76,8 @@ struct TStatisticsAggregator::TTxSchemeShardStats : public TTxBase { void Complete(const TActorContext&) override { SA_LOG_D("[" << Self->TabletID() << "] TTxSchemeShardStats::Complete"); + + Self->IsSchemeshardSeen = true; } }; diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp new file mode 100644 index 000000000000..85107784bc83 --- /dev/null +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -0,0 +1,72 @@ +#include + +#include + +#include + +namespace NKikimr { +namespace NStat { + +struct TTableInfo { + std::vector ShardIds; + ui64 SaTabletId; + TPathId DomainKey; + TPathId PathId; +}; + +std::vector CreateDatabaseTables(TTestEnv& env, ui8 tableCount, ui8 shardCount) { + auto init = [&] () { + CreateDatabase(env, "Database"); + for (ui8 tableId = 1; tableId <= tableCount; tableId++) { + CreateColumnStoreTable(env, "Database", Sprintf("Table%u", tableId), shardCount); + } + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SimulateSleep(TDuration::Seconds(10)); + initThread.join(); + + std::vector ret; + for (ui8 tableId = 1; tableId <= tableCount; tableId++) { + TTableInfo tableInfo; + const TString path = Sprintf("/Root/Database/Table%u", tableId); + tableInfo.ShardIds = GetColumnTableShards(runtime, sender, path); + tableInfo.PathId = ResolvePathId(runtime, path, &tableInfo.DomainKey, &tableInfo.SaTabletId); + ret.emplace_back(tableInfo); + } + return ret; +} + +Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { + Y_UNIT_TEST(AnalyzeOneColumnTable) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + + AnalyzeTable(runtime, tableInfo.PathId, tableInfo.ShardIds[0]); + + Analyze(runtime, {tableInfo.PathId}, tableInfo.SaTabletId); + } + + Y_UNIT_TEST(AnalyzeAnalyzeOneColumnTableSpecificColumns) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + + Analyze(runtime, {{tableInfo.PathId, {1, 2}}}, tableInfo.SaTabletId); + } + + Y_UNIT_TEST(AnalyzeTwoColumnTables) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfos = CreateDatabaseTables(env, 2, 1); + + Analyze(runtime, {tableInfos[0].PathId, tableInfos[1].PathId}, tableInfos[0].SaTabletId); + } +} + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp new file mode 100644 index 000000000000..3f77cb85e612 --- /dev/null +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp @@ -0,0 +1,104 @@ +#include + +#include + +#include +#include +#include +#include + +#include + +namespace NKikimr { +namespace NStat { + +namespace { + + +} // namespace + +Y_UNIT_TEST_SUITE(AnalyzeDatashard) { + + Y_UNIT_TEST(AnalyzeOneTable) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateUniformTable(env, "Database", "Table"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + Analyze(runtime, {{pathId}}, saTabletId); + + ValidateCountMin(runtime, pathId); + } + + Y_UNIT_TEST(AnalyzeTwoTables) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateUniformTable(env, "Database", "Table1"); + CreateUniformTable(env, "Database", "Table2"); + }; + // TODO remove thread + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + // TODO remove sleep + runtime.SimulateSleep(TDuration::Seconds(30)); + + ui64 saTabletId1; + auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1", nullptr, &saTabletId1); + auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2"); + + Analyze(runtime, {pathId1, pathId2}, saTabletId1); + + ValidateCountMin(runtime, pathId1); + ValidateCountMin(runtime, pathId2); + } + + + Y_UNIT_TEST(DropTableNavigateError) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateUniformTable(env, "Database", "Table"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId = 0; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto init2 = [&] () { + DropTable(env, "Database", "Table"); + }; + std::thread init2Thread(init2); + + runtime.SimulateSleep(TDuration::Seconds(5)); + init2Thread.join(); + + Analyze(runtime, {pathId}, saTabletId); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + ValidateCountMinAbsense(runtime, pathId); + } +} + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp new file mode 100644 index 000000000000..ad7c926b78a0 --- /dev/null +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp @@ -0,0 +1,406 @@ +#include + +#include + +#include +#include +#include +#include + +#include + +namespace NKikimr { +namespace NStat { + +Y_UNIT_TEST_SUITE(TraverseColumnShard) { + + Y_UNIT_TEST(TraverseColumnTable) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(30)); + initThread.join(); + + auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeResolve) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId = 0; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto sender = runtime.AllocateEdgeActor(); + int observerCount = 0; + auto observer = runtime.AddObserver( + [&](TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) + { + if (observerCount++ == 2) { + RebootTablet(runtime, saTabletId, sender); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeReqDistribution) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId = 0; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto sender = runtime.AllocateEdgeActor(); + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvHive::TEvRequestTabletDistribution::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + RebootTablet(runtime, saTabletId, sender); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeAggregate) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId = 0; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto sender = runtime.AllocateEdgeActor(); + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvStatistics::TEvAggregateStatistics::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + RebootTablet(runtime, saTabletId, sender); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeSave) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId = 0; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto sender = runtime.AllocateEdgeActor(); + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + RebootTablet(runtime, saTabletId, sender); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableRebootSaTabletInAggregate) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + ui64 saTabletId = 0; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto sender = runtime.AllocateEdgeActor(); + int observerCount = 0; + auto observer = runtime.AddObserver( + [&](TEvStatistics::TEvStatisticsRequest::TPtr& ev) + { + if (observerCount++ == 5) { + RebootTablet(runtime, saTabletId, sender); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableHiveDistributionZeroNodes) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvHive::TEvResponseTabletDistribution::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + auto& record = ev->Get()->Record; + + NKikimrHive::TEvResponseTabletDistribution newRecord; + std::vector unknownTablets; + + for (auto& node : record.GetNodes()) { + auto* newNode = newRecord.AddNodes(); + newNode->SetNodeId(node.GetNodeId()); + int index = 0; + for (auto tabletId : node.GetTabletIds()) { + if (index < 7) { + newNode->AddTabletIds(tabletId); + } else { + unknownTablets.push_back(tabletId); + } + ++index; + } + } + auto* unknownNode = newRecord.AddNodes(); + unknownNode->SetNodeId(0); + for (auto tabletId : unknownTablets) { + unknownNode->AddTabletIds(tabletId); + } + + record.Swap(&newRecord); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableHiveDistributionAbsentNodes) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvHive::TEvResponseTabletDistribution::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + auto& record = ev->Get()->Record; + + NKikimrHive::TEvResponseTabletDistribution newRecord; + + for (auto& node : record.GetNodes()) { + auto* newNode = newRecord.AddNodes(); + newNode->SetNodeId(node.GetNodeId()); + int index = 0; + for (auto tabletId : node.GetTabletIds()) { + if (index < 7) { + newNode->AddTabletIds(tabletId); + } + ++index; + } + } + + record.Swap(&newRecord); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 10); + } + + Y_UNIT_TEST(TraverseColumnTableAggrStatUnavailableNode) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + auto& record = ev->Get()->Record; + + NKikimrStat::TEvAggregateStatisticsResponse newRecord; + newRecord.SetRound(record.GetRound()); + newRecord.MutableColumns()->Swap(record.MutableColumns()); + + auto* failedTablet = newRecord.AddFailedTablets(); + failedTablet->SetError(NKikimrStat::TEvAggregateStatisticsResponse::TYPE_UNAVAILABLE_NODE); + failedTablet->SetTabletId(72075186224037900); + failedTablet->SetNodeId(2); + + record.Swap(&newRecord); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(30)); + + auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 11); // 10 for first round, 1 for second + } + + Y_UNIT_TEST(TraverseColumnTableAggrStatNonLocalTablet) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateColumnStoreTable(env, "Database", "Table", 10); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + bool observerFirstExec = true; + auto observer = runtime.AddObserver( + [&](TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev) + { + if (observerFirstExec) { + observerFirstExec = false; + auto& record = ev->Get()->Record; + + NKikimrStat::TEvAggregateStatisticsResponse newRecord; + newRecord.SetRound(record.GetRound()); + newRecord.MutableColumns()->Swap(record.MutableColumns()); + + auto* failedTablet = newRecord.AddFailedTablets(); + failedTablet->SetError(NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET); + failedTablet->SetTabletId(72075186224037900); + failedTablet->SetNodeId(3); + + record.Swap(&newRecord); + } + }); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(probe, 11); // 10 for first round, 1 for second + } + +} + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp new file mode 100644 index 000000000000..747826dcac84 --- /dev/null +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp @@ -0,0 +1,161 @@ +#include + +#include + +#include +#include +#include +#include + +#include + +namespace NKikimr { +namespace NStat { + +namespace { + + +} // namespace + +Y_UNIT_TEST_SUITE(TraverseDatashard) { + + Y_UNIT_TEST(TraverseOneTable) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateUniformTable(env, "Database", "Table"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + ValidateCountMin(runtime, pathId); + } + + Y_UNIT_TEST(TraverseTwoTables) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateUniformTable(env, "Database", "Table1"); + CreateUniformTable(env, "Database", "Table2"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1"); + auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2"); + ValidateCountMin(runtime, pathId1); + ValidateCountMin(runtime, pathId2); + } + + Y_UNIT_TEST(TraverseOneTableServerless) { + TTestEnv env(1, 1); + + auto init = [&] () { + CreateDatabase(env, "Shared"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + TPathId domainKey; + ResolvePathId(runtime, "/Root/Shared", &domainKey); + + auto init2 = [&] () { + CreateServerlessDatabase(env, "Serverless", domainKey); + CreateUniformTable(env, "Serverless", "Table"); + }; + std::thread init2Thread(init2); + + runtime.SimulateSleep(TDuration::Seconds(5)); + init2Thread.join(); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + auto pathId = ResolvePathId(runtime, "/Root/Serverless/Table"); + ValidateCountMin(runtime, pathId); + } + + Y_UNIT_TEST(TraverseTwoTablesServerless) { + TTestEnv env(1, 1); + + auto init = [&] () { + CreateDatabase(env, "Shared"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + TPathId domainKey; + ResolvePathId(runtime, "/Root/Shared", &domainKey); + + auto init2 = [&] () { + CreateServerlessDatabase(env, "Serverless", domainKey); + CreateUniformTable(env, "Serverless", "Table1"); + CreateUniformTable(env, "Serverless", "Table2"); + }; + std::thread init2Thread(init2); + + runtime.SimulateSleep(TDuration::Seconds(5)); + init2Thread.join(); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + auto pathId1 = ResolvePathId(runtime, "/Root/Serverless/Table1"); + auto pathId2 = ResolvePathId(runtime, "/Root/Serverless/Table2"); + ValidateCountMin(runtime, pathId1); + ValidateCountMin(runtime, pathId2); + } + + Y_UNIT_TEST(TraverseTwoTablesTwoServerlessDbs) { + TTestEnv env(1, 1); + + auto init = [&] () { + CreateDatabase(env, "Shared"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(5)); + initThread.join(); + + TPathId domainKey; + ResolvePathId(runtime, "/Root/Shared", &domainKey); + + auto init2 = [&] () { + CreateServerlessDatabase(env, "Serverless1", domainKey); + CreateServerlessDatabase(env, "Serverless2", domainKey); + CreateUniformTable(env, "Serverless1", "Table1"); + CreateUniformTable(env, "Serverless2", "Table2"); + }; + std::thread init2Thread(init2); + + runtime.SimulateSleep(TDuration::Seconds(5)); + init2Thread.join(); + + runtime.SimulateSleep(TDuration::Seconds(60)); + + auto pathId1 = ResolvePathId(runtime, "/Root/Serverless1/Table1"); + auto pathId2 = ResolvePathId(runtime, "/Root/Serverless2/Table2"); + ValidateCountMin(runtime, pathId1); + ValidateCountMin(runtime, pathId2); + } + +} + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/aggregator/ut/ya.make b/ydb/core/statistics/aggregator/ut/ya.make new file mode 100644 index 000000000000..830728bf21fd --- /dev/null +++ b/ydb/core/statistics/aggregator/ut/ya.make @@ -0,0 +1,30 @@ +UNITTEST_FOR(ydb/core/statistics/aggregator) + +FORK_SUBTESTS() + +IF (WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +YQL_LAST_ABI_VERSION() + +PEERDIR( + library/cpp/testing/unittest + ydb/core/protos + ydb/core/testlib/default + ydb/core/statistics/ut_common +) + +SRCS( + ut_analyze_datashard.cpp + ut_analyze_columnshard.cpp + ut_traverse_datashard.cpp + ut_traverse_columnshard.cpp +) + +END() diff --git a/ydb/core/statistics/aggregator/ya.make b/ydb/core/statistics/aggregator/ya.make index 3223cd9e4d9c..f4003bcdade1 100644 --- a/ydb/core/statistics/aggregator/ya.make +++ b/ydb/core/statistics/aggregator/ya.make @@ -7,17 +7,19 @@ SRCS( aggregator_impl.cpp schema.h schema.cpp + tx_ack_timeout.cpp + tx_aggr_stat_response.cpp + tx_analyze_table.cpp tx_configure.cpp - tx_delete_query_response.cpp + tx_datashard_scan_response.cpp + tx_finish_trasersal.cpp tx_init.cpp tx_init_schema.cpp tx_navigate.cpp tx_resolve.cpp - tx_save_query_response.cpp - tx_scan_table.cpp - tx_schedule_scan.cpp + tx_response_tablet_distribution.cpp + tx_schedule_traversal.cpp tx_schemeshard_stats.cpp - tx_statistics_scan_response.cpp ) PEERDIR( @@ -26,9 +28,14 @@ PEERDIR( ydb/core/protos ydb/core/tablet ydb/core/tablet_flat + ydb/core/statistics/database ydb/library/minsketch ) YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/statistics/save_load_stats.cpp b/ydb/core/statistics/database/database.cpp similarity index 99% rename from ydb/core/statistics/save_load_stats.cpp rename to ydb/core/statistics/database/database.cpp index 082b9f58b5bb..f3f591cced1c 100644 --- a/ydb/core/statistics/save_load_stats.cpp +++ b/ydb/core/statistics/database/database.cpp @@ -1,6 +1,6 @@ -#include "save_load_stats.h" +#include "database.h" -#include "events.h" +#include #include #include @@ -74,7 +74,6 @@ class TSaveStatisticsQuery : public NKikimr::TQueryBase { const ui64 StatType; const std::vector ColumnTags; const std::vector Data; - public: TSaveStatisticsQuery(const TPathId& pathId, ui64 statType, std::vector&& columnTags, std::vector&& data) diff --git a/ydb/core/statistics/save_load_stats.h b/ydb/core/statistics/database/database.h similarity index 100% rename from ydb/core/statistics/save_load_stats.h rename to ydb/core/statistics/database/database.h diff --git a/ydb/core/statistics/ut/ut_save_load_stats.cpp b/ydb/core/statistics/database/ut/ut_database.cpp similarity index 97% rename from ydb/core/statistics/ut/ut_save_load_stats.cpp rename to ydb/core/statistics/database/ut/ut_database.cpp index 50510b3d9a99..cb097b43283e 100644 --- a/ydb/core/statistics/ut/ut_save_load_stats.cpp +++ b/ydb/core/statistics/database/ut/ut_database.cpp @@ -1,7 +1,7 @@ -#include "ut_common.h" +#include #include -#include +#include #include diff --git a/ydb/core/statistics/ut/ya.make b/ydb/core/statistics/database/ut/ya.make similarity index 64% rename from ydb/core/statistics/ut/ya.make rename to ydb/core/statistics/database/ut/ya.make index d2f04f691779..fb0e5fcb6b9c 100644 --- a/ydb/core/statistics/ut/ya.make +++ b/ydb/core/statistics/database/ut/ya.make @@ -1,4 +1,4 @@ -UNITTEST_FOR(ydb/core/statistics) +UNITTEST_FOR(ydb/core/statistics/database) FORK_SUBTESTS() @@ -15,15 +15,13 @@ YQL_LAST_ABI_VERSION() PEERDIR( library/cpp/testing/unittest + ydb/core/protos ydb/core/testlib/default + ydb/core/statistics/ut_common ) SRCS( - ut_common.h - ut_common.cpp - ut_aggregator.cpp - ut_statistics.cpp - ut_save_load_stats.cpp + ut_database.cpp ) END() diff --git a/ydb/core/statistics/database/ya.make b/ydb/core/statistics/database/ya.make new file mode 100644 index 000000000000..b88855356fab --- /dev/null +++ b/ydb/core/statistics/database/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +SRCS( + database.h + database.cpp +) + +PEERDIR( + ydb/core/base + ydb/core/engine/minikql + ydb/core/protos + ydb/core/tablet + ydb/core/tablet_flat + ydb/library/minsketch +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/statistics/events.h b/ydb/core/statistics/events.h index 021977ab19c1..ca23149140fa 100644 --- a/ydb/core/statistics/events.h +++ b/ydb/core/statistics/events.h @@ -64,23 +64,54 @@ struct TEvStatistics { EvStatTableCreationResponse, EvSaveStatisticsQueryResponse, + EvDeleteStatisticsQueryResponse, EvLoadStatisticsQueryResponse, - EvScanTable, - EvScanTableResponse, - - EvDeleteStatisticsQueryResponse, + EvAnalyze, + EvAnalyzeResponse, + EvAnalyzeStatus, + EvAnalyzeStatusResponse, - EvScanTableAccepted, - EvGetScanStatus, - EvGetScanStatusResponse, + EvAnalyzeTable, + EvAnalyzeTableResponse, EvStatisticsRequest, EvStatisticsResponse, + EvAggregateStatistics, + EvAggregateStatisticsResponse, + EvAggregateKeepAlive, + EvAggregateKeepAliveAck, + + EvFinishTraversal, + EvEnd }; + struct TEvAggregateKeepAlive : public TEventPB< + TEvAggregateKeepAlive, + NKikimrStat::TEvAggregateKeepAlive, + EvAggregateKeepAlive> + {}; + + struct TEvAggregateKeepAliveAck : public TEventPB< + TEvAggregateKeepAliveAck, + NKikimrStat::TEvAggregateKeepAliveAck, + EvAggregateKeepAliveAck> + {}; + + struct TEvAggregateStatistics : public TEventPB< + TEvAggregateStatistics, + NKikimrStat::TEvAggregateStatistics, + EvAggregateStatistics> + {}; + + struct TEvAggregateStatisticsResponse : public TEventPB< + TEvAggregateStatisticsResponse, + NKikimrStat::TEvAggregateStatisticsResponse, + EvAggregateStatisticsResponse> + {}; + struct TEvGetStatistics : public TEventLocal { EStatType StatType; std::vector StatRequests; @@ -175,36 +206,42 @@ struct TEvStatistics { bool Success = true; }; - struct TEvScanTable : public TEventPB< - TEvScanTable, - NKikimrStat::TEvScanTable, - EvScanTable> + struct TEvAnalyze : public TEventPB< + TEvAnalyze, + NKikimrStat::TEvAnalyze, + EvAnalyze> {}; - struct TEvScanTableAccepted : public TEventPB< - TEvScanTableAccepted, - NKikimrStat::TEvScanTableAccepted, - EvScanTableAccepted> + struct TEvAnalyzeResponse : public TEventPB< + TEvAnalyzeResponse, + NKikimrStat::TEvAnalyzeResponse, + EvAnalyzeResponse> {}; - struct TEvScanTableResponse : public TEventPB< - TEvScanTableResponse, - NKikimrStat::TEvScanTableResponse, - EvScanTableResponse> + struct TEvAnalyzeStatus : public TEventPB< + TEvAnalyzeStatus, + NKikimrStat::TEvAnalyzeStatus, + EvAnalyzeStatus> {}; - struct TEvGetScanStatus : public TEventPB< - TEvGetScanStatus, - NKikimrStat::TEvGetScanStatus, - EvGetScanStatus> + struct TEvAnalyzeStatusResponse : public TEventPB< + TEvAnalyzeStatusResponse, + NKikimrStat::TEvAnalyzeStatusResponse, + EvAnalyzeStatusResponse> {}; - struct TEvGetScanStatusResponse : public TEventPB< - TEvGetScanStatusResponse, - NKikimrStat::TEvGetScanStatusResponse, - EvGetScanStatusResponse> + struct TEvAnalyzeTable : public TEventPB< + TEvAnalyzeTable, + NKikimrStat::TEvAnalyzeTable, + EvAnalyzeTable> {}; + struct TEvAnalyzeTableResponse : public TEventPB< + TEvAnalyzeTableResponse, + NKikimrStat::TEvAnalyzeTableResponse, + EvAnalyzeTableResponse> + {}; + struct TEvStatisticsRequest : public TEventPB< TEvStatisticsRequest, NKikimrStat::TEvStatisticsRequest, diff --git a/ydb/core/statistics/service/http_request.cpp b/ydb/core/statistics/service/http_request.cpp new file mode 100644 index 000000000000..6e7b28081f7d --- /dev/null +++ b/ydb/core/statistics/service/http_request.cpp @@ -0,0 +1,163 @@ +#include "http_request.h" + + +#include +#include +#include +#include + +namespace NKikimr { +namespace NStat { + +THttpRequest::THttpRequest(EType type, const TString& path, TActorId replyToActorId) + : Type(type) + , Path(path) + , ReplyToActorId(replyToActorId) +{} + +void THttpRequest::Bootstrap() { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto navigate = std::make_unique(); + auto& entry = navigate->ResultSet.emplace_back(); + entry.Path = SplitPath(Path); + entry.Operation = TNavigate::EOp::OpTable; + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + navigate->Cookie = FirstRoundCookie; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + + Become(&THttpRequest::StateWork); +} + +void THttpRequest::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + std::unique_ptr navigate(ev->Get()->Request.Release()); + Y_ABORT_UNLESS(navigate->ResultSet.size() == 1); + auto& entry = navigate->ResultSet.front(); + + if (navigate->Cookie == SecondRoundCookie) { + if (entry.Status != TNavigate::EStatus::Ok) { + HttpReply("Internal error"); + return; + } + if (entry.DomainInfo->Params.HasStatisticsAggregator()) { + StatisticsAggregatorId = entry.DomainInfo->Params.GetStatisticsAggregator(); + } + ResolveSuccess(); + return; + } + + if (entry.Status != TNavigate::EStatus::Ok) { + switch (entry.Status) { + case TNavigate::EStatus::PathErrorUnknown: + HttpReply("Path does not exist"); + return; + case TNavigate::EStatus::PathNotPath: + HttpReply("Invalid path"); + return; + case TNavigate::EStatus::PathNotTable: + HttpReply("Path is not a table"); + return; + default: + HttpReply("Internal error"); + return; + } + } + + PathId = entry.TableId.PathId; + + auto& domainInfo = entry.DomainInfo; + ui64 aggregatorId = 0; + if (domainInfo->Params.HasStatisticsAggregator()) { + aggregatorId = domainInfo->Params.GetStatisticsAggregator(); + } + bool isServerless = domainInfo->IsServerless(); + TPathId domainKey = domainInfo->DomainKey; + TPathId resourcesDomainKey = domainInfo->ResourcesDomainKey; + + auto navigateDomainKey = [this] (TPathId domainKey) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto navigate = std::make_unique(); + auto& entry = navigate->ResultSet.emplace_back(); + entry.TableId = TTableId(domainKey.OwnerId, domainKey.LocalPathId); + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + entry.RedirectRequired = false; + navigate->Cookie = SecondRoundCookie; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + }; + + if (!isServerless) { + if (aggregatorId) { + StatisticsAggregatorId = aggregatorId; + ResolveSuccess(); + } else { + navigateDomainKey(domainKey); + } + } else { + navigateDomainKey(resourcesDomainKey); + } +} + +void THttpRequest::Handle(TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + switch (record.GetStatus()) { + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_UNSPECIFIED: + HttpReply("Status is unspecified"); + break; + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION: + HttpReply("No analyze operation"); + break; + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED: + HttpReply("Analyze is enqueued"); + break; + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS: + HttpReply("Analyze is in progress"); + break; + } +} + +void THttpRequest::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { + HttpReply("Delivery problem"); +} + +void THttpRequest::ResolveSuccess() { + if (StatisticsAggregatorId == 0) { + HttpReply("No statistics aggregator"); + return; + } + + if (Type == ANALYZE) { + auto analyze = std::make_unique(); + auto& record = analyze->Record; + PathIdFromPathId(PathId, record.AddTables()->MutablePathId()); + + Send(MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(analyze.release(), StatisticsAggregatorId, true)); + + HttpReply("Analyze sent"); + } else { + auto getStatus = std::make_unique(); + auto& record = getStatus->Record; + PathIdFromPathId(PathId, record.MutablePathId()); + + Send(MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(getStatus.release(), StatisticsAggregatorId, true)); + } +} + +void THttpRequest::HttpReply(const TString& msg) { + Send(ReplyToActorId, new NMon::TEvHttpInfoRes(msg)); + PassAway(); +} + +void THttpRequest::PassAway() { + Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + TBase::PassAway(); +} + + + +} // NStat +} // NKikimr \ No newline at end of file diff --git a/ydb/core/statistics/service/http_request.h b/ydb/core/statistics/service/http_request.h new file mode 100644 index 000000000000..874725440898 --- /dev/null +++ b/ydb/core/statistics/service/http_request.h @@ -0,0 +1,65 @@ +#include + +#include +#include + +#include +#include +#include + +namespace NKikimr { +namespace NStat { + +class THttpRequest : public NActors::TActorBootstrapped { +public: + using TBase = TActorBootstrapped; + + static constexpr auto ActorActivityType() { + return NKikimrServices::TActivity::STAT_SERVICE_HTTP_REQUEST; + } + + void Bootstrap(); + + enum EType { + ANALYZE, + STATUS + }; + + THttpRequest(EType type, const TString& path, TActorId replyToActorId); + +private: + STFUNC(StateWork) { + switch(ev->GetTypeRewrite()) { + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + hFunc(TEvStatistics::TEvAnalyzeStatusResponse, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + IgnoreFunc(TEvStatistics::TEvAnalyzeResponse); + default: + LOG_CRIT_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "NStat::THttpRequest: unexpected event# " << ev->GetTypeRewrite()); + } + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev); + void Handle(TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev); + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&); + + void ResolveSuccess(); + void HttpReply(const TString& msg); + + void PassAway(); + +private: + const EType Type; + const TString Path; + const TActorId ReplyToActorId; + + TPathId PathId; + ui64 StatisticsAggregatorId = 0; + + static const ui64 FirstRoundCookie = 1; + static const ui64 SecondRoundCookie = 2; +}; + +} // NStat +} // NKikimr \ No newline at end of file diff --git a/ydb/core/statistics/service/service.cpp b/ydb/core/statistics/service/service.cpp new file mode 100644 index 000000000000..d828bc550b07 --- /dev/null +++ b/ydb/core/statistics/service/service.cpp @@ -0,0 +1,50 @@ +#include "service.h" + + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace NKikimr { +namespace NStat { + +static constexpr TDuration DefaultAggregateKeepAlivePeriod = TDuration::MilliSeconds(500); +static constexpr TDuration DefaultAggregateKeepAliveTimeout = TDuration::Seconds(3); +static constexpr TDuration DefaultAggregateKeepAliveAckTimeout = TDuration::Seconds(3); +static constexpr size_t DefaultMaxInFlightTabletRequests = 5; +static constexpr size_t DefaultFanOutFactor = 5; + + + +TStatServiceSettings::TStatServiceSettings() + : AggregateKeepAlivePeriod(DefaultAggregateKeepAlivePeriod) + , AggregateKeepAliveTimeout(DefaultAggregateKeepAliveTimeout) + , AggregateKeepAliveAckTimeout(DefaultAggregateKeepAliveAckTimeout) + , MaxInFlightTabletRequests(DefaultMaxInFlightTabletRequests) + , FanOutFactor(DefaultFanOutFactor) +{} + +NActors::TActorId MakeStatServiceID(ui32 node) { + const char x[12] = "StatService"; + return NActors::TActorId(node, TStringBuf(x, 12)); +} + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/service/service.h b/ydb/core/statistics/service/service.h new file mode 100644 index 000000000000..abe356fe0271 --- /dev/null +++ b/ydb/core/statistics/service/service.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +namespace NKikimr::NStat { + +struct TStatServiceSettings { + TDuration AggregateKeepAlivePeriod; + TDuration AggregateKeepAliveTimeout; + TDuration AggregateKeepAliveAckTimeout; + size_t MaxInFlightTabletRequests; + size_t FanOutFactor; + + TStatServiceSettings(); + + TStatServiceSettings& SetAggregateKeepAlivePeriod(const TDuration& val) { + AggregateKeepAlivePeriod = val; + return *this; + } + + TStatServiceSettings& SetAggregateKeepAliveTimeout(const TDuration& val) { + AggregateKeepAliveTimeout = val; + return *this; + } + + TStatServiceSettings& SetAggregateKeepAliveAckTimeout(const TDuration& val) { + AggregateKeepAliveAckTimeout = val; + return *this; + } + + TStatServiceSettings& SetMaxInFlightTabletRequests(size_t val) { + MaxInFlightTabletRequests = val; + return *this; + } + + TStatServiceSettings& SetFanOutFactor(size_t val) { + FanOutFactor = val; + return *this; + } +}; + +NActors::TActorId MakeStatServiceID(ui32 node); + +THolder CreateStatService(const TStatServiceSettings& settings = TStatServiceSettings()); + +} // NKikimr::NStat diff --git a/ydb/core/statistics/stat_service.cpp b/ydb/core/statistics/service/service_impl.cpp similarity index 52% rename from ydb/core/statistics/stat_service.cpp rename to ydb/core/statistics/service/service_impl.cpp index 548a80c89028..113e4466bbe2 100644 --- a/ydb/core/statistics/stat_service.cpp +++ b/ydb/core/statistics/service/service_impl.cpp @@ -1,6 +1,8 @@ -#include "stat_service.h" -#include "events.h" -#include "save_load_stats.h" +#include "service.h" +#include "http_request.h" + +#include +#include #include #include @@ -11,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -21,211 +25,135 @@ namespace NKikimr { namespace NStat { -class THttpRequest : public TActorBootstrapped { -public: - using TBase = TActorBootstrapped; - - static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::STAT_SERVICE_HTTP_REQUEST; - } - - void Bootstrap() { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - auto navigate = std::make_unique(); - auto& entry = navigate->ResultSet.emplace_back(); - entry.Path = SplitPath(Path); - entry.Operation = TNavigate::EOp::OpTable; - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - navigate->Cookie = FirstRoundCookie; - - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); - - Become(&THttpRequest::StateWork); - } - - STFUNC(StateWork) { - switch(ev->GetTypeRewrite()) { - hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); - hFunc(TEvStatistics::TEvScanTableAccepted, Handle); - hFunc(TEvStatistics::TEvGetScanStatusResponse, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); - IgnoreFunc(TEvStatistics::TEvScanTableResponse); - default: - LOG_CRIT_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "NStat::THttpRequest: unexpected event# " << ev->GetTypeRewrite()); - } - } - enum EType { - ANALYZE, - STATUS - }; +struct TAggregationStatistics { + using TColumnsStatistics = ::google::protobuf::RepeatedPtrField<::NKikimrStat::TColumnStatistics>; - THttpRequest(EType type, const TString& path, TActorId replyToActorId) - : Type(type) - , Path(path) - , ReplyToActorId(replyToActorId) + TAggregationStatistics(size_t nodesCount) + : Nodes(nodesCount) {} -private: - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - std::unique_ptr navigate(ev->Get()->Request.Release()); - Y_ABORT_UNLESS(navigate->ResultSet.size() == 1); - auto& entry = navigate->ResultSet.front(); + struct TFailedTablet { + using EErrorType = NKikimrStat::TEvAggregateStatisticsResponse::EErrorType; - if (navigate->Cookie == SecondRoundCookie) { - if (entry.Status != TNavigate::EStatus::Ok) { - HttpReply("Internal error"); - return; - } - if (entry.DomainInfo->Params.HasStatisticsAggregator()) { - StatisticsAggregatorId = entry.DomainInfo->Params.GetStatisticsAggregator(); - } - ResolveSuccess(); - return; - } + ui64 TabletId; + ui32 NodeId; + EErrorType Error; - if (entry.Status != TNavigate::EStatus::Ok) { - switch (entry.Status) { - case TNavigate::EStatus::PathErrorUnknown: - HttpReply("Path does not exist"); - return; - case TNavigate::EStatus::PathNotPath: - HttpReply("Invalid path"); - return; - case TNavigate::EStatus::PathNotTable: - HttpReply("Path is not a table"); - return; - default: - HttpReply("Internal error"); - return; - } - } - - PathId = entry.TableId.PathId; - - auto& domainInfo = entry.DomainInfo; - ui64 aggregatorId = 0; - if (domainInfo->Params.HasStatisticsAggregator()) { - aggregatorId = domainInfo->Params.GetStatisticsAggregator(); - } - bool isServerless = domainInfo->IsServerless(); - TPathId domainKey = domainInfo->DomainKey; - TPathId resourcesDomainKey = domainInfo->ResourcesDomainKey; + TFailedTablet(ui64 tabletId, ui32 nodeId, EErrorType error) + : TabletId(tabletId) + , NodeId(nodeId) + , Error(error) {} + }; - auto navigateDomainKey = [this] (TPathId domainKey) { - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - auto navigate = std::make_unique(); - auto& entry = navigate->ResultSet.emplace_back(); - entry.TableId = TTableId(domainKey.OwnerId, domainKey.LocalPathId); - entry.Operation = TNavigate::EOp::OpPath; - entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; - entry.RedirectRequired = false; - navigate->Cookie = SecondRoundCookie; + struct TTablets { + ui32 NodeId; + std::vector Ids; + }; - Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + struct TNode { + enum class EStatus: ui8 { + None, + Processing, + Processed, + Unavailable }; - if (!isServerless) { - if (aggregatorId) { - StatisticsAggregatorId = aggregatorId; - ResolveSuccess(); - } else { - navigateDomainKey(domainKey); - } - } else { - navigateDomainKey(resourcesDomainKey); - } - } - - void Handle(TEvStatistics::TEvScanTableAccepted::TPtr&) { - HttpReply("Scan accepted"); - } + ui64 LastHeartbeat{ 0 }; + std::vector Tablets; + TActorId Actor; + EStatus Status{ EStatus::None }; + }; - void Handle(TEvStatistics::TEvGetScanStatusResponse::TPtr& ev) { - auto& record = ev->Get()->Record; - switch (record.GetStatus()) { - case NKikimrStat::TEvGetScanStatusResponse::NO_OPERATION: - HttpReply("No scan operation"); - break; - case NKikimrStat::TEvGetScanStatusResponse::ENQUEUED: - HttpReply("Scan is enqueued"); - break; - case NKikimrStat::TEvGetScanStatusResponse::IN_PROGRESS: - HttpReply("Scan is in progress"); - break; - } - } + struct TLocalTablets { + size_t NextTablet{ 0 }; + ui32 InFlight{ 0 }; + std::vector Ids; + std::unordered_set FinishedTablets; + }; - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr&) { - HttpReply("Delivery problem"); - } + struct ColumnStatistics { + std::unique_ptr Statistics; + ui32 ContainedInResponse{ 0 }; + }; - void ResolveSuccess() { - if (StatisticsAggregatorId == 0) { - HttpReply("No statistics aggregator"); - return; - } + ui64 Round{ 0 }; + ui64 Cookie{ 0 }; + TPathId PathId; + ui64 LastAckHeartbeat{ 0 }; + TActorId ParentNode; + std::vector Nodes; + size_t PprocessedNodes{ 0 }; - if (Type == ANALYZE) { - auto scanTable = std::make_unique(); - auto& record = scanTable->Record; - PathIdFromPathId(PathId, record.MutablePathId()); + std::unordered_map CountMinSketches; + ui32 TotalStatisticsResponse{ 0 }; - Send(MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward(scanTable.release(), StatisticsAggregatorId, true)); - } else { - auto getStatus = std::make_unique(); - auto& record = getStatus->Record; - PathIdFromPathId(PathId, record.MutablePathId()); + std::vector ColumnTags; + TLocalTablets LocalTablets; + std::vector FailedTablets; - Send(MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward(getStatus.release(), StatisticsAggregatorId, true)); - } + bool IsCompleted() const noexcept { + return PprocessedNodes == Nodes.size() && LocalTablets.InFlight == 0; } - void HttpReply(const TString& msg) { - Send(ReplyToActorId, new NMon::TEvHttpInfoRes(msg)); - PassAway(); - } - - void PassAway() { - Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - TBase::PassAway(); + TNode* GetProcessingChildNode(ui32 nodeId) { + for (size_t i = 0; i < Nodes.size(); ++i) { + if (Nodes[i].Actor.NodeId() == nodeId) { + return Nodes[i].Status == TAggregationStatistics::TNode::EStatus::Processing + ? &Nodes[i] : nullptr; + } + } + LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Child node with the specified id was not found"); + return nullptr; } - -private: - const EType Type; - const TString Path; - const TActorId ReplyToActorId; - - TPathId PathId; - ui64 StatisticsAggregatorId = 0; - - static const ui64 FirstRoundCookie = 1; - static const ui64 SecondRoundCookie = 2; }; - class TStatService : public TActorBootstrapped { public: using TBase = TActorBootstrapped; + TStatService(const TStatServiceSettings& settings) + : Settings(settings) + , AggregationStatistics(settings.FanOutFactor) + {} + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::STAT_SERVICE; } struct TEvPrivate { enum EEv { - EvRequestTimeout = EventSpaceBegin(TEvents::ES_PRIVATE), + EvRequestTimeout = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + EvDispatchKeepAlive, + EvKeepAliveTimeout, + EvKeepAliveAckTimeout, EvEnd }; - struct TEvRequestTimeout : public TEventLocal { + struct TEvRequestTimeout : public NActors::TEventLocal { std::unordered_set NeedSchemeShards; - TActorId PipeClientId; + NActors::TActorId PipeClientId; + }; + + struct TEvDispatchKeepAlive: public NActors::TEventLocal { + TEvDispatchKeepAlive(ui64 round): Round(round) {} + + ui64 Round; + }; + + struct TEvKeepAliveAckTimeout: public NActors::TEventLocal { + TEvKeepAliveAckTimeout(ui64 round): Round(round) {} + + ui64 Round; + }; + + struct TEvKeepAliveTimeout: public NActors::TEventLocal { + TEvKeepAliveTimeout(ui64 round, ui32 nodeId): Round(round), NodeId(nodeId) {} + + ui64 Round; + ui32 NodeId; }; }; @@ -254,17 +182,29 @@ class TStatService : public TActorBootstrapped { hFunc(TEvStatistics::TEvGetStatistics, Handle); hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); hFunc(TEvStatistics::TEvPropagateStatistics, Handle); + hFunc(TEvStatistics::TEvAggregateStatistics, Handle); IgnoreFunc(TEvStatistics::TEvPropagateStatisticsResponse); hFunc(TEvTabletPipe::TEvClientConnected, Handle); hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); hFunc(TEvStatistics::TEvStatisticsIsDisabled, Handle); hFunc(TEvStatistics::TEvLoadStatisticsQueryResponse, Handle); hFunc(TEvPrivate::TEvRequestTimeout, Handle); + + hFunc(TEvStatistics::TEvAggregateKeepAliveAck, Handle); + hFunc(TEvPrivate::TEvKeepAliveAckTimeout, Handle); + hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); + hFunc(TEvPrivate::TEvDispatchKeepAlive, Handle); + hFunc(TEvPrivate::TEvKeepAliveTimeout, Handle); + hFunc(TEvPipeCache::TEvGetTabletNodeResult, Handle); + hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + hFunc(TEvStatistics::TEvStatisticsResponse, Handle); + hFunc(TEvStatistics::TEvAggregateStatisticsResponse, Handle); + hFunc(NMon::TEvHttpInfo, Handle); cFunc(TEvents::TEvPoison::EventType, PassAway); default: LOG_CRIT_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "NStat::TStatService: unexpected event# " << ev->GetTypeRewrite()); + "NStat::TStatService: unexpected event# " << ev->GetTypeRewrite() << " " << ev->ToString()); } } @@ -289,6 +229,497 @@ class TStatService : public TActorBootstrapped { Send(ev->Sender, response.release(), 0, ev->Cookie); } + bool IsNotCurrentRound(ui64 round) { + if (round != AggregationStatistics.Round) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Event round " << round << " is different from the current " << AggregationStatistics.Round); + return true; + } + return false; + } + + // + // returns true if the tablet processing has not been completed yet + // + bool OnTabletFinished(ui64 tabletId) { + auto& localTablets = AggregationStatistics.LocalTablets; + auto isFinished = !localTablets.FinishedTablets.emplace(tabletId).second; + if (isFinished) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "OnTabletFinished: table " << tabletId << " has already been processed"); + return false; + } + + --localTablets.InFlight; + return true; + } + + void OnAggregateStatisticsFinished() { + SendAggregateStatisticsResponse(); + + Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + + TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); + std::swap(AggregationStatistics, aggregationStatistics); + } + + void SendRequestToNextTablet() { + auto& localTablets = AggregationStatistics.LocalTablets; + if (localTablets.NextTablet >= localTablets.Ids.size()) { + return; + } + + const auto tabletId = localTablets.Ids[localTablets.NextTablet]; + ++localTablets.NextTablet; + ++localTablets.InFlight; + Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvGetTabletNode(tabletId), 0, AggregationStatistics.Round); + } + + void Handle(TEvPipeCache::TEvGetTabletNodeResult::TPtr& ev) { + const auto msg = ev->Get(); + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Received TEvGetTabletNodeResult NodeId: " << msg->NodeId << ", TabletId: " << msg->TabletId); + + const auto round = ev->Cookie; + if (IsNotCurrentRound(round)) { + return; + } + + const auto currentNodeId = ev->Recipient.NodeId(); + + // there is no need for retries, as the tablet must be local + // no problems are expected in resolving + if (currentNodeId != msg->NodeId) { + const auto tabletFinished = OnTabletFinished(msg->TabletId); + Y_ABORT_UNLESS(tabletFinished); + + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Tablet is not local. Table node: " << msg->NodeId << ", current node: " << ev->Recipient.NodeId()); + + const auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; + AggregationStatistics.FailedTablets.emplace_back(msg->TabletId, msg->NodeId, error); + + SendRequestToNextTablet(); + + if (AggregationStatistics.IsCompleted()) { + OnAggregateStatisticsFinished(); + } + + return; + } + + auto request = std::make_unique(); + auto& record = request->Record; + record.MutableTypes()->Add(NKikimr::NStat::COUNT_MIN_SKETCH); + + auto* path = record.MutableTable()->MutablePathId(); + path->SetOwnerId(AggregationStatistics.PathId.OwnerId); + path->SetLocalId(AggregationStatistics.PathId.LocalPathId); + + auto* columnTags = record.MutableTable()->MutableColumnTags(); + for (const auto& tag : AggregationStatistics.ColumnTags) { + columnTags->Add(tag); + } + + Send(MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(request.release(), msg->TabletId, true, round), + IEventHandle::FlagTrackDelivery, round); + } + + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Received TEvDeliveryProblem TabletId: " << ev->Get()->TabletId); + + const auto round = ev->Cookie; + const auto tabletId = ev->Get()->TabletId; + if (IsNotCurrentRound(round) + || !OnTabletFinished(tabletId)) { + return; + } + + const auto nodeId = ev->Recipient.NodeId(); + const auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; + AggregationStatistics.FailedTablets.emplace_back(tabletId, nodeId, error); + + SendRequestToNextTablet(); + + if (AggregationStatistics.IsCompleted()) { + OnAggregateStatisticsFinished(); + } + } + + void AggregateStatistics(const TAggregationStatistics::TColumnsStatistics& columnsStatistics) { + ++AggregationStatistics.TotalStatisticsResponse; + + for (const auto& column : columnsStatistics) { + const auto tag = column.GetTag(); + + for (auto& statistic : column.GetStatistics()) { + if (statistic.GetType() == NKikimr::NStat::COUNT_MIN_SKETCH) { + auto data = statistic.GetData().Data(); + auto sketch = reinterpret_cast(data); + auto& current = AggregationStatistics.CountMinSketches[tag]; + + if (current.Statistics == nullptr) { + current.Statistics.reset(TCountMinSketch::Create()); + } + + ++current.ContainedInResponse; + *current.Statistics += *sketch; + } + } + } + } + + void Handle(TEvStatistics::TEvStatisticsResponse::TPtr& ev) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Received TEvStatisticsResponse TabletId: " << ev->Get()->Record.GetShardTabletId()); + + const auto round = ev->Cookie; + const auto& record = ev->Get()->Record; + if (IsNotCurrentRound(round) + || !OnTabletFinished(record.GetShardTabletId())) { + return; + } + + AggregateStatistics(record.GetColumns()); + + SendRequestToNextTablet(); + + if (AggregationStatistics.IsCompleted()) { + OnAggregateStatisticsFinished(); + } + } + + void Handle(TEvStatistics::TEvAggregateKeepAliveAck::TPtr& ev) { + const auto& record = ev->Get()->Record; + const auto round = record.GetRound(); + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvAggregateKeepAliveAck"); + return; + } + + AggregationStatistics.LastAckHeartbeat = GetCycleCountFast(); + } + + void Handle(TEvPrivate::TEvKeepAliveAckTimeout::TPtr& ev) { + const auto round = ev->Get()->Round; + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvKeepAliveAckTimeout"); + return; + } + + const auto maxDuration = DurationToCycles(Settings.AggregateKeepAliveAckTimeout); + const auto deadline = AggregationStatistics.LastAckHeartbeat + maxDuration; + const auto now = GetCycleCountFast(); + + if (deadline >= now) { + Schedule(Settings.AggregateKeepAliveAckTimeout, new TEvPrivate::TEvKeepAliveAckTimeout(round)); + return; + } + + // the parent node is unavailable + // invalidate the subtree with the root in the current node + LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Parent node " << AggregationStatistics.ParentNode.NodeId() << " is unavailable"); + + TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); + std::swap(AggregationStatistics, aggregationStatistics); + + Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + } + + void Handle(TEvPrivate::TEvDispatchKeepAlive::TPtr& ev) { + const auto round = ev->Get()->Round; + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvDispatchKeepAlive"); + return; + } + + auto keepAlive = std::make_unique(); + keepAlive->Record.SetRound(round); + Send(AggregationStatistics.ParentNode, keepAlive.release()); + Schedule(Settings.AggregateKeepAlivePeriod, new TEvPrivate::TEvDispatchKeepAlive(round)); + } + + void Handle(TEvPrivate::TEvKeepAliveTimeout::TPtr& ev) { + const auto round = ev->Get()->Round; + + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvKeepAliveTimeout"); + return; + } + + const auto nodeId = ev->Get()->NodeId; + auto node = AggregationStatistics.GetProcessingChildNode(nodeId); + + if (node == nullptr) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvKeepAliveTimeout"); + return; + } + + const auto maxDuration = DurationToCycles(Settings.AggregateKeepAliveTimeout); + const auto deadline = node->LastHeartbeat + maxDuration; + const auto now = GetCycleCountFast(); + + if (deadline >= now) { + Schedule(Settings.AggregateKeepAliveTimeout, new TEvPrivate::TEvKeepAliveTimeout(round, nodeId)); + return; + } + + node->Status = TAggregationStatistics::TNode::EStatus::Unavailable; + ++AggregationStatistics.PprocessedNodes; + LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Node " << nodeId << " is unavailable"); + + if (AggregationStatistics.IsCompleted()) { + OnAggregateStatisticsFinished(); + } + } + + void Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev) { + const auto& record = ev->Get()->Record; + const auto round = record.GetRound(); + + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvAggregateKeepAlive"); + return; + } + + const auto nodeId = ev->Sender.NodeId(); + auto node = AggregationStatistics.GetProcessingChildNode(nodeId); + + if (node == nullptr) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvAggregateKeepAlive"); + return; + } + + auto response = std::make_unique(); + response->Record.SetRound(round); + Send(ev->Sender, response.release()); + + node->LastHeartbeat = GetCycleCountFast(); + } + + void Handle(TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Received TEvAggregateStatisticsResponse SenderNodeId: " << ev->Sender.NodeId()); + + const auto& record = ev->Get()->Record; + const auto round = record.GetRound(); + + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvAggregateStatisticsResponse"); + return; + } + + const auto nodeId = ev->Sender.NodeId(); + auto node = AggregationStatistics.GetProcessingChildNode(nodeId); + + if (node == nullptr) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvAggregateStatisticsResponse"); + return; + } + + node->Status = TAggregationStatistics::TNode::EStatus::Processed; + ++AggregationStatistics.PprocessedNodes; + + AggregateStatistics(record.GetColumns()); + + const auto size = AggregationStatistics.FailedTablets.size(); + AggregationStatistics.FailedTablets.reserve(size + record.GetFailedTablets().size()); + + for (const auto& fail : record.GetFailedTablets()) { + AggregationStatistics.FailedTablets.emplace_back( + fail.GetTabletId(), fail.GetNodeId(), fail.GetError() + ); + } + + if (AggregationStatistics.IsCompleted()) { + OnAggregateStatisticsFinished(); + } + } + + void AddUnavailableTablets(const TAggregationStatistics::TNode& node, + NKikimrStat::TEvAggregateStatisticsResponse& response) { + if (node.Status != TAggregationStatistics::TNode::EStatus::Unavailable) { + return; + } + + for (const auto& range : node.Tablets) { + for (const auto& tabletId : range.Ids) { + auto failedTablet = response.AddFailedTablets(); + failedTablet->SetNodeId(range.NodeId); + failedTablet->SetTabletId(tabletId); + failedTablet->SetError(NKikimrStat::TEvAggregateStatisticsResponse::TYPE_UNAVAILABLE_NODE); + } + } + } + + void SendAggregateStatisticsResponse() { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Send aggregate statistics response to node: " << AggregationStatistics.ParentNode.NodeId()); + + auto response = std::make_unique(); + auto& record = response->Record; + record.SetRound(AggregationStatistics.Round); + + const auto& countMinSketches = AggregationStatistics.CountMinSketches; + + for (auto it = countMinSketches.begin(); it != countMinSketches.end(); ++it) { + if (it->second.ContainedInResponse != AggregationStatistics.TotalStatisticsResponse) { + continue; + } + + auto column = record.AddColumns(); + column->SetTag(it->first); + + auto data = it->second.Statistics->AsStringBuf(); + auto statistics = column->AddStatistics(); + statistics->SetType(NKikimr::NStat::COUNT_MIN_SKETCH); + statistics->SetData(data.Data(), data.Size()); + } + + auto failedTablets = record.MutableFailedTablets(); + failedTablets->Reserve(AggregationStatistics.FailedTablets.size()); + + for (const auto& fail : AggregationStatistics.FailedTablets) { + auto failedTablet = failedTablets->Add(); + failedTablet->SetNodeId(fail.NodeId); + failedTablet->SetTabletId(fail.TabletId); + failedTablet->SetError(fail.Error); + } + + for (auto& node : AggregationStatistics.Nodes) { + AddUnavailableTablets(node, record); + } + + Send(AggregationStatistics.ParentNode, response.release(), 0, AggregationStatistics.Cookie); + } + + void SendRequestToNode(TAggregationStatistics::TNode& node, const NKikimrStat::TEvAggregateStatistics& record) { + if (node.Tablets.empty()) { + node.Status = TAggregationStatistics::TNode::EStatus::Processed; + ++AggregationStatistics.PprocessedNodes; + return; + } + + auto request = std::make_unique(); + request->Record.SetRound(AggregationStatistics.Round); + request->Record.MutableNodes()->Reserve(node.Tablets.size()); + + const auto& columnTags = record.GetColumnTags(); + if (!columnTags.empty()) { + request->Record.MutableColumnTags()->Assign(columnTags.begin(), columnTags.end()); + } + + auto pathId = request->Record.MutablePathId(); + pathId->SetOwnerId(AggregationStatistics.PathId.OwnerId); + pathId->SetLocalId(AggregationStatistics.PathId.LocalPathId); + + for (const auto& range : node.Tablets) { + auto recordNode = request->Record.AddNodes(); + recordNode->SetNodeId(range.NodeId); + + auto tabletIds = recordNode->MutableTabletIds(); + tabletIds->Reserve(range.Ids.size()); + + for (const auto& tabletId : range.Ids) { + tabletIds->Add(tabletId); + } + } + + // sending the request to the first node of the range + const auto nodeId = node.Tablets[0].NodeId; + node.Actor = MakeStatServiceID(nodeId); + node.Status = TAggregationStatistics::TNode::EStatus::Processing; + + Send(node.Actor, request.release()); + Schedule(Settings.AggregateKeepAliveTimeout, + new TEvPrivate::TEvKeepAliveTimeout(AggregationStatistics.Round, nodeId)); + } + + void Handle(TEvStatistics::TEvAggregateStatistics::TPtr& ev) { + const auto& record = ev->Get()->Record; + const auto round = record.GetRound(); + + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Received TEvAggregateStatistics from node: " << ev->Sender.NodeId() + << ", Round: " << round << ", current Round: " << AggregationStatistics.Round); + + // reset previous state + if (AggregationStatistics.Round != 0) { + TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); + std::swap(AggregationStatistics, aggregationStatistics); + } + + AggregationStatistics.Round = round; + AggregationStatistics.Cookie = ev->Cookie; + AggregationStatistics.ParentNode = ev->Sender; + + // schedule keep alive with the parent node + Schedule(Settings.AggregateKeepAlivePeriod, new TEvPrivate::TEvDispatchKeepAlive(round)); + + const auto& pathId = record.GetPathId(); + AggregationStatistics.PathId.OwnerId = pathId.GetOwnerId(); + AggregationStatistics.PathId.LocalPathId = pathId.GetLocalId(); + + for (const auto tag : record.GetColumnTags()) { + AggregationStatistics.ColumnTags.emplace_back(tag); + } + + const auto currentNodeId = ev->Recipient.NodeId(); + const auto& nodes = record.GetNodes(); + + // divide the entire range of nodes into two parts, + // forming the right and left child nodes + size_t k = 0; + for (const auto& node : nodes) { + ++k; + + if (node.GetNodeId() == currentNodeId) { + AggregationStatistics.LocalTablets.Ids.reserve(node.GetTabletIds().size()); + + for (const auto& tabletId : node.GetTabletIds()) { + AggregationStatistics.LocalTablets.Ids.push_back(tabletId); + } + continue; + } + + TAggregationStatistics::TTablets nodeTablets; + nodeTablets.NodeId = node.GetNodeId(); + nodeTablets.Ids.reserve(node.GetTabletIds().size()); + for (const auto& tabletId : node.GetTabletIds()) { + nodeTablets.Ids.push_back(tabletId); + } + + AggregationStatistics.Nodes[k % Settings.FanOutFactor].Tablets.push_back(std::move(nodeTablets)); + } + + for (auto& node : AggregationStatistics.Nodes) { + SendRequestToNode(node, record); + } + + // to check the locality of the tablets, + // send requests to receive the IDs of the nodes + // where the tablets are located + auto& localTablets = AggregationStatistics.LocalTablets; + const auto count = std::min(Settings.MaxInFlightTabletRequests, + localTablets.Ids.size()); + for (size_t i = 0; i < count; ++i) { + SendRequestToNextTablet(); + } + } + void Handle(TEvStatistics::TEvGetStatistics::TPtr& ev) { ui64 requestId = NextRequestId++; @@ -837,6 +1268,9 @@ class TStatService : public TActorBootstrapped { } private: + TStatServiceSettings Settings; + TAggregationStatistics AggregationStatistics; + bool EnableStatistics = false; bool EnableColumnStatistics = false; @@ -884,9 +1318,10 @@ class TStatService : public TActorBootstrapped { static constexpr TDuration RequestTimeout = TDuration::MilliSeconds(100); }; -THolder CreateStatService() { - return MakeHolder(); +THolder CreateStatService(const TStatServiceSettings& settings) { + return MakeHolder(settings); } + } // NStat } // NKikimr diff --git a/ydb/core/statistics/ut/ut_statistics.cpp b/ydb/core/statistics/service/ut/ut_basic_statistics.cpp similarity index 98% rename from ydb/core/statistics/ut/ut_statistics.cpp rename to ydb/core/statistics/service/ut/ut_basic_statistics.cpp index ab7846b21fe9..898e147cfabf 100644 --- a/ydb/core/statistics/ut/ut_statistics.cpp +++ b/ydb/core/statistics/service/ut/ut_basic_statistics.cpp @@ -1,9 +1,9 @@ -#include "ut_common.h" +#include #include #include -#include +#include #include #include @@ -82,7 +82,7 @@ void ValidateRowCount(TTestActorRuntime& runtime, ui32 nodeIndex, TPathId pathId } // namespace -Y_UNIT_TEST_SUITE(Statistics) { +Y_UNIT_TEST_SUITE(BasicStatistics) { Y_UNIT_TEST(Simple) { TTestEnv env(1, 1); diff --git a/ydb/core/statistics/service/ut/ut_service.cpp b/ydb/core/statistics/service/ut/ut_service.cpp new file mode 100644 index 000000000000..84d3c2827987 --- /dev/null +++ b/ydb/core/statistics/service/ut/ut_service.cpp @@ -0,0 +1,391 @@ +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace NKikimr { +namespace NStat { + +using EResponseStatus = NKikimrStat::TEvStatisticsResponse::EStatus; +using EErrorType = NKikimrStat::TEvAggregateStatisticsResponse::EErrorType; + +struct TAggregateStatisticsRequest { + struct TTablets { + ui32 NodeId; + std::vector Ids; + }; + ui64 Round; + TPathId PathId; + std::vector Nodes; + std::vector ColumnTags; +}; + +struct TColumnItem { + ui32 Tag; + std::vector Cells; +}; + +struct TStatisticsResponse { + ui64 TabletId; + std::vector Columns; + EResponseStatus Status; +}; + +struct TAggregateStatisticsResponse { + struct TFailedTablet { + ui64 TabletId; + ui32 NodeId; + EErrorType Error; + }; + + ui64 Round; + std::vector Columns; + std::vector FailedTablets; +}; + +std::unique_ptr CreateStatisticsRequest(const TAggregateStatisticsRequest& data) { + auto ev = std::make_unique(); + auto& record = ev->Record; + record.SetRound(data.Round); + + PathIdFromPathId(data.PathId, record.MutablePathId()); + + auto columnTags = record.MutableColumnTags(); + for (auto tag : data.ColumnTags) { + columnTags->Add(tag); + } + + for (const auto& tablets : data.Nodes) { + auto node = record.AddNodes(); + node->SetNodeId(tablets.NodeId); + + auto tabletIds = node->MutableTabletIds(); + for (auto tabletId : tablets.Ids) { + tabletIds->Add(tabletId); + } + } + + return std::move(ev); +} + +std::unique_ptr CreateAggregateStatisticsResponse(const TAggregateStatisticsResponse& data) { + auto ev = std::make_unique(); + auto& record = ev->Record; + record.SetRound(data.Round); + + for (const auto& fail : data.FailedTablets) { + auto failedTablets = record.AddFailedTablets(); + failedTablets->SetTabletId(fail.TabletId); + failedTablets->SetNodeId(fail.NodeId); + failedTablets->SetError(fail.Error); + } + + for (const auto& col : data.Columns) { + auto column = record.AddColumns(); + column->SetTag(col.Tag); + + auto statistics = column->AddStatistics(); + statistics->SetType(NKikimr::NStat::COUNT_MIN_SKETCH); + auto sketch = std::unique_ptr(TCountMinSketch::Create()); + + for (const auto& cell : col.Cells) { + sketch->Count(cell.data(), cell.size()); + } + + auto buf = sketch->AsStringBuf(); + statistics->SetData(buf.Data(), buf.Size()); + } + + return std::move(ev); +} + +std::unique_ptr CreateStatisticsResponse(const TStatisticsResponse& data) { + auto ev = std::make_unique(); + auto& record = ev->Record; + record.SetShardTabletId(data.TabletId); + record.SetStatus(data.Status); + + for (const auto& col : data.Columns) { + auto column = record.AddColumns(); + column->SetTag(col.Tag); + + auto statistics = column->AddStatistics(); + statistics->SetType(NKikimr::NStat::COUNT_MIN_SKETCH); + auto sketch = std::unique_ptr(TCountMinSketch::Create()); + + for (const auto& cell : col.Cells) { + sketch->Count(cell.data(), cell.size()); + } + + auto buf = sketch->AsStringBuf(); + statistics->SetData(buf.Data(), buf.Size()); + } + + return std::move(ev); +} + +TStatServiceSettings GetDefaultSettings() { + auto settings = TStatServiceSettings(); + settings.AggregateKeepAlivePeriod = TDuration::Seconds(15); + settings.AggregateKeepAliveTimeout = TDuration::Seconds(30); + settings.AggregateKeepAliveAckTimeout = TDuration::Seconds(30); + settings.FanOutFactor = 2; + return settings; +} + +std::unordered_map InitializeRuntime(TTestActorRuntime& runtime, ui32 nodesCount, + const TStatServiceSettings& settings = GetDefaultSettings()) { + runtime.SetLogPriority(NKikimrServices::STATISTICS, NLog::EPriority::PRI_DEBUG); + runtime.SetScheduledEventFilter([](TTestActorRuntimeBase&, TAutoPtr&, TDuration, TInstant&){ + return false; + }); + + TIntrusivePtr nameserverTable(new TTableNameserverSetup()); + TPortManager pm; + + for (ui32 i = 1; i <= nodesCount; ++i) { + nameserverTable->StaticNodeTable[i] = std::pair("127.0.0." + std::to_string(i), pm.GetPort(12000 + i)); + } + + auto nameserviceActor = GetNameserviceActorId(); + auto pipeActor = MakePipePerNodeCacheID(false); + auto pipeConfig = MakeIntrusive(); + + std::unordered_map indexToNodeId; + + for (ui32 i = 0; i < nodesCount; ++i) { + ui32 nodeId = runtime.GetNodeId(i); + indexToNodeId.emplace(i, nodeId); + + auto actorId = NStat::MakeStatServiceID(nodeId); + runtime.AddLocalService(actorId, TActorSetupCmd(NStat::CreateStatService(settings).Release(), TMailboxType::HTSwap, 0), i); + runtime.AddLocalService(pipeActor, TActorSetupCmd(CreatePipePerNodeCache(pipeConfig), TMailboxType::HTSwap, 0), i); + runtime.AddLocalService(nameserviceActor, TActorSetupCmd(CreateNameserverTable(nameserverTable), TMailboxType::Simple, 0), i); + } + + TTestActorRuntime::TEgg egg{ new TAppData(0, 0, 0, 0, { }, nullptr, nullptr, nullptr, nullptr), nullptr, nullptr, {} }; + runtime.Initialize(egg); + + return indexToNodeId; +} + +std::unordered_map ReverseMap(const std::unordered_map& map) { + std::unordered_map res; + for (auto it = map.begin(); it != map.end(); ++it) { + res.emplace(it->second, it->first); + } + return res; +} + + +Y_UNIT_TEST_SUITE(StatisticsService) { + Y_UNIT_TEST(ShouldBeVisitEveryNodeAndTablet) { + size_t nodeCount = 10; + auto runtime = TTestActorRuntime(nodeCount, 1, false); + auto indexToNodeIdMap = InitializeRuntime(runtime, nodeCount); + auto nodeIdToIndexMap = ReverseMap(indexToNodeIdMap); + std::vector nodesTablets = {{.NodeId = indexToNodeIdMap[0], .Ids{0}}, + {.NodeId = indexToNodeIdMap[1], .Ids{1}}, {.NodeId = indexToNodeIdMap[2], .Ids{2}}, + {.NodeId = indexToNodeIdMap[3], .Ids{3}}, {.NodeId = indexToNodeIdMap[4], .Ids{4}}, + {.NodeId = indexToNodeIdMap[5], .Ids{5}}, {.NodeId = indexToNodeIdMap[6], .Ids{6}}, + {.NodeId = indexToNodeIdMap[7], .Ids{7}}, {.NodeId = indexToNodeIdMap[8], .Ids{8}}, + {.NodeId = indexToNodeIdMap[9], .Ids{9}}}; + + std::unordered_map nodes; + std::unordered_map tablets; + std::vector observers; + observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvAggregateStatistics::TPtr& ev) { + if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { + ++nodes[ev->Recipient.NodeId()]; + } + })); + observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNode::TPtr& ev) { + if (nodeIdToIndexMap.find(ev->Sender.NodeId()) != nodeIdToIndexMap.end()) { + ++tablets[ev->Get()->TabletId]; + ev.Reset(); + } + })); + + auto sender = runtime.AllocateEdgeActor(); + runtime.Send(NStat::MakeStatServiceID(indexToNodeIdMap[0]), sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + .Round = 1, + .PathId{3, 3}, + .Nodes{ nodesTablets }, + .ColumnTags{1} + }).release()); + runtime.DispatchEvents(TDispatchOptions{ + .CustomFinalCondition = [&]() { + return nodes.size() >= 10 && tablets.size() >= 10; + } + }); + + for (auto it = nodes.begin(); it != nodes.end(); ++it) { + UNIT_ASSERT_VALUES_EQUAL(it->second, 1); + } + for (auto it = tablets.begin(); it != tablets.end(); ++it) { + UNIT_ASSERT_VALUES_EQUAL(it->second, 1); + } + UNIT_ASSERT_VALUES_EQUAL(nodesTablets.size(), nodes.size()); + UNIT_ASSERT_VALUES_EQUAL(nodesTablets.size(), tablets.size()); + } + + Y_UNIT_TEST(ShouldBeAtMostMaxInFlightTabletRequests) { + size_t nodeCount = 1; + auto runtime = TTestActorRuntime(nodeCount, 1, false); + auto settings = GetDefaultSettings() + .SetMaxInFlightTabletRequests(3); + auto indexToNodeIdMap = InitializeRuntime(runtime, nodeCount, settings); + auto nodeIdToIndexMap = ReverseMap(indexToNodeIdMap); + std::vector localTabletsIds = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector nodesTablets = {{.NodeId = indexToNodeIdMap[0], .Ids{localTabletsIds}}}; + + size_t inFlight = 0; + size_t maxInFlight = 0; + size_t tabletsCount = 0; + + std::vector observers; + observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNode::TPtr& ev) { + auto it = nodeIdToIndexMap.find(ev->Sender.NodeId()); + if (it == nodeIdToIndexMap.end()) { + return; + } + auto senderNodeIndex = it->second; + ++inFlight; + ++tabletsCount; + maxInFlight = std::max(maxInFlight, inFlight); + + auto tabletId = ev->Get()->TabletId; + + if (tabletId % 3 == 1) { // non local + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + new TEvPipeCache::TEvGetTabletNodeResult(tabletId, 10), 0, ev->Cookie), senderNodeIndex, true); + } else if(tabletId % 3 == 2) { // delivery problem + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + new TEvPipeCache::TEvDeliveryProblem(tabletId, true), 0, ev->Cookie), senderNodeIndex, true); + } else { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = tabletId, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), senderNodeIndex, true); + } + ev.Reset(); + })); + observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { + if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { + --inFlight; + } + })); + observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNodeResult::TPtr& ev) { + if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { + --inFlight; + } + })); + observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvStatisticsResponse::TPtr& ev) { + if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { + --inFlight; + } + })); + + auto sender = runtime.AllocateEdgeActor(); + runtime.Send(NStat::MakeStatServiceID(indexToNodeIdMap[0]), sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + .Round = 1, + .PathId{3, 3}, + .Nodes{ nodesTablets }, + .ColumnTags{1} + }).release()); + runtime.DispatchEvents(TDispatchOptions{ + .CustomFinalCondition = [&]() { + return tabletsCount >= localTabletsIds.size(); + } + }); + + UNIT_ASSERT_LT(maxInFlight, settings.MaxInFlightTabletRequests + 1); + } + + Y_UNIT_TEST(ShouldBeCorrectlyAggregateStatisticsFromAllNodes) { + size_t nodeCount = 4; + auto runtime = TTestActorRuntime(nodeCount, 1, false); + auto indexToNodeIdMap = InitializeRuntime(runtime, nodeCount); + auto nodeIdToIndexMap = ReverseMap(indexToNodeIdMap); + std::vector localTabletsIds = {1, 2, 3}; + std::vector nodesTablets = {{.NodeId = indexToNodeIdMap[0], .Ids{localTabletsIds}}, + {.NodeId = indexToNodeIdMap[1], .Ids{4}}, {.NodeId = indexToNodeIdMap[2], .Ids{5}}, + {.NodeId = indexToNodeIdMap[3], .Ids{6}}}; + + std::vector observers; + observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNode::TPtr& ev) { + auto tabletId = ev->Get()->TabletId; + auto senderNodeId = ev->Sender.NodeId(); + auto senderNodeIndex = nodeIdToIndexMap.find(senderNodeId); + + if (senderNodeIndex == nodeIdToIndexMap.end()) { + return; + } + + if (tabletId <= localTabletsIds.back()) { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = ev->Get()->TabletId, + .Columns{ + TColumnItem{.Tag = 1, .Cells{"1", "2"}}, + TColumnItem{.Tag = 2, .Cells{"3"}} + }, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), senderNodeIndex->second, true); + } else { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = ev->Get()->TabletId, + .Columns{ + TColumnItem{.Tag = 2, .Cells{"3"}} + }, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), senderNodeIndex->second, true); + } + + ev.Reset(); + })); + + auto sender = runtime.AllocateEdgeActor(); + runtime.Send(NStat::MakeStatServiceID(indexToNodeIdMap[0]), sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + .Round = 1, + .PathId{3, 3}, + .Nodes{ nodesTablets }, + .ColumnTags{1, 2} + }).release()); + auto res = runtime.GrabEdgeEvent(sender); + const auto& record = res->Get()->Record; + + std::unordered_map> expected = { + {1, {{"1", localTabletsIds.size()}, {"2", localTabletsIds.size()}}}, + {2, {{"3", nodesTablets.size() - 1 + localTabletsIds.size()}}}, + }; + + const auto& columns = record.GetColumns(); + for (const auto& column : columns) { + const auto tag = column.GetTag(); + + for (auto& statistic : column.GetStatistics()) { + if (statistic.GetType() == NKikimr::NStat::COUNT_MIN_SKETCH) { + auto data = statistic.GetData().Data(); + auto sketch = reinterpret_cast(data); + + const auto& cells = expected[tag]; + for (auto it = cells.begin(); it != cells.end(); ++it) { + UNIT_ASSERT_VALUES_EQUAL(it->second, sketch->Probe(it->first.data(), it->first.size())); + } + } + } + } + } +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/statistics/service/ut/ya.make b/ydb/core/statistics/service/ut/ya.make new file mode 100644 index 000000000000..7c07171048c5 --- /dev/null +++ b/ydb/core/statistics/service/ut/ya.make @@ -0,0 +1,29 @@ +UNITTEST_FOR(ydb/core/statistics/service) + +FORK_SUBTESTS() + +IF (WITH_VALGRIND) + TIMEOUT(3600) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +YQL_LAST_ABI_VERSION() + +PEERDIR( + library/cpp/testing/unittest + ydb/core/protos + ydb/core/testlib/default + ydb/core/statistics/ut_common +) + +SRCS( + ut_basic_statistics.cpp + ut_service.cpp +) + +END() + diff --git a/ydb/core/statistics/service/ya.make b/ydb/core/statistics/service/ya.make new file mode 100644 index 000000000000..70d536ba7e6d --- /dev/null +++ b/ydb/core/statistics/service/ya.make @@ -0,0 +1,28 @@ +LIBRARY() + +SRCS( + http_request.h + http_request.cpp + service.h + service.cpp + service_impl.cpp +) + +PEERDIR( + ydb/core/base + ydb/core/engine/minikql + ydb/core/protos + ydb/core/tablet + ydb/core/tablet_flat + ydb/core/statistics/database + ydb/library/minsketch +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS( + ut +) + diff --git a/ydb/core/statistics/stat_service.h b/ydb/core/statistics/stat_service.h deleted file mode 100644 index 7680afd5633c..000000000000 --- a/ydb/core/statistics/stat_service.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include - -namespace NKikimr { -namespace NStat { - -inline NActors::TActorId MakeStatServiceID(ui32 node) { - const char x[12] = "StatService"; - return NActors::TActorId(node, TStringBuf(x, 12)); -} - -THolder CreateStatService(); - -} // NStat -} // NKikimr diff --git a/ydb/core/statistics/ut/ut_aggregator.cpp b/ydb/core/statistics/ut/ut_aggregator.cpp deleted file mode 100644 index 3afbdc9b4ca2..000000000000 --- a/ydb/core/statistics/ut/ut_aggregator.cpp +++ /dev/null @@ -1,304 +0,0 @@ -#include "ut_common.h" - -#include - -#include -#include -#include - -#include -#include -#include - -#include - -namespace NKikimr { -namespace NStat { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NScheme; - -namespace { - -void CreateUniformTable(TTestEnv& env, const TString& databaseName, const TString& tableName) { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExecuteSchemeQuery(Sprintf(R"( - CREATE TABLE `Root/%s/%s` ( - Key Uint64, - Value Uint64, - PRIMARY KEY (Key) - ) - WITH ( UNIFORM_PARTITIONS = 4 ); - )", databaseName.c_str(), tableName.c_str())).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - TStringBuilder replace; - replace << Sprintf("REPLACE INTO `Root/%s/%s` (Key, Value) VALUES ", - databaseName.c_str(), tableName.c_str()); - for (ui32 i = 0; i < 4; ++i) { - if (i > 0) { - replace << ", "; - } - ui64 value = 4000000000000000000ull * (i + 1); - replace << Sprintf("(%" PRIu64 "ul, %" PRIu64 "ul)", value, value); - } - replace << ";"; - result = session.ExecuteDataQuery(replace, TTxControl::BeginTx().CommitTx()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); -} - -void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableName) { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExecuteSchemeQuery(Sprintf(R"( - DROP TABLE `Root/%s/%s`; - )", databaseName.c_str(), tableName.c_str())).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); -} - -void ValidateCountMin(TTestActorRuntime& runtime, TPathId pathId) { - auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); - - NStat::TRequest req; - req.PathId = pathId; - req.ColumnTag = 1; - - auto evGet = std::make_unique(); - evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH; - evGet->StatRequests.push_back(req); - - auto sender = runtime.AllocateEdgeActor(1); - runtime.Send(statServiceId, sender, evGet.release(), 1, true); - auto evResult = runtime.GrabEdgeEventRethrow(sender); - - UNIT_ASSERT(evResult); - UNIT_ASSERT(evResult->Get()); - UNIT_ASSERT(evResult->Get()->StatResponses.size() == 1); - - auto rsp = evResult->Get()->StatResponses[0]; - auto stat = rsp.CountMinSketch; - UNIT_ASSERT(rsp.Success); - UNIT_ASSERT(stat.CountMin); - - for (ui32 i = 0; i < 4; ++i) { - ui64 value = 4000000000000000000ull * (i + 1); - auto probe = stat.CountMin->Probe((const char *)&value, sizeof(ui64)); - UNIT_ASSERT_VALUES_EQUAL(probe, 1); - } -} - -void ValidateCountMinAbsense(TTestActorRuntime& runtime, TPathId pathId) { - auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); - - NStat::TRequest req; - req.PathId = pathId; - req.ColumnTag = 1; - - auto evGet = std::make_unique(); - evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH; - evGet->StatRequests.push_back(req); - - auto sender = runtime.AllocateEdgeActor(1); - runtime.Send(statServiceId, sender, evGet.release(), 1, true); - auto evResult = runtime.GrabEdgeEventRethrow(sender); - - UNIT_ASSERT(evResult); - UNIT_ASSERT(evResult->Get()); - UNIT_ASSERT(evResult->Get()->StatResponses.size() == 1); - - auto rsp = evResult->Get()->StatResponses[0]; - UNIT_ASSERT(!rsp.Success); -} - -} // namespace - -Y_UNIT_TEST_SUITE(StatisticsAggregator) { - - Y_UNIT_TEST(ScanOneTable) { - TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateUniformTable(env, "Database", "Table"); - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 tabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &tabletId); - - auto ev = std::make_unique(); - auto& record = ev->Record; - PathIdFromPathId(pathId, record.MutablePathId()); - - auto sender = runtime.AllocateEdgeActor(); - runtime.SendToPipe(tabletId, sender, ev.release()); - runtime.GrabEdgeEventRethrow(sender); - - ValidateCountMin(runtime, pathId); - } - - Y_UNIT_TEST(ScanTwoTables) { - TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateUniformTable(env, "Database", "Table1"); - CreateUniformTable(env, "Database", "Table2"); - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - runtime.SimulateSleep(TDuration::Seconds(60)); - - auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1"); - auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2"); - - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); - } - - Y_UNIT_TEST(ScanOneTableServerless) { - TTestEnv env(1, 1); - - auto init = [&] () { - CreateDatabase(env, "Shared"); - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - TPathId domainKey; - ResolvePathId(runtime, "/Root/Shared", &domainKey); - - auto init2 = [&] () { - CreateServerlessDatabase(env, "Serverless", domainKey); - CreateUniformTable(env, "Serverless", "Table"); - }; - std::thread init2Thread(init2); - - runtime.SimulateSleep(TDuration::Seconds(5)); - init2Thread.join(); - - runtime.SimulateSleep(TDuration::Seconds(60)); - - auto pathId = ResolvePathId(runtime, "/Root/Serverless/Table"); - ValidateCountMin(runtime, pathId); - } - - Y_UNIT_TEST(ScanTwoTablesServerless) { - TTestEnv env(1, 1); - - auto init = [&] () { - CreateDatabase(env, "Shared"); - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - TPathId domainKey; - ResolvePathId(runtime, "/Root/Shared", &domainKey); - - auto init2 = [&] () { - CreateServerlessDatabase(env, "Serverless", domainKey); - CreateUniformTable(env, "Serverless", "Table1"); - CreateUniformTable(env, "Serverless", "Table2"); - }; - std::thread init2Thread(init2); - - runtime.SimulateSleep(TDuration::Seconds(5)); - init2Thread.join(); - - runtime.SimulateSleep(TDuration::Seconds(60)); - - auto pathId1 = ResolvePathId(runtime, "/Root/Serverless/Table1"); - auto pathId2 = ResolvePathId(runtime, "/Root/Serverless/Table2"); - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); - } - - Y_UNIT_TEST(ScanTwoTablesTwoServerlessDbs) { - TTestEnv env(1, 1); - - auto init = [&] () { - CreateDatabase(env, "Shared"); - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - TPathId domainKey; - ResolvePathId(runtime, "/Root/Shared", &domainKey); - - auto init2 = [&] () { - CreateServerlessDatabase(env, "Serverless1", domainKey); - CreateServerlessDatabase(env, "Serverless2", domainKey); - CreateUniformTable(env, "Serverless1", "Table1"); - CreateUniformTable(env, "Serverless2", "Table2"); - }; - std::thread init2Thread(init2); - - runtime.SimulateSleep(TDuration::Seconds(5)); - init2Thread.join(); - - runtime.SimulateSleep(TDuration::Seconds(60)); - - auto pathId1 = ResolvePathId(runtime, "/Root/Serverless1/Table1"); - auto pathId2 = ResolvePathId(runtime, "/Root/Serverless2/Table2"); - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); - } - - Y_UNIT_TEST(DropTableNavigateError) { - TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateUniformTable(env, "Database", "Table"); - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 tabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &tabletId); - - auto init2 = [&] () { - DropTable(env, "Database", "Table"); - }; - std::thread init2Thread(init2); - - runtime.SimulateSleep(TDuration::Seconds(5)); - init2Thread.join(); - - auto ev = std::make_unique(); - auto& record = ev->Record; - PathIdFromPathId(pathId, record.MutablePathId()); - - auto sender = runtime.AllocateEdgeActor(); - runtime.SendToPipe(tabletId, sender, ev.release()); - - runtime.SimulateSleep(TDuration::Seconds(60)); - - ValidateCountMinAbsense(runtime, pathId); - } - -} - -} // NStat -} // NKikimr diff --git a/ydb/core/statistics/ut/ut_common.cpp b/ydb/core/statistics/ut/ut_common.cpp deleted file mode 100644 index ef9138246b59..000000000000 --- a/ydb/core/statistics/ut/ut_common.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include "ut_common.h" - -#include - -namespace NKikimr { -namespace NStat { - -NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(const TString &name, const TStoragePools &pools) { - NKikimrSubDomains::TSubDomainSettings subdomain; - subdomain.SetName(name); - for (auto& pool: pools) { - *subdomain.AddStoragePools() = pool; - } - return subdomain; -} - -NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString &name, const TStoragePools &pools) { - NKikimrSubDomains::TSubDomainSettings subdomain; - subdomain.SetName(name); - subdomain.SetCoordinators(1); - subdomain.SetMediators(1); - subdomain.SetPlanResolution(50); - subdomain.SetTimeCastBucketsPerMediator(2); - for (auto& pool: pools) { - *subdomain.AddStoragePools() = pool; - } - return subdomain; -} - -TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools) { - auto mbusPort = PortManager.GetPort(); - auto grpcPort = PortManager.GetPort(); - - Settings = new Tests::TServerSettings(mbusPort); - Settings->SetDomainName("Root"); - Settings->SetNodeCount(staticNodes); - Settings->SetDynamicNodeCount(dynamicNodes); - Settings->SetUseRealThreads(false); - - NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableStatistics(true); - featureFlags.SetEnableColumnStatistics(true); - Settings->SetFeatureFlags(featureFlags); - - for (ui32 i : xrange(storagePools)) { - TString poolName = Sprintf("test%d", i); - Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2); - } - - Server = new Tests::TServer(*Settings); - Server->EnableGRpc(grpcPort); - - auto sender = Server->GetRuntime()->AllocateEdgeActor(); - Server->SetupRootStoragePools(sender); - - Client = MakeHolder(*Settings); - - Tenants = MakeHolder(Server); - - Endpoint = "localhost:" + ToString(grpcPort); - DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint); - Driver = MakeHolder(DriverConfig); - - Server->GetRuntime()->SetLogPriority(NKikimrServices::STATISTICS, NActors::NLog::PRI_DEBUG); -} - -TTestEnv::~TTestEnv() { - Driver->Stop(true); -} - -TStoragePools TTestEnv::GetPools() const { - TStoragePools pools; - for (const auto& [kind, pool] : Settings->StoragePoolTypes) { - pools.emplace_back(pool.GetName(), kind); - } - return pools; -} - -void CreateDatabase(TTestEnv& env, const TString& databaseName, size_t nodeCount) { - auto subdomain = GetSubDomainDeclareSettings(databaseName); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().CreateExtSubdomain("/Root", subdomain)); - - env.GetTenants().Run("/Root/" + databaseName, nodeCount); - - auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools()); - subdomainSettings.SetExternalSchemeShard(true); - subdomainSettings.SetExternalStatisticsAggregator(true); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); -} - -void CreateServerlessDatabase(TTestEnv& env, const TString& databaseName, TPathId resourcesDomainKey) { - auto subdomain = GetSubDomainDeclareSettings(databaseName); - subdomain.MutableResourcesDomainKey()->SetSchemeShard(resourcesDomainKey.OwnerId); - subdomain.MutableResourcesDomainKey()->SetPathId(resourcesDomainKey.LocalPathId); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().CreateExtSubdomain("/Root", subdomain)); - - env.GetTenants().Run("/Root/" + databaseName, 0); - - auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools()); - subdomainSettings.SetExternalSchemeShard(true); - UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, - env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); -} - -TPathId ResolvePathId(TTestActorRuntime& runtime, const TString& path, - TPathId* domainKey, ui64* tabletId) -{ - auto sender = runtime.AllocateEdgeActor(); - - using TNavigate = NSchemeCache::TSchemeCacheNavigate; - using TEvRequest = TEvTxProxySchemeCache::TEvNavigateKeySet; - using TEvResponse = TEvTxProxySchemeCache::TEvNavigateKeySetResult; - - auto request = std::make_unique(); - auto& entry = request->ResultSet.emplace_back(); - entry.Path = SplitPath(path); - entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; - entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpPath; - entry.ShowPrivatePath = true; - runtime.Send(MakeSchemeCacheID(), sender, new TEvRequest(request.release())); - - auto ev = runtime.GrabEdgeEventRethrow(sender); - UNIT_ASSERT(ev); - UNIT_ASSERT(ev->Get()); - std::unique_ptr response(ev->Get()->Request.Release()); - UNIT_ASSERT(response->ResultSet.size() == 1); - auto& resultEntry = response->ResultSet[0]; - - if (domainKey) { - *domainKey = resultEntry.DomainInfo->DomainKey; - } - - if (tabletId && resultEntry.DomainInfo->Params.HasStatisticsAggregator()) { - *tabletId = resultEntry.DomainInfo->Params.GetStatisticsAggregator(); - } - - return resultEntry.TableId.PathId; -} - -} // NStat -} // NKikimr diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp new file mode 100644 index 000000000000..8005f02a00da --- /dev/null +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -0,0 +1,370 @@ +#include "ut_common.h" + +#include +#include + +#include +#include +#include + +// TODO remove SDK +#include +#include +#include + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NScheme; + +namespace NKikimr { +namespace NStat { + +NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings(const TString &name, const TStoragePools &pools) { + NKikimrSubDomains::TSubDomainSettings subdomain; + subdomain.SetName(name); + for (auto& pool: pools) { + *subdomain.AddStoragePools() = pool; + } + return subdomain; +} + +NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString &name, const TStoragePools &pools) { + NKikimrSubDomains::TSubDomainSettings subdomain; + subdomain.SetName(name); + subdomain.SetCoordinators(1); + subdomain.SetMediators(1); + subdomain.SetPlanResolution(50); + subdomain.SetTimeCastBucketsPerMediator(2); + for (auto& pool: pools) { + *subdomain.AddStoragePools() = pool; + } + return subdomain; +} + +TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools) { + auto mbusPort = PortManager.GetPort(); + auto grpcPort = PortManager.GetPort(); + + Settings = new Tests::TServerSettings(mbusPort); + Settings->SetDomainName("Root"); + Settings->SetNodeCount(staticNodes); + Settings->SetDynamicNodeCount(dynamicNodes); + Settings->SetUseRealThreads(false); + + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableStatistics(true); + featureFlags.SetEnableColumnStatistics(true); + Settings->SetFeatureFlags(featureFlags); + + for (ui32 i : xrange(storagePools)) { + TString poolName = Sprintf("test%d", i); + Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2); + } + + Server = new Tests::TServer(*Settings); + Server->EnableGRpc(grpcPort); + + auto sender = Server->GetRuntime()->AllocateEdgeActor(); + Server->SetupRootStoragePools(sender); + + Client = MakeHolder(*Settings); + + Tenants = MakeHolder(Server); + + Endpoint = "localhost:" + ToString(grpcPort); + DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint); + Driver = MakeHolder(DriverConfig); + + Server->GetRuntime()->SetLogPriority(NKikimrServices::STATISTICS, NActors::NLog::PRI_DEBUG); +} + +TTestEnv::~TTestEnv() { + Driver->Stop(true); +} + +TStoragePools TTestEnv::GetPools() const { + TStoragePools pools; + for (const auto& [kind, pool] : Settings->StoragePoolTypes) { + pools.emplace_back(pool.GetName(), kind); + } + return pools; +} + +void CreateDatabase(TTestEnv& env, const TString& databaseName, size_t nodeCount) { + auto subdomain = GetSubDomainDeclareSettings(databaseName); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().CreateExtSubdomain("/Root", subdomain)); + + env.GetTenants().Run("/Root/" + databaseName, nodeCount); + + auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools()); + subdomainSettings.SetExternalSchemeShard(true); + subdomainSettings.SetExternalStatisticsAggregator(true); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); +} + +void CreateServerlessDatabase(TTestEnv& env, const TString& databaseName, TPathId resourcesDomainKey) { + auto subdomain = GetSubDomainDeclareSettings(databaseName); + subdomain.MutableResourcesDomainKey()->SetSchemeShard(resourcesDomainKey.OwnerId); + subdomain.MutableResourcesDomainKey()->SetPathId(resourcesDomainKey.LocalPathId); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().CreateExtSubdomain("/Root", subdomain)); + + env.GetTenants().Run("/Root/" + databaseName, 0); + + auto subdomainSettings = GetSubDomainDefaultSettings(databaseName, env.GetPools()); + subdomainSettings.SetExternalSchemeShard(true); + UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK, + env.GetClient().AlterExtSubdomain("/Root", subdomainSettings)); +} + +TPathId ResolvePathId(TTestActorRuntime& runtime, const TString& path, TPathId* domainKey, ui64* saTabletId) +{ + auto sender = runtime.AllocateEdgeActor(); + + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + using TEvRequest = TEvTxProxySchemeCache::TEvNavigateKeySet; + using TEvResponse = TEvTxProxySchemeCache::TEvNavigateKeySetResult; + + auto request = std::make_unique(); + auto& entry = request->ResultSet.emplace_back(); + entry.Path = SplitPath(path); + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::EOp::OpPath; + entry.ShowPrivatePath = true; + runtime.Send(MakeSchemeCacheID(), sender, new TEvRequest(request.release())); + + auto ev = runtime.GrabEdgeEventRethrow(sender); + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()); + std::unique_ptr response(ev->Get()->Request.Release()); + UNIT_ASSERT(response->ResultSet.size() == 1); + auto& resultEntry = response->ResultSet[0]; + + if (domainKey) { + *domainKey = resultEntry.DomainInfo->DomainKey; + } + + if (saTabletId && resultEntry.DomainInfo->Params.HasStatisticsAggregator()) { + *saTabletId = resultEntry.DomainInfo->Params.GetStatisticsAggregator(); + } + + return resultEntry.TableId.PathId; +} + +NKikimrScheme::TEvDescribeSchemeResult DescribeTable(TTestActorRuntime& runtime, TActorId sender, const TString &path) +{ + TAutoPtr handle; + + auto request = MakeHolder(); + request->Record.MutableDescribePath()->SetPath(path); + request->Record.MutableDescribePath()->MutableOptions()->SetShowPrivateTable(true); + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); + auto reply = runtime.GrabEdgeEventRethrow(handle); + + return *reply->MutableRecord(); +} + +TVector GetTableShards(TTestActorRuntime& runtime, TActorId sender, const TString &path) +{ + TVector shards; + auto lsResult = DescribeTable(runtime, sender, path); + for (auto &part : lsResult.GetPathDescription().GetTablePartitions()) + shards.push_back(part.GetDatashardId()); + + return shards; +} + +TVector GetColumnTableShards(TTestActorRuntime& runtime, TActorId sender,const TString &path) +{ + TVector shards; + auto lsResult = DescribeTable(runtime, sender, path); + for (auto &part : lsResult.GetPathDescription().GetColumnTableDescription().GetSharding().GetColumnShards()) + shards.push_back(part); + + return shards; +} + +void CreateUniformTable(TTestEnv& env, const TString& databaseName, const TString& tableName) { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery(Sprintf(R"( + CREATE TABLE `Root/%s/%s` ( + Key Uint64, + Value Uint64, + PRIMARY KEY (Key) + ) + WITH ( UNIFORM_PARTITIONS = 4 ); + )", databaseName.c_str(), tableName.c_str())).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + TStringBuilder replace; + replace << Sprintf("REPLACE INTO `Root/%s/%s` (Key, Value) VALUES ", + databaseName.c_str(), tableName.c_str()); + for (ui32 i = 0; i < 4; ++i) { + if (i > 0) { + replace << ", "; + } + ui64 value = 4000000000000000000ull * (i + 1); + replace << Sprintf("(%" PRIu64 "ul, %" PRIu64 "ul)", value, value); + } + replace << ";"; + result = session.ExecuteDataQuery(replace, TTxControl::BeginTx().CommitTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); +} + +void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TString& tableName, + int shardCount) +{ + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto fullTableName = Sprintf("Root/%s/%s", databaseName.c_str(), tableName.c_str()); + auto result = session.ExecuteSchemeQuery(Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint64 NOT NULL, + Value Uint64, + PRIMARY KEY (Key) + ) + PARTITION BY HASH(Key) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d + ); + )", fullTableName.c_str(), shardCount)).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + NYdb::TValueBuilder rows; + rows.BeginList(); + for (size_t i = 0; i < 100; ++i) { + auto key = TValueBuilder().Uint64(i).Build(); + auto value = TValueBuilder().OptionalUint64(i).Build(); + rows.AddListItem(); + rows.BeginStruct(); + rows.AddMember("Key", key); + rows.AddMember("Value", value); + rows.EndStruct(); + } + rows.EndList(); + + result = client.BulkUpsert(fullTableName, rows.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); +} + +void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableName) { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery(Sprintf(R"( + DROP TABLE `Root/%s/%s`; + )", databaseName.c_str(), tableName.c_str())).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); +} + +std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPathId pathId) { + auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); + + NStat::TRequest req; + req.PathId = pathId; + req.ColumnTag = 1; + + auto evGet = std::make_unique(); + evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH; + evGet->StatRequests.push_back(req); + + auto sender = runtime.AllocateEdgeActor(1); + runtime.Send(statServiceId, sender, evGet.release(), 1, true); + auto evResult = runtime.GrabEdgeEventRethrow(sender); + + UNIT_ASSERT(evResult); + UNIT_ASSERT(evResult->Get()); + UNIT_ASSERT(evResult->Get()->StatResponses.size() == 1); + + auto rsp = evResult->Get()->StatResponses[0]; + auto stat = rsp.CountMinSketch; + UNIT_ASSERT(rsp.Success); + UNIT_ASSERT(stat.CountMin); + + return stat.CountMin; +} + +void ValidateCountMin(TTestActorRuntime& runtime, TPathId pathId) { + auto countMin = ExtractCountMin(runtime, pathId); + + for (ui32 i = 0; i < 4; ++i) { + ui64 value = 4000000000000000000ull * (i + 1); + auto probe = countMin->Probe((const char *)&value, sizeof(ui64)); + UNIT_ASSERT_VALUES_EQUAL(probe, 1); + } +} + +void ValidateCountMinAbsense(TTestActorRuntime& runtime, TPathId pathId) { + auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); + + NStat::TRequest req; + req.PathId = pathId; + req.ColumnTag = 1; + + auto evGet = std::make_unique(); + evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH; + evGet->StatRequests.push_back(req); + + auto sender = runtime.AllocateEdgeActor(1); + runtime.Send(statServiceId, sender, evGet.release(), 1, true); + auto evResult = runtime.GrabEdgeEventRethrow(sender); + + UNIT_ASSERT(evResult); + UNIT_ASSERT(evResult->Get()); + UNIT_ASSERT(evResult->Get()->StatResponses.size() == 1); + + auto rsp = evResult->Get()->StatResponses[0]; + UNIT_ASSERT(!rsp.Success); +} + +TAnalyzedTable::TAnalyzedTable(const TPathId& pathId) + : PathId(pathId) +{} + +TAnalyzedTable::TAnalyzedTable(const TPathId& pathId, const std::vector& columnTags) + : PathId(pathId) + , ColumnTags(columnTags) +{} + +void TAnalyzedTable::ToProto(NKikimrStat::TTable& tableProto) const { + PathIdFromPathId(PathId, tableProto.MutablePathId()); + tableProto.MutableColumnTags()->Add(ColumnTags.begin(), ColumnTags.end()); +} + + +void Analyze(TTestActorRuntime& runtime, const std::vector& tables, ui64 saTabletId) { + const ui64 cookie = 555; + auto ev = std::make_unique(); + NKikimrStat::TEvAnalyze& record = ev->Record; + record.SetCookie(cookie); + record.AddTypes(NKikimrStat::EColumnStatisticType::TYPE_COUNT_MIN_SKETCH); + for (const TAnalyzedTable& table : tables) + table.ToProto(*record.AddTables()); + + auto sender = runtime.AllocateEdgeActor(); + runtime.SendToPipe(saTabletId, sender, ev.release()); + auto evResponse = runtime.GrabEdgeEventRethrow(sender); + + UNIT_ASSERT_VALUES_EQUAL(evResponse->Get()->Record.GetCookie(), cookie); +} + +void AnalyzeTable(TTestActorRuntime& runtime, const TAnalyzedTable& table, ui64 shardTabletId) { + auto ev = std::make_unique(); + auto& record = ev->Record; + table.ToProto(*record.MutableTable()); + record.AddTypes(NKikimrStat::EColumnStatisticType::TYPE_COUNT_MIN_SKETCH); + + auto sender = runtime.AllocateEdgeActor(); + runtime.SendToPipe(shardTabletId, sender, ev.release()); + runtime.GrabEdgeEventRethrow(sender); +} + + +} // NStat +} // NKikimr diff --git a/ydb/core/statistics/ut/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h similarity index 55% rename from ydb/core/statistics/ut/ut_common.h rename to ydb/core/statistics/ut_common/ut_common.h index e73c2d69e9e5..532ce0250e10 100644 --- a/ydb/core/statistics/ut/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -3,6 +3,10 @@ #include #include +namespace NKikimrStat { + class TTable; +} + namespace NKikimr { namespace NStat { @@ -60,8 +64,31 @@ void CreateDatabase(TTestEnv& env, const TString& databaseName, size_t nodeCount void CreateServerlessDatabase(TTestEnv& env, const TString& databaseName, TPathId resourcesDomainKey); -TPathId ResolvePathId(TTestActorRuntime& runtime, const TString& path, - TPathId* domainKey = nullptr, ui64* tabletId = nullptr); +TPathId ResolvePathId(TTestActorRuntime& runtime, const TString& path, TPathId* domainKey = nullptr, ui64* saTabletId = nullptr); + + +TVector GetTableShards(TTestActorRuntime& runtime, TActorId sender, const TString &path); +TVector GetColumnTableShards(TTestActorRuntime& runtime, TActorId sender,const TString &path); + +void CreateUniformTable(TTestEnv& env, const TString& databaseName, const TString& tableName); +void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TString& tableName, int shardCount); +void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableName); + +std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPathId pathId); +void ValidateCountMin(TTestActorRuntime& runtime, TPathId pathId); +void ValidateCountMinAbsense(TTestActorRuntime& runtime, TPathId pathId); + +struct TAnalyzedTable { + TPathId PathId; + std::vector ColumnTags; + + TAnalyzedTable(const TPathId& pathId); + TAnalyzedTable(const TPathId& pathId, const std::vector& columnTags); + void ToProto(NKikimrStat::TTable& tableProto) const; +}; + +void Analyze(TTestActorRuntime& runtime, const std::vector& table, ui64 saTabletId); +void AnalyzeTable(TTestActorRuntime& runtime, const TAnalyzedTable& table, ui64 shardTabletId); } // namespace NStat } // namespace NKikimr diff --git a/ydb/core/statistics/ut_common/ya.make b/ydb/core/statistics/ut_common/ya.make new file mode 100644 index 000000000000..38e6f63e24ed --- /dev/null +++ b/ydb/core/statistics/ut_common/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +SRCS( + ut_common.cpp + ut_common.h +) + +PEERDIR( + ydb/core/testlib +) + +YQL_LAST_ABI_VERSION() + +END() + diff --git a/ydb/core/statistics/ya.make b/ydb/core/statistics/ya.make index d19742d6ee4b..ec4a6bd7a38f 100644 --- a/ydb/core/statistics/ya.make +++ b/ydb/core/statistics/ya.make @@ -1,11 +1,8 @@ LIBRARY() SRCS( + common.h events.h - stat_service.h - stat_service.cpp - save_load_stats.h - save_load_stats.cpp ) PEERDIR( @@ -21,8 +18,13 @@ END() RECURSE( aggregator + database + service + ut_common ) RECURSE_FOR_TESTS( - ut + aggregator/ut + database/ut + service/ut ) diff --git a/ydb/core/testlib/basics/services.cpp b/ydb/core/testlib/basics/services.cpp index 80d526f7815e..a0bda8923784 100644 --- a/ydb/core/testlib/basics/services.cpp +++ b/ydb/core/testlib/basics/services.cpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include diff --git a/ydb/core/testlib/basics/ya.make b/ydb/core/testlib/basics/ya.make index 642640447975..67fef3796fbc 100644 --- a/ydb/core/testlib/basics/ya.make +++ b/ydb/core/testlib/basics/ya.make @@ -21,6 +21,7 @@ PEERDIR( ydb/core/mind ydb/core/node_whiteboard ydb/core/quoter + ydb/core/statistics/service ydb/core/tablet_flat ydb/core/testlib/actors ydb/core/tx/columnshard diff --git a/ydb/core/tx/columnshard/columnshard__statistics.cpp b/ydb/core/tx/columnshard/columnshard__statistics.cpp index 779af24b4e1e..152445ce1ed4 100644 --- a/ydb/core/tx/columnshard/columnshard__statistics.cpp +++ b/ydb/core/tx/columnshard/columnshard__statistics.cpp @@ -5,12 +5,36 @@ namespace NKikimr::NColumnShard { +void TColumnShard::Handle(NStat::TEvStatistics::TEvAnalyzeTable::TPtr& ev, const TActorContext&) { + + // TODO Start a potentially long analysis process. + // ... + + + + // Return the response when the analysis is completed + auto response = std::make_unique(); + Send(ev->Sender, response.release(), 0, ev->Cookie); +} + void TColumnShard::Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev, const TActorContext&) { auto response = std::make_unique(); auto& record = response->Record; record.SetShardTabletId(TabletID()); - record.SetStatus(NKikimrStat::TEvStatisticsResponse::SUCCESS); + record.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS); + + std::unique_ptr sketch(TCountMinSketch::Create()); + ui32 value = 1; + sketch->Count((const char*)&value, sizeof(value)); + TString strSketch(sketch->AsStringBuf()); + + auto* column = record.AddColumns(); + column->SetTag(1); + + auto* statistic = column->AddStatistics(); + statistic->SetType(NStat::COUNT_MIN_SKETCH); + statistic->SetData(std::move(strSketch)); Send(ev->Sender, response.release(), 0, ev->Cookie); } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 2fefad7e9e1a..3500852fc734 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -224,6 +224,7 @@ class TColumnShard void Handle(TEvPrivate::TEvTieringModified::TPtr& ev, const TActorContext&); void Handle(TEvPrivate::TEvNormalizerResult::TPtr& ev, const TActorContext&); + void Handle(NStat::TEvStatistics::TEvAnalyzeTable::TPtr& ev, const TActorContext& ctx); void Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev, const TActorContext& ctx); void Handle(NActors::TEvents::TEvUndelivered::TPtr& ev, const TActorContext&); @@ -380,6 +381,7 @@ class TColumnShard HFunc(TEvPrivate::TEvGarbageCollectionFinished, Handle); HFunc(TEvPrivate::TEvTieringModified, Handle); + HFunc(NStat::TEvStatistics::TEvAnalyzeTable, Handle); HFunc(NStat::TEvStatistics::TEvStatisticsRequest, Handle); HFunc(NActors::TEvents::TEvUndelivered, Handle); diff --git a/ydb/core/tx/datashard/datashard__column_stats.cpp b/ydb/core/tx/datashard/datashard__column_stats.cpp index 9a926e0c92cb..105c3155d499 100644 --- a/ydb/core/tx/datashard/datashard__column_stats.cpp +++ b/ydb/core/tx/datashard/datashard__column_stats.cpp @@ -63,13 +63,13 @@ class TStatisticsScan: public NTable::IScan { record.SetShardTabletId(ShardTabletId); if (abort != EAbort::None) { - record.SetStatus(NKikimrStat::TEvStatisticsResponse::ABORTED); + record.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_ABORTED); TlsActivationContext->Send(new IEventHandle(ReplyTo, TActorId(), response.release(), 0, Cookie)); delete this; return nullptr; } - record.SetStatus(NKikimrStat::TEvStatisticsResponse::SUCCESS); + record.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS); auto tags = Scheme->Tags(); for (size_t t = 0; t < tags.size(); ++t) { auto* column = record.AddColumns(); @@ -132,16 +132,16 @@ void TDataShard::HandleSafe(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev auto response = std::make_unique(); response->Record.SetShardTabletId(TabletID()); - const auto& tableId = record.GetTableId(); - if (PathOwnerId != tableId.GetOwnerId()) { - response->Record.SetStatus(NKikimrStat::TEvStatisticsResponse::ERROR); + const auto& pathId = record.GetTable().GetPathId(); + if (PathOwnerId != pathId.GetOwnerId()) { + response->Record.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_ERROR); Send(ev->Sender, response.release(), 0, ev->Cookie); return; } - auto infoIt = TableInfos.find(tableId.GetTableId()); + auto infoIt = TableInfos.find(pathId.GetLocalId()); if (infoIt == TableInfos.end()) { - response->Record.SetStatus(NKikimrStat::TEvStatisticsResponse::ERROR); + response->Record.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_ERROR); Send(ev->Sender, response.release(), 0, ev->Cookie); return; } diff --git a/ydb/core/tx/datashard/datashard_ut_column_stats.cpp b/ydb/core/tx/datashard/datashard_ut_column_stats.cpp index 459887375da5..2f7434f1972d 100644 --- a/ydb/core/tx/datashard/datashard_ut_column_stats.cpp +++ b/ydb/core/tx/datashard/datashard_ut_column_stats.cpp @@ -39,14 +39,14 @@ Y_UNIT_TEST_SUITE(StatisticsScan) { ui64 shardId = shards.at(0); auto request = std::make_unique(); - auto* reqTableId = request->Record.MutableTableId(); + auto* reqTableId = request->Record.MutableTable()->MutablePathId(); reqTableId->SetOwnerId(tableId.PathId.OwnerId); - reqTableId->SetTableId(tableId.PathId.LocalPathId); + reqTableId->SetLocalId(tableId.PathId.LocalPathId); runtime.SendToPipe(shardId, sender, request.release()); auto response = runtime.GrabEdgeEventRethrow(sender); auto& record = response->Get()->Record; - UNIT_ASSERT(record.GetStatus() == NKikimrStat::TEvStatisticsResponse::SUCCESS); + UNIT_ASSERT(record.GetStatus() == NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS); UNIT_ASSERT(record.ColumnsSize() == 2); for (ui32 i = 0; i < 2; ++i) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 4cb4ddd93d22..a3ff33cb2cc8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -7371,6 +7371,7 @@ void TSchemeShard::SendBaseStatsToSA() { entryPathId->SetLocalId(pathId.LocalPathId); entry->SetRowCount(aggregated.RowCount); entry->SetBytesSize(aggregated.DataSize); + entry->SetIsColumnTable(false); ++count; } auto columnTablesPathIds = ColumnTables.GetAllPathIds(); @@ -7383,6 +7384,7 @@ void TSchemeShard::SendBaseStatsToSA() { entryPathId->SetLocalId(pathId.LocalPathId); entry->SetRowCount(aggregated.RowCount); entry->SetBytesSize(aggregated.DataSize); + entry->SetIsColumnTable(true); ++count; } From cff3246f52c2b41277706def35902c2c5fe76f85 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:33:40 +0000 Subject: [PATCH 02/17] Statistics: OperationId is generated by KQP (#7694) --- ydb/core/protos/out/out.cpp | 5 ++ ydb/core/protos/statistics.proto | 8 +-- .../statistics/aggregator/aggregator_impl.cpp | 19 +++--- .../statistics/aggregator/aggregator_impl.h | 8 +-- ydb/core/statistics/aggregator/schema.h | 4 +- .../aggregator/tx_analyze_table.cpp | 28 ++++----- .../aggregator/tx_finish_trasersal.cpp | 10 ++-- ydb/core/statistics/aggregator/tx_init.cpp | 14 +---- .../aggregator/ut/ut_analyze_columnshard.cpp | 59 +++++++++++++++++-- .../aggregator/ut/ut_analyze_datashard.cpp | 6 +- ydb/core/statistics/database/database.cpp | 10 ++-- .../statistics/database/ut/ut_database.cpp | 37 ++++++++++++ ydb/core/statistics/service/http_request.cpp | 18 +++++- ydb/core/statistics/service/http_request.h | 1 + ydb/core/statistics/ut_common/ut_common.cpp | 28 ++++++--- ydb/core/statistics/ut_common/ut_common.h | 10 +++- ydb/library/query_actor/query_actor.cpp | 11 +++- ydb/library/query_actor/query_actor.h | 3 +- ydb/library/table_creator/table_creator.cpp | 17 ++++-- ydb/library/table_creator/table_creator.h | 3 +- 20 files changed, 213 insertions(+), 86 deletions(-) diff --git a/ydb/core/protos/out/out.cpp b/ydb/core/protos/out/out.cpp index eee0c8d61b22..735ac9b34a44 100644 --- a/ydb/core/protos/out/out.cpp +++ b/ydb/core/protos/out/out.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -238,3 +239,7 @@ Y_DECLARE_OUT_SPEC(, NKikimrDataEvents::TEvWrite::TOperation::EOperationType, st Y_DECLARE_OUT_SPEC(, NKikimrDataEvents::TEvWrite::ETxMode, stream, value) { stream << NKikimrDataEvents::TEvWrite::ETxMode_Name(value); } + +Y_DECLARE_OUT_SPEC(, NKikimrStat::TEvAnalyzeStatusResponse_EStatus, stream, value) { + stream << NKikimrStat::TEvAnalyzeStatusResponse_EStatus_Name(value); +} diff --git a/ydb/core/protos/statistics.proto b/ydb/core/protos/statistics.proto index 4e28439d41db..439feacc2c4a 100644 --- a/ydb/core/protos/statistics.proto +++ b/ydb/core/protos/statistics.proto @@ -81,24 +81,24 @@ message TTable { // KQP -> SA message TEvAnalyze { - optional uint64 Cookie = 1; // request cookie to match response item + optional bytes OperationId = 1; // unique identifier to match response item repeated TTable Tables = 2; // list of analyzed tables and columns repeated EColumnStatisticType Types = 3; // list of statistics types requested. Empty means asking for all available. } // SA -> KQP message TEvAnalyzeResponse { - optional uint64 Cookie = 1; + optional bytes OperationId = 1; } // KQP -> SA message TEvAnalyzeStatus { - optional NKikimrProto.TPathID PathId = 1; + optional bytes OperationId = 1; // unique identifier to match response item } // SA -> KQP message TEvAnalyzeStatusResponse { - optional NKikimrProto.TPathID PathId = 1; + optional bytes OperationId = 1; enum EStatus { STATUS_UNSPECIFIED = 0; diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 6a92bf8c2534..12ba2dc9c1c4 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -433,17 +433,18 @@ void TStatisticsAggregator::Handle(TEvStatistics::TEvStatTableCreationResponse:: } void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyzeStatus::TPtr& ev) { - auto& inRecord = ev->Get()->Record; - auto pathId = PathIdFromPathId(inRecord.GetPathId()); + const auto& inRecord = ev->Get()->Record; + const TString operationId = inRecord.GetOperationId(); auto response = std::make_unique(); auto& outRecord = response->Record; + outRecord.SetOperationId(operationId); - if (TraversalTableId.PathId == pathId) { + if (ForceTraversalOperationId == operationId) { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS); } else { if (std::any_of(ForceTraversals.begin(), ForceTraversals.end(), - [&pathId](const TForceTraversal& elem) { return elem.PathId == pathId;})) { + [&operationId](const TForceTraversal& elem) { return elem.OperationId == operationId;})) { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); } else { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); @@ -586,7 +587,6 @@ void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { pathId = operation.PathId; ForceTraversalOperationId = operation.OperationId; - ForceTraversalCookie = operation.Cookie; ForceTraversalColumnTags = operation.ColumnTags; ForceTraversalTypes = operation.Types; ForceTraversalReplyToActorId = operation.ReplyToActorId; @@ -678,22 +678,17 @@ void TStatisticsAggregator::PersistStartKey(NIceDb::TNiceDb& db) { void TStatisticsAggregator::PersistForceTraversal(NIceDb::TNiceDb& db) { PersistSysParam(db, Schema::SysParam_ForceTraversalOperationId, ToString(ForceTraversalOperationId)); - PersistSysParam(db, Schema::SysParam_ForceTraversalCookie, ToString(ForceTraversalCookie)); + PersistSysParam(db, Schema::SysParam_ForceTraversalCookie, ForceTraversalOperationId); PersistSysParam(db, Schema::SysParam_ForceTraversalColumnTags, ToString(ForceTraversalColumnTags)); PersistSysParam(db, Schema::SysParam_ForceTraversalTypes, ToString(ForceTraversalTypes)); } -void TStatisticsAggregator::PersistNextForceTraversalOperationId(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_NextForceTraversalOperationId, ToString(NextForceTraversalOperationId)); -} - void TStatisticsAggregator::PersistGlobalTraversalRound(NIceDb::TNiceDb& db) { PersistSysParam(db, Schema::SysParam_GlobalTraversalRound, ToString(GlobalTraversalRound)); } void TStatisticsAggregator::ResetTraversalState(NIceDb::TNiceDb& db) { - ForceTraversalOperationId = 0; - ForceTraversalCookie = 0; + ForceTraversalOperationId.clear(); TraversalTableId.PathId = TPathId(); ForceTraversalColumnTags.clear(); ForceTraversalTypes.clear(); diff --git a/ydb/core/statistics/aggregator/aggregator_impl.h b/ydb/core/statistics/aggregator/aggregator_impl.h index ca8872dcb4e5..60dc0998a855 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.h +++ b/ydb/core/statistics/aggregator/aggregator_impl.h @@ -147,7 +147,6 @@ class TStatisticsAggregator : public TActor, public NTabl void PersistTraversal(NIceDb::TNiceDb& db); void PersistForceTraversal(NIceDb::TNiceDb& db); void PersistStartKey(NIceDb::TNiceDb& db); - void PersistNextForceTraversalOperationId(NIceDb::TNiceDb& db); void PersistGlobalTraversalRound(NIceDb::TNiceDb& db); void ResetTraversalState(NIceDb::TNiceDb& db); @@ -306,15 +305,13 @@ class TStatisticsAggregator : public TActor, public NTabl private: // stored in local db - ui64 ForceTraversalOperationId = 0; - ui64 ForceTraversalCookie = 0; + TString ForceTraversalOperationId; TString ForceTraversalColumnTags; TString ForceTraversalTypes; TTableId TraversalTableId; bool TraversalIsColumnTable = false; TSerializedCellVec TraversalStartKey; TInstant TraversalStartTime; - ui64 NextForceTraversalOperationId = 0; size_t GlobalTraversalRound = 1; @@ -327,8 +324,7 @@ class TStatisticsAggregator : public TActor, public NTabl TTraversalsByTime ScheduleTraversalsByTime; struct TForceTraversal { - ui64 OperationId = 0; - ui64 Cookie = 0; + TString OperationId; TPathId PathId; TString ColumnTags; TString Types; diff --git a/ydb/core/statistics/aggregator/schema.h b/ydb/core/statistics/aggregator/schema.h index dc843a425cc7..036902199f1d 100644 --- a/ydb/core/statistics/aggregator/schema.h +++ b/ydb/core/statistics/aggregator/schema.h @@ -50,7 +50,7 @@ struct TAggregatorSchema : NIceDb::Schema { struct OperationId : Column<1, NScheme::NTypeIds::Uint64> {}; struct OwnerId : Column<2, NScheme::NTypeIds::Uint64> {}; struct LocalPathId : Column<3, NScheme::NTypeIds::Uint64> {}; - struct Cookie : Column<4, NScheme::NTypeIds::Uint64> {}; + struct Cookie : Column<4, NScheme::NTypeIds::String> {}; struct ColumnTags : Column<5, NScheme::NTypeIds::String> {}; struct Types : Column<6, NScheme::NTypeIds::String> {}; @@ -87,7 +87,7 @@ struct TAggregatorSchema : NIceDb::Schema { static constexpr ui64 SysParam_ForceTraversalColumnTags = 7; static constexpr ui64 SysParam_ForceTraversalTypes = 8; static constexpr ui64 SysParam_TraversalStartTime = 9; - static constexpr ui64 SysParam_NextForceTraversalOperationId = 10; + // deprecated 10 static constexpr ui64 SysParam_TraversalIsColumnTable = 11; static constexpr ui64 SysParam_GlobalTraversalRound = 12; }; diff --git a/ydb/core/statistics/aggregator/tx_analyze_table.cpp b/ydb/core/statistics/aggregator/tx_analyze_table.cpp index be97dae4cc31..df8ecd34820f 100644 --- a/ydb/core/statistics/aggregator/tx_analyze_table.cpp +++ b/ydb/core/statistics/aggregator/tx_analyze_table.cpp @@ -19,7 +19,7 @@ struct TStatisticsAggregator::TTxAnalyzeTable : public TTxBase { TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE; } bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute"); + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute. ReplyToActorId " << ReplyToActorId << " , Record " << Record); if (!Self->EnableColumnStatistics) { return true; @@ -27,27 +27,31 @@ struct TStatisticsAggregator::TTxAnalyzeTable : public TTxBase { NIceDb::TNiceDb db(txc.DB); - const ui64 cookie = Record.GetCookie(); + const TString operationId = Record.GetOperationId(); const TString types = JoinVectorIntoString(TVector(Record.GetTypes().begin(), Record.GetTypes().end()), ","); for (const auto& table : Record.GetTables()) { const TPathId pathId = PathIdFromPathId(table.GetPathId()); const TString columnTags = JoinVectorIntoString(TVector{table.GetColumnTags().begin(),table.GetColumnTags().end()},","); - // drop request with the same cookie and path from this sender - if (std::any_of(Self->ForceTraversals.begin(), Self->ForceTraversals.end(), - [this, &pathId, &cookie](const TForceTraversal& elem) { + // check existing force traversal with the same cookie and path + auto forceTraversal = std::find_if(Self->ForceTraversals.begin(), Self->ForceTraversals.end(), + [&pathId, &operationId](const TForceTraversal& elem) { return elem.PathId == pathId - && elem.Cookie == cookie - && elem.ReplyToActorId == ReplyToActorId - ;})) { + && elem.OperationId == operationId;}); + + // update existing force traversal + if (forceTraversal != Self->ForceTraversals.end()) { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute. Update existing force traversal. PathId " << pathId << " , ReplyToActorId " << ReplyToActorId); + forceTraversal->ReplyToActorId = ReplyToActorId; return true; } + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute. Create new force traversal operation for pathId " << pathId); + // create new force trasersal TForceTraversal operation { - .OperationId = Self->NextForceTraversalOperationId, - .Cookie = cookie, + .OperationId = operationId, .PathId = pathId, .ColumnTags = columnTags, .Types = types, @@ -66,8 +70,6 @@ struct TStatisticsAggregator::TTxAnalyzeTable : public TTxBase { */ } - Self->PersistNextForceTraversalOperationId(db); - return true; } @@ -77,8 +79,6 @@ struct TStatisticsAggregator::TTxAnalyzeTable : public TTxBase { }; void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyze::TPtr& ev) { - ++NextForceTraversalOperationId; - Execute(new TTxAnalyzeTable(this, ev->Get()->Record, ev->Sender), TActivationContext::AsActorContext()); } diff --git a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp index 6d9d165def1c..b9f7fc597fa4 100644 --- a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp +++ b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp @@ -5,15 +5,13 @@ namespace NKikimr::NStat { struct TStatisticsAggregator::TTxFinishTraversal : public TTxBase { - ui64 OperationId; - ui64 Cookie; + TString OperationId; TPathId PathId; TActorId ReplyToActorId; TTxFinishTraversal(TSelf* self) : TTxBase(self) , OperationId(self->ForceTraversalOperationId) - , Cookie(self->ForceTraversalCookie) , PathId(self->TraversalTableId.PathId) , ReplyToActorId(self->ForceTraversalReplyToActorId) {} @@ -43,12 +41,12 @@ struct TStatisticsAggregator::TTxFinishTraversal : public TTxBase { if (operationsRemain) { SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete. Don't send TEvAnalyzeResponse. " << - "There are pending operations, Cookie " << Cookie << " , ActorId=" << ReplyToActorId); + "There are pending operations, OperationId " << OperationId << " , ActorId=" << ReplyToActorId); } else { SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete. " << - "Send TEvAnalyzeResponse, Cookie=" << Cookie << ", ActorId=" << ReplyToActorId); + "Send TEvAnalyzeResponse, OperationId=" << OperationId << ", ActorId=" << ReplyToActorId); auto response = std::make_unique(); - response->Record.SetCookie(Cookie); + response->Record.SetOperationId(OperationId); ctx.Send(ReplyToActorId, response.release()); } } diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index f50ea80d2705..e4012d7bbe39 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -55,15 +55,10 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal start key"); break; case Schema::SysParam_ForceTraversalOperationId: { - Self->ForceTraversalOperationId = FromString(value); + Self->ForceTraversalOperationId = value; SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal operation id: " << value); break; } - case Schema::SysParam_ForceTraversalCookie: { - Self->ForceTraversalCookie = FromString(value); - SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal cookie: " << value); - break; - } case Schema::SysParam_TraversalTableOwnerId: Self->TraversalTableId.PathId.OwnerId = FromString(value); SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal table owner id: " @@ -90,11 +85,6 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal start time: " << us); break; } - case Schema::SysParam_NextForceTraversalOperationId: { - Self->NextForceTraversalOperationId = FromString(value); - SA_LOG_D("[" << Self->TabletID() << "] Loaded next traversal operation id: " << value); - break; - } case Schema::SysParam_TraversalIsColumnTable: { Self->TraversalIsColumnTable = FromString(value); SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal IsColumnTable: " << value); @@ -217,7 +207,7 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { ui64 operationId = rowset.GetValue(); ui64 ownerId = rowset.GetValue(); ui64 localPathId = rowset.GetValue(); - ui64 cookie = rowset.GetValue(); + TString cookie = rowset.GetValue(); TString columnTags = rowset.GetValue(); TString types = rowset.GetValue(); diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 85107784bc83..5b90aa0362f8 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -46,9 +46,9 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto& runtime = *env.GetServer().GetRuntime(); auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; - AnalyzeTable(runtime, tableInfo.PathId, tableInfo.ShardIds[0]); + AnalyzeTable(runtime, tableInfo.ShardIds[0], tableInfo.PathId); - Analyze(runtime, {tableInfo.PathId}, tableInfo.SaTabletId); + Analyze(runtime, tableInfo.SaTabletId, {tableInfo.PathId}); } Y_UNIT_TEST(AnalyzeAnalyzeOneColumnTableSpecificColumns) { @@ -56,7 +56,7 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto& runtime = *env.GetServer().GetRuntime(); auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; - Analyze(runtime, {{tableInfo.PathId, {1, 2}}}, tableInfo.SaTabletId); + Analyze(runtime, tableInfo.SaTabletId, {{tableInfo.PathId, {1, 2}}}); } Y_UNIT_TEST(AnalyzeTwoColumnTables) { @@ -64,8 +64,59 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto& runtime = *env.GetServer().GetRuntime(); auto tableInfos = CreateDatabaseTables(env, 2, 1); - Analyze(runtime, {tableInfos[0].PathId, tableInfos[1].PathId}, tableInfos[0].SaTabletId); + Analyze(runtime, tableInfos[0].SaTabletId, {tableInfos[0].PathId, tableInfos[1].PathId}); } + + Y_UNIT_TEST(AnalyzeStatus) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + auto schemeShardStatsBlocker = runtime.AddObserver([&](auto& ev) { + ev.Reset(); + }); + + auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + + const TString operationId = "operationId"; + + AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); + + auto analyzeRequest = MakeAnalyzeRequest({{tableInfo.PathId, {1, 2}}}, operationId); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest.release()); + + AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); + + schemeShardStatsBlocker.Remove(); + + auto analyzeResonse = runtime.GrabEdgeEventRethrow(sender); + UNIT_ASSERT_VALUES_EQUAL(analyzeResonse->Get()->Record.GetOperationId(), operationId); + + AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); + } + + Y_UNIT_TEST(AnalyzeSameOperationId) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + + auto sender = runtime.AllocateEdgeActor(); + const TString operationId = "operationId"; + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}, operationId); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}, operationId); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + auto response1 = runtime.GrabEdgeEventRethrow(sender); + UNIT_ASSERT(response1); + UNIT_ASSERT_VALUES_EQUAL(response1->Get()->Record.GetOperationId(), operationId); + + auto response2 = runtime.GrabEdgeEventRethrow(sender, TDuration::Seconds(5)); + UNIT_ASSERT(!response2); + } + } } // NStat diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp index 3f77cb85e612..d491aad97420 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp @@ -36,7 +36,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { runtime.SimulateSleep(TDuration::Seconds(30)); - Analyze(runtime, {{pathId}}, saTabletId); + Analyze(runtime, saTabletId, {{pathId}}); ValidateCountMin(runtime, pathId); } @@ -62,7 +62,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1", nullptr, &saTabletId1); auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2"); - Analyze(runtime, {pathId1, pathId2}, saTabletId1); + Analyze(runtime, saTabletId1, {pathId1, pathId2}); ValidateCountMin(runtime, pathId1); ValidateCountMin(runtime, pathId2); @@ -92,7 +92,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { runtime.SimulateSleep(TDuration::Seconds(5)); init2Thread.join(); - Analyze(runtime, {pathId}, saTabletId); + Analyze(runtime, saTabletId, {pathId}); runtime.SimulateSleep(TDuration::Seconds(60)); diff --git a/ydb/core/statistics/database/database.cpp b/ydb/core/statistics/database/database.cpp index f3f591cced1c..cb3e9dab7707 100644 --- a/ydb/core/statistics/database/database.cpp +++ b/ydb/core/statistics/database/database.cpp @@ -32,7 +32,9 @@ class TStatisticsTableCreator : public TActorBootstrapped&& columnTags, std::vector&& data) - : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}) + : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}, {}, true) , PathId(pathId) , StatType(statType) , ColumnTags(std::move(columnTags)) @@ -164,7 +166,7 @@ class TLoadStatisticsQuery : public NKikimr::TQueryBase { public: TLoadStatisticsQuery(const TPathId& pathId, ui64 statType, ui32 columnTag, ui64 cookie) - : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}) + : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}, {}, true) , PathId(pathId) , StatType(statType) , ColumnTag(columnTag) @@ -246,7 +248,7 @@ class TDeleteStatisticsQuery : public NKikimr::TQueryBase { public: TDeleteStatisticsQuery(const TPathId& pathId) - : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}) + : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}, {}, true) , PathId(pathId) { } diff --git a/ydb/core/statistics/database/ut/ut_database.cpp b/ydb/core/statistics/database/ut/ut_database.cpp index cb097b43283e..50b19d056a91 100644 --- a/ydb/core/statistics/database/ut/ut_database.cpp +++ b/ydb/core/statistics/database/ut/ut_database.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include namespace NKikimr::NStat { @@ -87,6 +89,41 @@ Y_UNIT_TEST_SUITE(StatisticsSaveLoad) { auto loadResponseA = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(!loadResponseA->Get()->Success); } + + Y_UNIT_TEST(ForbidAccess) { + TTestEnv env(1, 1); + auto init = [&] () { + CreateDatabase(env, "Database"); + CreateUniformTable(env, "Database", "Table"); + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + runtime.SimulateSleep(TDuration::Seconds(10)); + initThread.join(); + + NYdb::EStatus status; + auto test = [&] () { + auto driverConfig = NYdb::TDriverConfig() + .SetEndpoint(env.GetEndpoint()) + .SetAuthToken("user@builtin"); + auto driver = NYdb::TDriver(driverConfig); + auto db = NYdb::NTable::TTableClient(driver); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteDataQuery(R"( + SELECT * FROM `/Root/Database/.metadata/_statistics`; + )", NYdb::NTable::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + status = result.GetStatus(); + }; + std::thread testThread(test); + + runtime.SimulateSleep(TDuration::Seconds(10)); + testThread.join(); + + UNIT_ASSERT_VALUES_EQUAL(status, NYdb::EStatus::SCHEME_ERROR); + } + } } // NKikimr::NStat diff --git a/ydb/core/statistics/service/http_request.cpp b/ydb/core/statistics/service/http_request.cpp index 6e7b28081f7d..1c00bf0e39b7 100644 --- a/ydb/core/statistics/service/http_request.cpp +++ b/ydb/core/statistics/service/http_request.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -9,10 +10,16 @@ namespace NKikimr { namespace NStat { +TString MakeOperationId() { + TULIDGenerator ulidGen; + return ulidGen.Next(TActivationContext::Now()).ToBinary(); +} + THttpRequest::THttpRequest(EType type, const TString& path, TActorId replyToActorId) : Type(type) , Path(path) , ReplyToActorId(replyToActorId) + , OperationId(MakeOperationId() ) {} void THttpRequest::Bootstrap() { @@ -102,6 +109,14 @@ void THttpRequest::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& void THttpRequest::Handle(TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev) { auto& record = ev->Get()->Record; + + if (record.GetOperationId() != OperationId) { + ALOG_ERROR(NKikimrServices::STATISTICS, + "THttpRequest, TEvAnalyzeStatusResponse has operationId=" << record.GetOperationId() + << " , but expected " << OperationId); + HttpReply("Wrong OperationId"); + } + switch (record.GetStatus()) { case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_UNSPECIFIED: HttpReply("Status is unspecified"); @@ -131,6 +146,7 @@ void THttpRequest::ResolveSuccess() { if (Type == ANALYZE) { auto analyze = std::make_unique(); auto& record = analyze->Record; + record.SetOperationId(OperationId); PathIdFromPathId(PathId, record.AddTables()->MutablePathId()); Send(MakePipePerNodeCacheID(false), @@ -140,7 +156,7 @@ void THttpRequest::ResolveSuccess() { } else { auto getStatus = std::make_unique(); auto& record = getStatus->Record; - PathIdFromPathId(PathId, record.MutablePathId()); + record.SetOperationId(OperationId); Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(getStatus.release(), StatisticsAggregatorId, true)); diff --git a/ydb/core/statistics/service/http_request.h b/ydb/core/statistics/service/http_request.h index 874725440898..24f6bddfb440 100644 --- a/ydb/core/statistics/service/http_request.h +++ b/ydb/core/statistics/service/http_request.h @@ -55,6 +55,7 @@ class THttpRequest : public NActors::TActorBootstrapped { const TActorId ReplyToActorId; TPathId PathId; + TString OperationId; ui64 StatisticsAggregatorId = 0; static const ui64 FirstRoundCookie = 1; diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index 8005f02a00da..a95e14037e0f 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -1,6 +1,5 @@ #include "ut_common.h" -#include #include #include @@ -337,24 +336,27 @@ void TAnalyzedTable::ToProto(NKikimrStat::TTable& tableProto) const { tableProto.MutableColumnTags()->Add(ColumnTags.begin(), ColumnTags.end()); } - -void Analyze(TTestActorRuntime& runtime, const std::vector& tables, ui64 saTabletId) { - const ui64 cookie = 555; +std::unique_ptr MakeAnalyzeRequest(const std::vector& tables, const TString operationId) { auto ev = std::make_unique(); NKikimrStat::TEvAnalyze& record = ev->Record; - record.SetCookie(cookie); + record.SetOperationId(operationId); record.AddTypes(NKikimrStat::EColumnStatisticType::TYPE_COUNT_MIN_SKETCH); for (const TAnalyzedTable& table : tables) table.ToProto(*record.AddTables()); + return ev; +} + +void Analyze(TTestActorRuntime& runtime, ui64 saTabletId, const std::vector& tables, const TString operationId) { + auto ev = MakeAnalyzeRequest(tables, operationId); auto sender = runtime.AllocateEdgeActor(); runtime.SendToPipe(saTabletId, sender, ev.release()); auto evResponse = runtime.GrabEdgeEventRethrow(sender); - UNIT_ASSERT_VALUES_EQUAL(evResponse->Get()->Record.GetCookie(), cookie); + UNIT_ASSERT_VALUES_EQUAL(evResponse->Get()->Record.GetOperationId(), operationId); } -void AnalyzeTable(TTestActorRuntime& runtime, const TAnalyzedTable& table, ui64 shardTabletId) { +void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyzedTable& table) { auto ev = std::make_unique(); auto& record = ev->Record; table.ToProto(*record.MutableTable()); @@ -365,6 +367,18 @@ void AnalyzeTable(TTestActorRuntime& runtime, const TAnalyzedTable& table, ui64 runtime.GrabEdgeEventRethrow(sender); } +void AnalyzeStatus(TTestActorRuntime& runtime, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus) { + auto analyzeStatusRequest = std::make_unique(); + analyzeStatusRequest->Record.SetOperationId(operationId); + auto sender = runtime.AllocateEdgeActor(); + runtime.SendToPipe(saTabletId, sender, analyzeStatusRequest.release()); + + auto analyzeStatusResponse = runtime.GrabEdgeEventRethrow(sender); + UNIT_ASSERT(analyzeStatusResponse); + UNIT_ASSERT_VALUES_EQUAL(analyzeStatusResponse->Get()->Record.GetOperationId(), operationId); + UNIT_ASSERT_VALUES_EQUAL(analyzeStatusResponse->Get()->Record.GetStatus(), expectedStatus); +} + } // NStat } // NKikimr diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index 532ce0250e10..20126bd5de29 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -87,8 +89,12 @@ struct TAnalyzedTable { void ToProto(NKikimrStat::TTable& tableProto) const; }; -void Analyze(TTestActorRuntime& runtime, const std::vector& table, ui64 saTabletId); -void AnalyzeTable(TTestActorRuntime& runtime, const TAnalyzedTable& table, ui64 shardTabletId); +std::unique_ptr MakeAnalyzeRequest(const std::vector& tables, const TString operationId = "operationId"); + +void Analyze(TTestActorRuntime& runtime, ui64 saTabletId, const std::vector& table, const TString operationId = "operationId"); +void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyzedTable& table); +void AnalyzeStatus(TTestActorRuntime& runtime, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus); + } // namespace NStat } // namespace NKikimr diff --git a/ydb/library/query_actor/query_actor.cpp b/ydb/library/query_actor/query_actor.cpp index 9c36858d1949..5a51e2b62009 100644 --- a/ydb/library/query_actor/query_actor.cpp +++ b/ydb/library/query_actor/query_actor.cpp @@ -99,10 +99,11 @@ TQueryBase::TEvQueryBasePrivate::TEvCommitTransactionResponse::TEvCommitTransact //// TQueryBase -TQueryBase::TQueryBase(ui64 logComponent, TString sessionId, TString database) +TQueryBase::TQueryBase(ui64 logComponent, TString sessionId, TString database, bool isSystemUser) : LogComponent(logComponent) , Database(std::move(database)) , SessionId(std::move(sessionId)) + , IsSystemUser(isSystemUser) {} void TQueryBase::Registered(NActors::TActorSystem* sys, const NActors::TActorId& owner) { @@ -221,7 +222,13 @@ void TQueryBase::RunDataQuery(const TString& sql, NYdb::TParamsBuilder* params, txControlProto->set_commit_tx(true); } - Subscribe(DoLocalRpc(std::move(request), Database, Nothing(), TActivationContext::ActorSystem(), true)); + TMaybe token = Nothing(); + if (IsSystemUser) { + token = NACLib::TSystemUsers::Metadata().SerializeAsString(); + } + + Subscribe( + DoLocalRpc(std::move(request), Database, token, TActivationContext::ActorSystem(), true)); } void TQueryBase::Handle(TEvQueryBasePrivate::TEvDataQueryResult::TPtr& ev) { diff --git a/ydb/library/query_actor/query_actor.h b/ydb/library/query_actor/query_actor.h index 3d1fce5c2b90..ecb7fff59746 100644 --- a/ydb/library/query_actor/query_actor.h +++ b/ydb/library/query_actor/query_actor.h @@ -115,7 +115,7 @@ class TQueryBase : public NActors::TActorBootstrapped { public: static constexpr char ActorName[] = "SQL_QUERY"; - explicit TQueryBase(ui64 logComponent, TString sessionId = {}, TString database = {}); + explicit TQueryBase(ui64 logComponent, TString sessionId = {}, TString database = {}, bool isSystemUser = false); void Bootstrap(); @@ -199,6 +199,7 @@ class TQueryBase : public NActors::TActorBootstrapped { const ui64 LogComponent; TString Database; TString SessionId; + bool IsSystemUser = false; TString TxId; bool DeleteSession = false; bool RunningQuery = false; diff --git a/ydb/library/table_creator/table_creator.cpp b/ydb/library/table_creator/table_creator.cpp index 769946ce258a..f208b14c8d00 100644 --- a/ydb/library/table_creator/table_creator.cpp +++ b/ydb/library/table_creator/table_creator.cpp @@ -34,12 +34,14 @@ using TTableCreatorRetryPolicy = IRetryPolicy; TVector columns, TVector keyColumns, NKikimrServices::EServiceKikimr logService, - TMaybe ttlSettings = Nothing()) + TMaybe ttlSettings = Nothing(), + bool isSystemUser = false) : PathComponents(std::move(pathComponents)) , Columns(std::move(columns)) , KeyColumns(std::move(keyColumns)) , LogService(logService) , TtlSettings(std::move(ttlSettings)) + , IsSystemUser(isSystemUser) , LogPrefix("Table " + TableName() + " updater. ") { Y_ABORT_UNLESS(!PathComponents.empty()); @@ -86,8 +88,8 @@ using TTableCreatorRetryPolicy = IRetryPolicy; pathComponents.emplace_back(PathComponents[i]); } modifyScheme.SetWorkingDir(CanonizePath(pathComponents)); - LOG_DEBUG_S(*TlsActivationContext, LogService, - LogPrefix << "Full table path:" << modifyScheme.GetWorkingDir() << "/" << TableName()); + TString fullPath = modifyScheme.GetWorkingDir() + "/" + TableName(); + LOG_DEBUG_S(*TlsActivationContext, LogService, LogPrefix << "Full table path:" << fullPath); modifyScheme.SetOperationType(OperationType); modifyScheme.SetInternal(true); modifyScheme.SetAllowAccessToPrivatePaths(true); @@ -108,6 +110,9 @@ using TTableCreatorRetryPolicy = IRetryPolicy; if (TtlSettings) { tableDesc->MutableTTLSettings()->CopyFrom(*TtlSettings); } + if (IsSystemUser) { + request->Record.SetUserToken(NACLib::TSystemUsers::Metadata().SerializeAsString()); + } Send(MakeTxProxyID(), std::move(request)); } @@ -376,6 +381,7 @@ using TTableCreatorRetryPolicy = IRetryPolicy; const TVector KeyColumns; NKikimrServices::EServiceKikimr LogService; const TMaybe TtlSettings; + bool IsSystemUser = false; NKikimrSchemeOp::EOperationType OperationType = NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable; NActors::TActorId Owner; NActors::TActorId SchemePipeActorId; @@ -473,10 +479,11 @@ NActors::IActor* CreateTableCreator( TVector columns, TVector keyColumns, NKikimrServices::EServiceKikimr logService, - TMaybe ttlSettings) + TMaybe ttlSettings, + bool isSystemUser) { return new TTableCreator(std::move(pathComponents), std::move(columns), - std::move(keyColumns), logService, std::move(ttlSettings)); + std::move(keyColumns), logService, std::move(ttlSettings), isSystemUser); } } // namespace NKikimr diff --git a/ydb/library/table_creator/table_creator.h b/ydb/library/table_creator/table_creator.h index d87dbafb5351..90d61e6ebc76 100644 --- a/ydb/library/table_creator/table_creator.h +++ b/ydb/library/table_creator/table_creator.h @@ -76,6 +76,7 @@ NActors::IActor* CreateTableCreator( TVector columns, TVector keyColumns, NKikimrServices::EServiceKikimr logService, - TMaybe ttlSettings = Nothing()); + TMaybe ttlSettings = Nothing(), + bool isSystemUser = false); } // namespace NKikimr From bd6322c0e848e19865b0796d574ffded41a35286 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:35:11 +0000 Subject: [PATCH 03/17] Statistics: Send TEvAnalyzeTable to shards (#7880) --- .../counters_statistics_aggregator.proto | 2 + ydb/core/protos/statistics.proto | 14 +- ydb/core/quoter/kesus_quoter_ut.cpp | 9 + .../statistics/aggregator/aggregator_impl.cpp | 227 ++++++--- .../statistics/aggregator/aggregator_impl.h | 73 ++- ydb/core/statistics/aggregator/schema.h | 43 +- ydb/core/statistics/aggregator/tx_analyze.cpp | 101 ++++ .../aggregator/tx_analyze_table.cpp | 85 ---- .../aggregator/tx_analyze_table_request.cpp | 79 ++++ .../aggregator/tx_analyze_table_response.cpp | 65 +++ .../aggregator/tx_finish_trasersal.cpp | 15 +- ydb/core/statistics/aggregator/tx_init.cpp | 103 ++-- ydb/core/statistics/aggregator/tx_resolve.cpp | 81 +++- .../tx_response_tablet_distribution.cpp | 59 ++- .../aggregator/tx_schedule_traversal.cpp | 27 +- .../aggregator/ut/ut_analyze_datashard.cpp | 2 +- ydb/core/statistics/aggregator/ya.make | 4 +- ydb/core/statistics/service/service_impl.cpp | 226 +++++---- ydb/core/statistics/service/ut/ut_service.cpp | 445 ++++++++++++------ ydb/core/testlib/actors/block_events.cpp | 1 + ydb/core/testlib/actors/block_events.h | 82 ++++ ydb/core/testlib/actors/test_runtime.h | 32 +- ydb/core/testlib/actors/test_runtime_ut.cpp | 206 +++++++- ydb/core/testlib/actors/ya.make | 3 + .../columnshard/columnshard__statistics.cpp | 6 +- .../datashard/datashard_ut_read_iterator.cpp | 53 +-- .../tx/datashard/datashard_ut_volatile.cpp | 6 +- ydb/library/actors/testlib/test_runtime.cpp | 38 +- .../partition_writer_cache_actor_fixture.cpp | 3 + 29 files changed, 1502 insertions(+), 588 deletions(-) create mode 100644 ydb/core/statistics/aggregator/tx_analyze.cpp delete mode 100644 ydb/core/statistics/aggregator/tx_analyze_table.cpp create mode 100644 ydb/core/statistics/aggregator/tx_analyze_table_request.cpp create mode 100644 ydb/core/statistics/aggregator/tx_analyze_table_response.cpp create mode 100644 ydb/core/testlib/actors/block_events.cpp create mode 100644 ydb/core/testlib/actors/block_events.h diff --git a/ydb/core/protos/counters_statistics_aggregator.proto b/ydb/core/protos/counters_statistics_aggregator.proto index 48fdd0a4bfaf..eb263bab214c 100644 --- a/ydb/core/protos/counters_statistics_aggregator.proto +++ b/ydb/core/protos/counters_statistics_aggregator.proto @@ -20,4 +20,6 @@ enum ETxTypes { TXTYPE_AGGR_STAT_RESPONSE = 10 [(TxTypeOpts) = {Name: "TxAggregateStatisticsResponse"}]; TXTYPE_RESPONSE_TABLET_DISTRIBUTION = 11 [(TxTypeOpts) = {Name: "TxResponseTabletDistribution"}]; TXTYPE_ACK_TIMEOUT = 12 [(TxTypeOpts) = {Name: "TxAckTimeout"}]; + TXTYPE_ANALYZE_TABLE_REQUEST = 13 [(TxTypeOpts) = {Name: "TxAnalyzeTableRequest"}]; + TXTYPE_ANALYZE_TABLE_RESPONSE = 14 [(TxTypeOpts) = {Name: "TxAnalyzeTableResponse"}]; } diff --git a/ydb/core/protos/statistics.proto b/ydb/core/protos/statistics.proto index 439feacc2c4a..08701e5ba138 100644 --- a/ydb/core/protos/statistics.proto +++ b/ydb/core/protos/statistics.proto @@ -111,13 +111,16 @@ message TEvAnalyzeStatusResponse { // SA -> Shard message TEvAnalyzeTable { - optional TTable Table = 1; // analyzed table - repeated EColumnStatisticType Types = 2; // list of statistics types requested. Empty means asking for all available. + optional bytes OperationId = 1; // unique identifier to match response item + optional TTable Table = 2; // analyzed table + repeated EColumnStatisticType Types = 3; // list of statistics types requested. Empty means asking for all available. } // Shard -> SA message TEvAnalyzeTableResponse { - optional NKikimrProto.TPathID PathId = 1; + optional bytes OperationId = 1; + optional NKikimrProto.TPathID PathId = 2; + optional fixed64 ShardTabletId = 3; } @@ -145,8 +148,9 @@ message TEvStatisticsResponse { enum EStatus { STATUS_UNSPECIFIED = 0; STATUS_SUCCESS = 1; - STATUS_ABORTED = 2; - STATUS_ERROR = 3; + STATUS_PROCESSING = 2; + STATUS_ABORTED = 3; + STATUS_ERROR = 4; } optional EStatus Status = 2; optional fixed64 ShardTabletId = 3; diff --git a/ydb/core/quoter/kesus_quoter_ut.cpp b/ydb/core/quoter/kesus_quoter_ut.cpp index 1e2e20846f03..3f79f4bef98a 100644 --- a/ydb/core/quoter/kesus_quoter_ut.cpp +++ b/ydb/core/quoter/kesus_quoter_ut.cpp @@ -237,6 +237,9 @@ Y_UNIT_TEST_SUITE(KesusProxyTest) { return setup.GetPipeFactory().GetPipesCreatedCount() >= 2; }; setup.GetRuntime().DispatchEvents(reconnected); + + // Dispatch some events to let poison pill reach the mock + setup.GetRuntime().SimulateSleep(TDuration::Zero()); } Y_UNIT_TEST(ReconnectsWithKesusWhenPipeDestroyed) { @@ -249,6 +252,9 @@ Y_UNIT_TEST_SUITE(KesusProxyTest) { setup.SendDestroyed(pipeMock); setup.WaitPipesCreated(2); + + // Dispatch some events to let poison pill reach the mock + setup.GetRuntime().SimulateSleep(TDuration::Zero()); } Y_UNIT_TEST(RejectsNotCanonizedResourceName) { @@ -391,6 +397,9 @@ Y_UNIT_TEST_SUITE(KesusProxyTest) { setup.WaitEvent(); setup.SendCloseSession("res", 42); setup.WaitEvent(); + + // Dispatch some events to let pending events reach their destinations + setup.GetRuntime().SimulateSleep(TDuration::Zero()); } void SendsProxySessionOnce(bool onSuccess) { diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 12ba2dc9c1c4..2a087684293f 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -398,7 +398,7 @@ size_t TStatisticsAggregator::PropagatePart(const std::vector& nodeIds, } void TStatisticsAggregator::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - if (!TraversalTableId.PathId) { + if (!TraversalPathId) { return; } auto tabletId = ev->Get()->TabletId; @@ -443,8 +443,8 @@ void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyzeStatus::TPtr& ev) { if (ForceTraversalOperationId == operationId) { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS); } else { - if (std::any_of(ForceTraversals.begin(), ForceTraversals.end(), - [&operationId](const TForceTraversal& elem) { return elem.OperationId == operationId;})) { + auto forceTraversalOperation = ForceTraversalOperation(operationId); + if (forceTraversalOperation) { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); } else { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); @@ -461,11 +461,7 @@ void TStatisticsAggregator::Handle(TEvPrivate::TEvRequestDistribution::TPtr&) { ++HiveRequestRound; auto reqDistribution = std::make_unique(); - reqDistribution->Record.MutableTabletIds()->Reserve(TabletsForReqDistribution.size()); - for (auto& tablet : TabletsForReqDistribution) { - reqDistribution->Record.AddTabletIds(tablet); - } - + reqDistribution->Record.MutableTabletIds()->Add(TabletsForReqDistribution.begin(), TabletsForReqDistribution.end()); Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(reqDistribution.release(), HiveId, true)); } @@ -485,9 +481,13 @@ void TStatisticsAggregator::InitializeStatisticsTable() { } void TStatisticsAggregator::Navigate() { + Y_ABORT_UNLESS(NavigateType == ENavigateType::Traversal && !NavigateAnalyzeOperationId + || NavigateType == ENavigateType::Analyze && NavigateAnalyzeOperationId); + Y_ABORT_UNLESS(NavigatePathId); + using TNavigate = NSchemeCache::TSchemeCacheNavigate; TNavigate::TEntry entry; - entry.TableId = TraversalTableId; + entry.TableId = NavigatePathId; entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; entry.Operation = TNavigate::OpTable; @@ -498,12 +498,16 @@ void TStatisticsAggregator::Navigate() { } void TStatisticsAggregator::Resolve() { + Y_ABORT_UNLESS(NavigateType == ENavigateType::Traversal && !NavigateAnalyzeOperationId + || NavigateType == ENavigateType::Analyze && NavigateAnalyzeOperationId); + Y_ABORT_UNLESS(NavigatePathId); + ++ResolveRound; TVector plusInf; TTableRange range(TraversalStartKey.GetCells(), true, plusInf, true, false); auto keyDesc = MakeHolder( - TraversalTableId, range, TKeyDesc::ERowOperation::Read, KeyColumnTypes, Columns); + NavigatePathId, range, TKeyDesc::ERowOperation::Read, KeyColumnTypes, Columns); auto request = std::make_unique(); request->ResultSet.emplace_back(std::move(keyDesc)); @@ -521,8 +525,8 @@ void TStatisticsAggregator::ScanNextDatashardRange() { auto request = std::make_unique(); auto& record = request->Record; auto* path = record.MutableTable()->MutablePathId(); - path->SetOwnerId(TraversalTableId.PathId.OwnerId); - path->SetLocalId(TraversalTableId.PathId.LocalPathId); + path->SetOwnerId(TraversalPathId.OwnerId); + path->SetLocalId(TraversalPathId.LocalPathId); record.SetStartKey(TraversalStartKey.GetBuffer()); Send(MakePipePerNodeCacheID(false), @@ -557,7 +561,7 @@ void TStatisticsAggregator::SaveStatisticsToTable() { data.push_back(strSketch); } - Register(CreateSaveStatisticsQuery(TraversalTableId.PathId, EStatType::COUNT_MIN_SKETCH, + Register(CreateSaveStatisticsQuery(TraversalPathId, EStatType::COUNT_MIN_SKETCH, std::move(columnTags), std::move(data))); } @@ -569,33 +573,79 @@ void TStatisticsAggregator::DeleteStatisticsFromTable() { PendingDeleteStatistics = false; - Register(CreateDeleteStatisticsQuery(TraversalTableId.PathId)); + Register(CreateDeleteStatisticsQuery(TraversalPathId)); } -void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { - if (!IsSchemeshardSeen) { - SA_LOG_T("[" << TabletID() << "] No info from schemeshard"); +void TStatisticsAggregator::ScheduleNextAnalyze(NIceDb::TNiceDb& db) { + Y_UNUSED(db); + if (ForceTraversals.empty()) { + SA_LOG_T("[" << TabletID() << "] ScheduleNextAnalyze. Empty ForceTraversals"); return; + } + SA_LOG_D("[" << TabletID() << "] ScheduleNextAnalyze"); + + for (TForceTraversalOperation& operation : ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + if (operationTable.Status == TForceTraversalTable::EStatus::None) { + std::optional isColumnTable = IsColumnTable(operationTable.PathId); + if (!isColumnTable) { + ForceTraversalOperationId = operation.OperationId; + TraversalPathId = operationTable.PathId; + DeleteStatisticsFromTable(); + return; + } + + if (*isColumnTable) { + NavigateAnalyzeOperationId = operation.OperationId; + NavigatePathId = operationTable.PathId; + Navigate(); + return; + } else { + SA_LOG_D("[" << TabletID() << "] ScheduleNextAnalyze. Skip analyze for datashard table " << operationTable.PathId); + UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::AnalyzeFinished, operation.OperationId, operationTable, db); + return; + } + } + } + + SA_LOG_D("[" << TabletID() << "] ScheduleNextAnalyze. All the force traversal tables sent the requests. OperationId=" << operation.OperationId); + continue; } + SA_LOG_D("[" << TabletID() << "] ScheduleNextAnalyze. All the force traversal operations sent the requests."); +} + +void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal"); + TPathId pathId; - if (!ForceTraversals.empty() && !LastTraversalWasForce) { + if (!LastTraversalWasForce) { LastTraversalWasForce = true; - TForceTraversal& operation = ForceTraversals.front(); - pathId = operation.PathId; + for (TForceTraversalOperation& operation : ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + if (operationTable.Status == TForceTraversalTable::EStatus::AnalyzeFinished) { + UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::TraversalStarted, operation.OperationId, operationTable, db); + pathId = operationTable.PathId; + break; + } + } + + if (!pathId) { + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. All the force traversal tables sent the requests. OperationId=" << operation.OperationId); + continue; + } - ForceTraversalOperationId = operation.OperationId; - ForceTraversalColumnTags = operation.ColumnTags; - ForceTraversalTypes = operation.Types; - ForceTraversalReplyToActorId = operation.ReplyToActorId; + ForceTraversalOperationId = operation.OperationId; + } - PersistForceTraversal(db); + if (!pathId) { + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. All the force traversal operations sent the requests."); + } + } -// db.Table().Key(operation.OperationId, operation.PathId.OwnerId, operation.PathId.LocalPathId).Delete(); - ForceTraversals.pop_front(); - } else if (!ScheduleTraversalsByTime.Empty()){ + if (!pathId && !ScheduleTraversalsByTime.Empty()){ LastTraversalWasForce = false; auto* oldestTable = ScheduleTraversalsByTime.Top(); @@ -606,24 +656,26 @@ void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { } pathId = oldestTable->PathId; - } else { - SA_LOG_E("[" << TabletID() << "] No schedule traversal from schemeshard."); + } + + if (!pathId) { + SA_LOG_E("[" << TabletID() << "] No traversal from schemeshard."); return; } - auto itPath = ScheduleTraversals.find(pathId); - if (itPath != ScheduleTraversals.end()) { - TraversalIsColumnTable = itPath->second.IsColumnTable; - } else { - SA_LOG_E("[" << TabletID() << "] traversal path " << pathId << " is not known to schemeshard"); + TraversalPathId = pathId; + + std::optional isColumnTable = IsColumnTable(pathId); + if (!isColumnTable){ + DeleteStatisticsFromTable(); return; } - TraversalTableId.PathId = pathId; + TraversalIsColumnTable = *isColumnTable; SA_LOG_D("[" << TabletID() << "] Start " << LastTraversalWasForceString() - << " traversal for path " << pathId); + << " traversal navigate for path " << pathId); StartTraversal(db); } @@ -635,11 +687,12 @@ void TStatisticsAggregator::StartTraversal(NIceDb::TNiceDb& db) { TraversalStartKey = TSerializedCellVec(); PersistStartKey(db); + NavigatePathId = TraversalPathId; Navigate(); } void TStatisticsAggregator::FinishTraversal(NIceDb::TNiceDb& db) { - auto pathId = TraversalTableId.PathId; + auto pathId = TraversalPathId; auto pathIt = ScheduleTraversals.find(pathId); if (pathIt != ScheduleTraversals.end()) { @@ -653,6 +706,19 @@ void TStatisticsAggregator::FinishTraversal(NIceDb::TNiceDb& db) { } } + auto forceTraversalOperation = CurrentForceTraversalOperation(); + if (forceTraversalOperation) { + auto operationTable = CurrentForceTraversalTable(); + + UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::TraversalFinished, forceTraversalOperation->OperationId, *operationTable, db); + + bool tablesRemained = std::any_of(forceTraversalOperation->Tables.begin(), forceTraversalOperation->Tables.end(), + [](const TForceTraversalTable& elem) { return elem.Status != TForceTraversalTable::EStatus::TraversalFinished;}); + if (!tablesRemained) { + DeleteForceTraversalOperation(ForceTraversalOperationId, db); + } + } + ResetTraversalState(db); } @@ -660,14 +726,80 @@ TString TStatisticsAggregator::LastTraversalWasForceString() const { return LastTraversalWasForce ? "force" : "schedule"; } +TStatisticsAggregator::TForceTraversalOperation* TStatisticsAggregator::CurrentForceTraversalOperation() { + return ForceTraversalOperation(ForceTraversalOperationId); +} + +TStatisticsAggregator::TForceTraversalOperation* TStatisticsAggregator::ForceTraversalOperation(const TString& operationId) { + auto forceTraversalOperation = std::find_if(ForceTraversals.begin(), ForceTraversals.end(), + [operationId](const TForceTraversalOperation& elem) { return elem.OperationId == operationId;}); + + if (forceTraversalOperation == ForceTraversals.end()) { + return nullptr; + } else { + return &*forceTraversalOperation; + } +} + +std::optional TStatisticsAggregator::IsColumnTable(const TPathId& pathId) const { + Y_ABORT_UNLESS(IsSchemeshardSeen); + + auto itPath = ScheduleTraversals.find(pathId); + if (itPath != ScheduleTraversals.end()) { + bool ret = itPath->second.IsColumnTable; + SA_LOG_D("[" << TabletID() << "] IsColumnTable. Path " << pathId << " is " + << (ret ? "column" : "data") << " table."); + return ret; + } else { + SA_LOG_E("[" << TabletID() << "] IsColumnTable. traversal path " << pathId << " is not known to schemeshard"); + return {}; + } +} + +void TStatisticsAggregator::DeleteForceTraversalOperation(const TString& operationId, NIceDb::TNiceDb& db) { + db.Table().Key(ForceTraversalOperationId).Delete(); + + auto operation = ForceTraversalOperation(operationId); + for(const TForceTraversalTable& table : operation->Tables) { + db.Table().Key(operationId, table.PathId.OwnerId, table.PathId.LocalPathId).Delete(); + } + + ForceTraversals.remove_if([operationId](const TForceTraversalOperation& elem) { return elem.OperationId == operationId;}); +} + +TStatisticsAggregator::TForceTraversalTable* TStatisticsAggregator::ForceTraversalTable(const TString& operationId, const TPathId& pathId) { + for (TForceTraversalOperation& operation : ForceTraversals) { + if (operation.OperationId == operationId) { + for (TForceTraversalTable& operationTable : operation.Tables) { + if (operationTable.PathId == pathId) { + return &operationTable; + } + } + } + } + + return nullptr; +} + +TStatisticsAggregator::TForceTraversalTable* TStatisticsAggregator::CurrentForceTraversalTable() { + return ForceTraversalTable(ForceTraversalOperationId, TraversalPathId); +} + +void TStatisticsAggregator::UpdateForceTraversalTableStatus(const TForceTraversalTable::EStatus status, const TString& operationId, TStatisticsAggregator::TForceTraversalTable& table, NIceDb::TNiceDb& db) { + table.Status = status; + db.Table().Key(operationId, table.PathId.OwnerId, table.PathId.LocalPathId) + .Update(NIceDb::TUpdate((ui64)status)); +} + + void TStatisticsAggregator::PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value) { db.Table().Key(id).Update( NIceDb::TUpdate(value)); } void TStatisticsAggregator::PersistTraversal(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_TraversalTableOwnerId, ToString(TraversalTableId.PathId.OwnerId)); - PersistSysParam(db, Schema::SysParam_TraversalTableLocalPathId, ToString(TraversalTableId.PathId.LocalPathId)); + PersistSysParam(db, Schema::SysParam_TraversalTableOwnerId, ToString(TraversalPathId.OwnerId)); + PersistSysParam(db, Schema::SysParam_TraversalTableLocalPathId, ToString(TraversalPathId.LocalPathId)); PersistSysParam(db, Schema::SysParam_TraversalStartTime, ToString(TraversalStartTime.MicroSeconds())); PersistSysParam(db, Schema::SysParam_TraversalIsColumnTable, ToString(TraversalIsColumnTable)); } @@ -676,30 +808,19 @@ void TStatisticsAggregator::PersistStartKey(NIceDb::TNiceDb& db) { PersistSysParam(db, Schema::SysParam_TraversalStartKey, TraversalStartKey.GetBuffer()); } -void TStatisticsAggregator::PersistForceTraversal(NIceDb::TNiceDb& db) { - PersistSysParam(db, Schema::SysParam_ForceTraversalOperationId, ToString(ForceTraversalOperationId)); - PersistSysParam(db, Schema::SysParam_ForceTraversalCookie, ForceTraversalOperationId); - PersistSysParam(db, Schema::SysParam_ForceTraversalColumnTags, ToString(ForceTraversalColumnTags)); - PersistSysParam(db, Schema::SysParam_ForceTraversalTypes, ToString(ForceTraversalTypes)); -} - void TStatisticsAggregator::PersistGlobalTraversalRound(NIceDb::TNiceDb& db) { PersistSysParam(db, Schema::SysParam_GlobalTraversalRound, ToString(GlobalTraversalRound)); } void TStatisticsAggregator::ResetTraversalState(NIceDb::TNiceDb& db) { ForceTraversalOperationId.clear(); - TraversalTableId.PathId = TPathId(); - ForceTraversalColumnTags.clear(); - ForceTraversalTypes.clear(); + TraversalPathId = {}; TraversalStartTime = TInstant::MicroSeconds(0); PersistTraversal(db); TraversalStartKey = TSerializedCellVec(); PersistStartKey(db); - ForceTraversalReplyToActorId = {}; - for (auto& [tag, _] : CountMinSketches) { db.Table().Key(tag).Delete(); } @@ -799,7 +920,7 @@ bool TStatisticsAggregator::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev str << "PendingRequests: " << PendingRequests.size() << Endl; str << "ProcessUrgentInFlight: " << ProcessUrgentInFlight << Endl << Endl; - str << "TraversalTableId: " << TraversalTableId << Endl; + str << "TraversalPathId: " << TraversalPathId << Endl; str << "Columns: " << Columns.size() << Endl; str << "DatashardRanges: " << DatashardRanges.size() << Endl; str << "CountMinSketches: " << CountMinSketches.size() << Endl << Endl; diff --git a/ydb/core/statistics/aggregator/aggregator_impl.h b/ydb/core/statistics/aggregator/aggregator_impl.h index 60dc0998a855..e6b63b79e37f 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.h +++ b/ydb/core/statistics/aggregator/aggregator_impl.h @@ -45,7 +45,9 @@ class TStatisticsAggregator : public TActor, public NTabl struct TTxInit; struct TTxConfigure; struct TTxSchemeShardStats; - struct TTxAnalyzeTable; + struct TTxAnalyze; + struct TTxAnalyzeTableRequest; + struct TTxAnalyzeTableResponse; struct TTxNavigate; struct TTxResolve; struct TTxDatashardScanResponse; @@ -65,6 +67,7 @@ class TStatisticsAggregator : public TActor, public NTabl EvRequestDistribution, EvResolve, EvAckTimeout, + EvSendAnalyze, EvEnd }; @@ -76,6 +79,7 @@ class TStatisticsAggregator : public TActor, public NTabl struct TEvScheduleTraversal : public TEventLocal {}; struct TEvRequestDistribution : public TEventLocal {}; struct TEvResolve : public TEventLocal {}; + struct TEvSendAnalyze : public TEventLocal {}; struct TEvAckTimeout : public TEventLocal { size_t SeqNo = 0; @@ -83,6 +87,7 @@ class TStatisticsAggregator : public TActor, public NTabl SeqNo = seqNo; } }; + }; private: @@ -109,6 +114,7 @@ class TStatisticsAggregator : public TActor, public NTabl void Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev); void Handle(TEvPrivate::TEvFastPropagateCheck::TPtr& ev); void Handle(TEvStatistics::TEvPropagateStatisticsResponse::TPtr& ev); + void Handle(TEvStatistics::TEvAnalyzeTableResponse::TPtr& ev); void Handle(TEvPrivate::TEvProcessUrgent::TPtr& ev); void Handle(TEvPrivate::TEvPropagateTimeout::TPtr& ev); @@ -135,6 +141,7 @@ class TStatisticsAggregator : public TActor, public NTabl void Handle(TEvPrivate::TEvRequestDistribution::TPtr& ev); void Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev); void Handle(TEvPrivate::TEvAckTimeout::TPtr& ev); + void Handle(TEvPrivate::TEvSendAnalyze::TPtr& ev); void InitializeStatisticsTable(); void Navigate(); @@ -145,15 +152,17 @@ class TStatisticsAggregator : public TActor, public NTabl void PersistSysParam(NIceDb::TNiceDb& db, ui64 id, const TString& value); void PersistTraversal(NIceDb::TNiceDb& db); - void PersistForceTraversal(NIceDb::TNiceDb& db); void PersistStartKey(NIceDb::TNiceDb& db); void PersistGlobalTraversalRound(NIceDb::TNiceDb& db); void ResetTraversalState(NIceDb::TNiceDb& db); + void ScheduleNextAnalyze(NIceDb::TNiceDb& db); void ScheduleNextTraversal(NIceDb::TNiceDb& db); void StartTraversal(NIceDb::TNiceDb& db); void FinishTraversal(NIceDb::TNiceDb& db); + std::optional IsColumnTable(const TPathId& pathId) const; + TString LastTraversalWasForceString() const; STFUNC(StateInit) { @@ -174,6 +183,7 @@ class TStatisticsAggregator : public TActor, public NTabl hFunc(TEvTabletPipe::TEvServerDisconnected, Handle); hFunc(TEvPrivate::TEvFastPropagateCheck, Handle); hFunc(TEvStatistics::TEvPropagateStatisticsResponse, Handle); + hFunc(TEvStatistics::TEvAnalyzeTableResponse, Handle); hFunc(TEvPrivate::TEvProcessUrgent, Handle); hFunc(TEvPrivate::TEvPropagateTimeout, Handle); @@ -193,6 +203,7 @@ class TStatisticsAggregator : public TActor, public NTabl hFunc(TEvPrivate::TEvRequestDistribution, Handle); hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); hFunc(TEvPrivate::TEvAckTimeout, Handle); + hFunc(TEvPrivate::TEvSendAnalyze, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -240,8 +251,6 @@ class TStatisticsAggregator : public TActor, public NTabl std::queue PendingRequests; bool ProcessUrgentInFlight = false; - TActorId ForceTraversalReplyToActorId = {}; - bool IsSchemeshardSeen = false; bool IsStatisticsTableCreated = false; bool PendingSaveStatistics = false; @@ -300,15 +309,25 @@ class TStatisticsAggregator : public TActor, public NTabl size_t KeepAliveSeqNo = 0; static constexpr TDuration KeepAliveTimeout = TDuration::Seconds(3); + static constexpr size_t SendAnalyzeCount = 100; + static constexpr TDuration SendAnalyzePeriod = TDuration::Seconds(1); + + enum ENavigateType { + Analyze, + Traversal + }; + ENavigateType NavigateType = Analyze; + TString NavigateAnalyzeOperationId; + TPathId NavigatePathId; + // alternate between forced and scheduled traversals bool LastTraversalWasForce = false; private: // stored in local db TString ForceTraversalOperationId; - TString ForceTraversalColumnTags; - TString ForceTraversalTypes; - TTableId TraversalTableId; + + TPathId TraversalPathId; bool TraversalIsColumnTable = false; TSerializedCellVec TraversalStartKey; TInstant TraversalStartTime; @@ -323,14 +342,48 @@ class TStatisticsAggregator : public TActor, public NTabl TTraversalsByTime; TTraversalsByTime ScheduleTraversalsByTime; - struct TForceTraversal { - TString OperationId; + + struct TAnalyzedShard { + ui64 ShardTabletId; + + enum class EStatus : ui8 { + None, + AnalyzeStarted, + AnalyzeFinished, + }; + EStatus Status = EStatus::None; + }; + + struct TForceTraversalTable { TPathId PathId; TString ColumnTags; + std::vector AnalyzedShards; + + enum class EStatus : ui8 { + None, + AnalyzeStarted, + AnalyzeFinished, + TraversalStarted, + TraversalFinished, + }; + EStatus Status = EStatus::None; + }; + struct TForceTraversalOperation { + TString OperationId; + std::vector Tables; TString Types; TActorId ReplyToActorId; }; - std::list ForceTraversals; + std::list ForceTraversals; + +private: + TForceTraversalOperation* CurrentForceTraversalOperation(); + TForceTraversalOperation* ForceTraversalOperation(const TString& operationId); + void DeleteForceTraversalOperation(const TString& operationId, NIceDb::TNiceDb& db); + + TForceTraversalTable* ForceTraversalTable(const TString& operationId, const TPathId& pathId); + TForceTraversalTable* CurrentForceTraversalTable(); + void UpdateForceTraversalTableStatus(const TForceTraversalTable::EStatus status, const TString& operationId, TStatisticsAggregator::TForceTraversalTable& table, NIceDb::TNiceDb& db); }; } // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/schema.h b/ydb/core/statistics/aggregator/schema.h index 036902199f1d..d897dc8cd8b5 100644 --- a/ydb/core/statistics/aggregator/schema.h +++ b/ydb/core/statistics/aggregator/schema.h @@ -45,32 +45,45 @@ struct TAggregatorSchema : NIceDb::Schema { IsColumnTable >; }; -/* - struct ForceTraversals : Table<5> { - struct OperationId : Column<1, NScheme::NTypeIds::Uint64> {}; + + // struct ForceTraversals : Table<5> + + struct ForceTraversalOperations : Table<6> { + struct OperationId : Column<1, NScheme::NTypeIds::String> {}; + struct Types : Column<2, NScheme::NTypeIds::String> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + OperationId, + Types + >; + }; + + struct ForceTraversalTables : Table<7> { + struct OperationId : Column<1, NScheme::NTypeIds::String> {}; struct OwnerId : Column<2, NScheme::NTypeIds::Uint64> {}; struct LocalPathId : Column<3, NScheme::NTypeIds::Uint64> {}; - struct Cookie : Column<4, NScheme::NTypeIds::String> {}; - struct ColumnTags : Column<5, NScheme::NTypeIds::String> {}; - struct Types : Column<6, NScheme::NTypeIds::String> {}; + struct ColumnTags : Column<4, NScheme::NTypeIds::String> {}; + struct Status : Column<5, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns< OperationId, OwnerId, LocalPathId, - Cookie, ColumnTags, - Types + Status >; }; -*/ + using TTables = SchemaTables< SysParams, BaseStatistics, ColumnStatistics, - ScheduleTraversals -// ForceTraversals + ScheduleTraversals, +// ForceTraversals, + ForceTraversalOperations, + ForceTraversalTables >; using TSettings = SchemaSettings< @@ -80,12 +93,12 @@ struct TAggregatorSchema : NIceDb::Schema { static constexpr ui64 SysParam_Database = 1; static constexpr ui64 SysParam_TraversalStartKey = 2; - static constexpr ui64 SysParam_ForceTraversalOperationId = 3; + // deprecated 3 static constexpr ui64 SysParam_TraversalTableOwnerId = 4; static constexpr ui64 SysParam_TraversalTableLocalPathId = 5; - static constexpr ui64 SysParam_ForceTraversalCookie = 6; - static constexpr ui64 SysParam_ForceTraversalColumnTags = 7; - static constexpr ui64 SysParam_ForceTraversalTypes = 8; + // deprecated 6 + // deprecated 7 + // deprecated 8 static constexpr ui64 SysParam_TraversalStartTime = 9; // deprecated 10 static constexpr ui64 SysParam_TraversalIsColumnTable = 11; diff --git a/ydb/core/statistics/aggregator/tx_analyze.cpp b/ydb/core/statistics/aggregator/tx_analyze.cpp new file mode 100644 index 000000000000..87043ce0f02b --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze.cpp @@ -0,0 +1,101 @@ +#include "aggregator_impl.h" + +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyze : public TTxBase { + const NKikimrStat::TEvAnalyze& Record; + TActorId ReplyToActorId; + + TTxAnalyze(TSelf* self, const NKikimrStat::TEvAnalyze& record, TActorId replyToActorId) + : TTxBase(self) + , Record(record) + , ReplyToActorId(replyToActorId) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Execute. ReplyToActorId " << ReplyToActorId << " , Record " << Record); + + if (!Self->EnableColumnStatistics) { + return true; + } + + NIceDb::TNiceDb db(txc.DB); + + const TString operationId = Record.GetOperationId(); + + // check existing force traversal with the same OperationId + const auto existingOperation = Self->ForceTraversalOperation(operationId); + + // update existing force traversal + if (existingOperation) { + if (existingOperation->Tables.size() == Record.TablesSize()) { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Execute. Update existing force traversal. OperationId " << operationId << " , ReplyToActorId " << ReplyToActorId); + existingOperation->ReplyToActorId = ReplyToActorId; + return true; + } else { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Execute. Delete broken force traversal. OperationId " << operationId << " , ReplyToActorId " << ReplyToActorId); + Self->DeleteForceTraversalOperation(operationId, db); + } + } + + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Execute. Create new force traversal operation, OperationId=" << operationId); + const TString types = JoinVectorIntoString(TVector(Record.GetTypes().begin(), Record.GetTypes().end()), ","); + + // create new force trasersal + TForceTraversalOperation operation { + .OperationId = operationId, + .Tables = {}, + .Types = types, + .ReplyToActorId = ReplyToActorId + }; + + for (const auto& table : Record.GetTables()) { + const TPathId pathId = PathIdFromPathId(table.GetPathId()); + const TString columnTags = JoinVectorIntoString(TVector{table.GetColumnTags().begin(),table.GetColumnTags().end()},","); + const auto status = TForceTraversalTable::EStatus::None; + + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Execute. Create new force traversal table, OperationId=" << operationId << " , PathId " << pathId); + + // create new force traversal + TForceTraversalTable operationTable { + .PathId = pathId, + .ColumnTags = columnTags, + .Status = status + }; + operation.Tables.emplace_back(operationTable); + + db.Table().Key(operationId, pathId.OwnerId, pathId.LocalPathId).Update( + NIceDb::TUpdate(operationId), + NIceDb::TUpdate(pathId.OwnerId), + NIceDb::TUpdate(pathId.LocalPathId), + NIceDb::TUpdate(columnTags), + NIceDb::TUpdate((ui64)status) + ); + } + + Self->ForceTraversals.emplace_back(operation); + + db.Table().Key(operationId).Update( + NIceDb::TUpdate(operationId), + NIceDb::TUpdate(types) + ); + + return true; + } + + void Complete(const TActorContext& /*ctx*/) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Complete"); + } +}; + +void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyze::TPtr& ev) { + Execute(new TTxAnalyze(this, ev->Get()->Record, ev->Sender), TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_analyze_table.cpp b/ydb/core/statistics/aggregator/tx_analyze_table.cpp deleted file mode 100644 index df8ecd34820f..000000000000 --- a/ydb/core/statistics/aggregator/tx_analyze_table.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "aggregator_impl.h" - -#include - -#include - -namespace NKikimr::NStat { - -struct TStatisticsAggregator::TTxAnalyzeTable : public TTxBase { - const NKikimrStat::TEvAnalyze& Record; - TActorId ReplyToActorId; - - TTxAnalyzeTable(TSelf* self, const NKikimrStat::TEvAnalyze& record, TActorId replyToActorId) - : TTxBase(self) - , Record(record) - , ReplyToActorId(replyToActorId) - {} - - TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE; } - - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute. ReplyToActorId " << ReplyToActorId << " , Record " << Record); - - if (!Self->EnableColumnStatistics) { - return true; - } - - NIceDb::TNiceDb db(txc.DB); - - const TString operationId = Record.GetOperationId(); - const TString types = JoinVectorIntoString(TVector(Record.GetTypes().begin(), Record.GetTypes().end()), ","); - - for (const auto& table : Record.GetTables()) { - const TPathId pathId = PathIdFromPathId(table.GetPathId()); - const TString columnTags = JoinVectorIntoString(TVector{table.GetColumnTags().begin(),table.GetColumnTags().end()},","); - - // check existing force traversal with the same cookie and path - auto forceTraversal = std::find_if(Self->ForceTraversals.begin(), Self->ForceTraversals.end(), - [&pathId, &operationId](const TForceTraversal& elem) { - return elem.PathId == pathId - && elem.OperationId == operationId;}); - - // update existing force traversal - if (forceTraversal != Self->ForceTraversals.end()) { - SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute. Update existing force traversal. PathId " << pathId << " , ReplyToActorId " << ReplyToActorId); - forceTraversal->ReplyToActorId = ReplyToActorId; - return true; - } - - SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Execute. Create new force traversal operation for pathId " << pathId); - - // create new force trasersal - TForceTraversal operation { - .OperationId = operationId, - .PathId = pathId, - .ColumnTags = columnTags, - .Types = types, - .ReplyToActorId = ReplyToActorId - }; - Self->ForceTraversals.emplace_back(operation); -/* - db.Table().Key(Self->NextForceTraversalOperationId, pathId.OwnerId, pathId.LocalPathId).Update( - NIceDb::TUpdate(Self->NextForceTraversalOperationId), - NIceDb::TUpdate(pathId.OwnerId), - NIceDb::TUpdate(pathId.LocalPathId), - NIceDb::TUpdate(cookie), - NIceDb::TUpdate(columnTags), - NIceDb::TUpdate(types) - ); -*/ - } - - return true; - } - - void Complete(const TActorContext& ctx) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTable::Complete"); - } -}; - -void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyze::TPtr& ev) { - Execute(new TTxAnalyzeTable(this, ev->Get()->Record, ev->Sender), TActivationContext::AsActorContext()); -} - -} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_analyze_table_request.cpp b/ydb/core/statistics/aggregator/tx_analyze_table_request.cpp new file mode 100644 index 000000000000..983a4761c78a --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze_table_request.cpp @@ -0,0 +1,79 @@ +#include "aggregator_impl.h" + +#include +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyzeTableRequest : public TTxBase { + std::vector> Events; + + TTxAnalyzeTableRequest(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE_REQUEST; } + + static std::unique_ptr MakeRequest(const TString& operationId, const TForceTraversalTable& operationTable) { + auto request = std::make_unique(); + auto& record = request->Record; + record.SetOperationId(operationId); + auto& table = *record.MutableTable(); + PathIdFromPathId(operationTable.PathId, table.MutablePathId()); + TVector columnTags = Scan(SplitString(operationTable.ColumnTags, ",")); + table.MutableColumnTags()->Add(columnTags.begin(), columnTags.end()); + return request; + } + + bool Execute(TTransactionContext&, const TActorContext&) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeTableRequest::Execute"); + + for (TForceTraversalOperation& operation : Self->ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + if (operationTable.Status == TForceTraversalTable::EStatus::AnalyzeStarted) { + for(TAnalyzedShard& analyzedShard : operationTable.AnalyzedShards) { + if (analyzedShard.Status == TAnalyzedShard::EStatus::None) { + analyzedShard.Status = TAnalyzedShard::EStatus::AnalyzeStarted; + + auto request = MakeRequest(operation.OperationId, operationTable); + Events.push_back(std::make_unique(request.release(), analyzedShard.ShardTabletId, true)); + + if (Events.size() == SendAnalyzeCount) + return true; + } + } + } + } + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + if (Events.size()) { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTableRequest::Complete. Send " << Events.size() << " events."); + } + else { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeTableRequest::Complete."); + } + + for (auto& ev : Events) { + Self->Send(MakePipePerNodeCacheID(false), ev.release() ); + } + + if (Events.size() == SendAnalyzeCount) { + ctx.Send(Self->SelfId(), new TEvPrivate::TEvSendAnalyze()); + } else { + ctx.Schedule(SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); + } + } +}; + +void TStatisticsAggregator::Handle(TEvPrivate::TEvSendAnalyze::TPtr&) { + Execute(new TTxAnalyzeTableRequest(this), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_analyze_table_response.cpp b/ydb/core/statistics/aggregator/tx_analyze_table_response.cpp new file mode 100644 index 000000000000..8834c6649698 --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze_table_response.cpp @@ -0,0 +1,65 @@ +#include "aggregator_impl.h" + +#include +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyzeTableResponse : public TTxBase { + NKikimrStat::TEvAnalyzeTableResponse Record; + + TTxAnalyzeTableResponse(TSelf* self, NKikimrStat::TEvAnalyzeTableResponse&& record) + : TTxBase(self) + , Record(std::move(record)) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE_RESPONSE; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTableResponse::Execute"); + + const TString operationId = Record.GetOperationId(); + const TPathId pathId = PathIdFromPathId(Record.GetPathId()); + auto operationTable = Self->ForceTraversalTable(operationId, pathId); + if (!operationTable) { + SA_LOG_E("[" << Self->TabletID() << "] TTxAnalyzeTableResponse::Complete. Unknown OperationTable. Record: " << Record.ShortDebugString()); + return true; + } + + auto analyzedShard = std::find_if(operationTable->AnalyzedShards.begin(), operationTable->AnalyzedShards.end(), + [tabletId = Record.GetShardTabletId()] (TAnalyzedShard& analyzedShard) { return analyzedShard.ShardTabletId == tabletId;}); + if (analyzedShard == operationTable->AnalyzedShards.end()) { + SA_LOG_E("[" << Self->TabletID() << "] TTxAnalyzeTableResponse::Complete. Unknown AnalyzedShards. Record: " << Record.ShortDebugString() << ", ShardTabletId " << Record.GetShardTabletId()); + return true; + } + if (analyzedShard->Status != TAnalyzedShard::EStatus::AnalyzeStarted) { + SA_LOG_E("[" << Self->TabletID() << "] TTxAnalyzeTableResponse::Complete. Unknown AnalyzedShards Status. Record: " << Record.ShortDebugString() << ", ShardTabletId " << Record.GetShardTabletId()); + } + + analyzedShard->Status = TAnalyzedShard::EStatus::AnalyzeFinished; + + bool completeResponse = std::any_of(operationTable->AnalyzedShards.begin(), operationTable->AnalyzedShards.end(), + [] (const TAnalyzedShard& analyzedShard) { return analyzedShard.Status == TAnalyzedShard::EStatus::AnalyzeFinished;}); + + if (!completeResponse) + return true; + + NIceDb::TNiceDb db(txc.DB); + Self->UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::AnalyzeFinished, operationId, *operationTable, db); + return true; + } + + void Complete(const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeTableResponse::Complete."); + } +}; + +void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyzeTableResponse::TPtr& ev) { + auto& record = ev->Get()->Record; + Execute(new TTxAnalyzeTableResponse(this, std::move(record)), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp index b9f7fc597fa4..a48dd94c2170 100644 --- a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp +++ b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp @@ -12,9 +12,13 @@ struct TStatisticsAggregator::TTxFinishTraversal : public TTxBase { TTxFinishTraversal(TSelf* self) : TTxBase(self) , OperationId(self->ForceTraversalOperationId) - , PathId(self->TraversalTableId.PathId) - , ReplyToActorId(self->ForceTraversalReplyToActorId) - {} + , PathId(self->TraversalPathId) + { + auto forceTraversal = Self->CurrentForceTraversalOperation(); + if (forceTraversal) { + ReplyToActorId = forceTraversal->ReplyToActorId; + } + } TTxType GetTxType() const override { return TXTYPE_FINISH_TRAVERSAL; } @@ -36,10 +40,9 @@ struct TStatisticsAggregator::TTxFinishTraversal : public TTxBase { return; } - bool operationsRemain = std::any_of(Self->ForceTraversals.begin(), Self->ForceTraversals.end(), - [this](const TForceTraversal& elem) { return elem.OperationId == OperationId;}); + auto forceTraversalRemained = Self->ForceTraversalOperation(OperationId); - if (operationsRemain) { + if (forceTraversalRemained) { SA_LOG_D("[" << Self->TabletID() << "] TTxFinishTraversal::Complete. Don't send TEvAnalyzeResponse. " << "There are pending operations, OperationId " << OperationId << " , ActorId=" << ReplyToActorId); } else { diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index e4012d7bbe39..0281115f9f26 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -3,6 +3,8 @@ #include #include +#include + namespace NKikimr::NStat { struct TStatisticsAggregator::TTxInit : public TTxBase { @@ -22,13 +24,15 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { auto baseStatisticsRowset = db.Table().Range().Select(); auto statisticsRowset = db.Table().Range().Select(); auto scheduleTraversalRowset = db.Table().Range().Select(); -// auto forceTraversalRowset = db.Table().Range().Select(); + auto forceTraversalOperationsRowset = db.Table().Range().Select(); + auto forceTraversalTablesRowset = db.Table().Range().Select(); if (!sysParamsRowset.IsReady() || !baseStatisticsRowset.IsReady() || !statisticsRowset.IsReady() || - !scheduleTraversalRowset.IsReady()) -// !forceTraversalRowset.IsReady()) + !scheduleTraversalRowset.IsReady() || + !forceTraversalOperationsRowset.IsReady() || + !forceTraversalTablesRowset.IsReady()) { return false; } @@ -54,31 +58,16 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { Self->TraversalStartKey = TSerializedCellVec(value); SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal start key"); break; - case Schema::SysParam_ForceTraversalOperationId: { - Self->ForceTraversalOperationId = value; - SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal operation id: " << value); - break; - } case Schema::SysParam_TraversalTableOwnerId: - Self->TraversalTableId.PathId.OwnerId = FromString(value); + Self->TraversalPathId.OwnerId = FromString(value); SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal table owner id: " - << Self->TraversalTableId.PathId.OwnerId); + << Self->TraversalPathId.OwnerId); break; case Schema::SysParam_TraversalTableLocalPathId: - Self->TraversalTableId.PathId.LocalPathId = FromString(value); + Self->TraversalPathId.LocalPathId = FromString(value); SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal table local path id: " - << Self->TraversalTableId.PathId.LocalPathId); - break; - case Schema::SysParam_ForceTraversalColumnTags: { - Self->ForceTraversalColumnTags = value; - SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal columns tags: " << value); - break; - } - case Schema::SysParam_ForceTraversalTypes: { - Self->ForceTraversalTypes = value; - SA_LOG_D("[" << Self->TabletID() << "] Loaded traversal types: " << value); + << Self->TraversalPathId.LocalPathId); break; - } case Schema::SysParam_TraversalStartTime: { auto us = FromString(value); Self->TraversalStartTime = TInstant::MicroSeconds(us); @@ -193,31 +182,22 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { << "table count# " << Self->ScheduleTraversals.size()); } - // ForceTraversals -/* + // ForceTraversalOperations { Self->ForceTraversals.clear(); - auto rowset = db.Table().Range().Select(); + auto rowset = db.Table().Range().Select(); if (!rowset.IsReady()) { return false; } while (!rowset.EndOfSet()) { - ui64 operationId = rowset.GetValue(); - ui64 ownerId = rowset.GetValue(); - ui64 localPathId = rowset.GetValue(); - TString cookie = rowset.GetValue(); - TString columnTags = rowset.GetValue(); - TString types = rowset.GetValue(); - - auto pathId = TPathId(ownerId, localPathId); + TString operationId = rowset.GetValue(); + TString types = rowset.GetValue(); - TForceTraversal operation { + TForceTraversalOperation operation { .OperationId = operationId, - .Cookie = cookie, - .PathId = pathId, - .ColumnTags = columnTags, + .Tables = {}, .Types = types, .ReplyToActorId = {} }; @@ -228,10 +208,50 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { } } - SA_LOG_D("[" << Self->TabletID() << "] Loaded ForceTraversals: " + SA_LOG_D("[" << Self->TabletID() << "] Loaded ForceTraversalOperations: " << "table count# " << Self->ForceTraversals.size()); } -*/ + + // ForceTraversalTables + { + auto rowset = db.Table().Range().Select(); + if (!rowset.IsReady()) { + return false; + } + + size_t size = 0; + while (!rowset.EndOfSet()) { + ++size; + + TString operationId = rowset.GetValue(); + ui64 ownerId = rowset.GetValue(); + ui64 localPathId = rowset.GetValue(); + TString columnTags = rowset.GetValue(); + ui64 status = rowset.GetValue(); + + auto pathId = TPathId(ownerId, localPathId); + + TForceTraversalTable operationTable { + .PathId = pathId, + .ColumnTags = columnTags, + .Status = (TForceTraversalTable::EStatus)status, + }; + auto forceTraversalOperation = Self->ForceTraversalOperation(operationId); + if (!forceTraversalOperation) { + SA_LOG_E("[" << Self->TabletID() << "] ForceTraversalTables contains unknown operationId: " << operationId); + continue; + } + forceTraversalOperation->Tables.emplace_back(operationTable); + + if (!rowset.Next()) { + return false; + } + } + + SA_LOG_D("[" << Self->TabletID() << "] Loaded ForceTraversalTables: " + << "table count# " << size); + } + return true; } @@ -246,10 +266,13 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { Self->Schedule(Self->PropagateInterval, new TEvPrivate::TEvPropagate()); Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); + Self->Schedule(Self->SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); Self->InitializeStatisticsTable(); - if (Self->TraversalTableId.PathId) { + if (Self->TraversalPathId) { + Self->NavigateType = ENavigateType::Traversal; + Self->NavigatePathId = Self->TraversalPathId; Self->Navigate(); } diff --git a/ydb/core/statistics/aggregator/tx_resolve.cpp b/ydb/core/statistics/aggregator/tx_resolve.cpp index 3215cde186b5..36d9d561144b 100644 --- a/ydb/core/statistics/aggregator/tx_resolve.cpp +++ b/ydb/core/statistics/aggregator/tx_resolve.cpp @@ -3,12 +3,14 @@ #include #include +#include + namespace NKikimr::NStat { struct TStatisticsAggregator::TTxResolve : public TTxBase { std::unique_ptr Request; bool Cancelled = false; - bool DoSend = true; + bool StartColumnShardEventDistribution = true; TTxResolve(TSelf* self, NSchemeCache::TSchemeCacheRequest* request) : TTxBase(self) @@ -17,14 +19,42 @@ struct TStatisticsAggregator::TTxResolve : public TTxBase { TTxType GetTxType() const override { return TXTYPE_RESOLVE; } - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxResolve::Execute"); + bool ExecuteAnalyze(const NSchemeCache::TSchemeCacheRequest::TEntry& entry, NIceDb::TNiceDb& db) { + Y_ABORT_UNLESS(Self->NavigateAnalyzeOperationId); + Y_ABORT_UNLESS(Self->NavigatePathId); - NIceDb::TNiceDb db(txc.DB); + if (entry.Status == NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist) { + // AnalyzedShards will be empty and Analyze will complete without sending event to shards + return true; + } - Y_ABORT_UNLESS(Request->ResultSet.size() == 1); - const auto& entry = Request->ResultSet.front(); + if (entry.Status != NSchemeCache::TSchemeCacheRequest::EStatus::OkData) { + Cancelled = true; + return true; + } + + auto& partitioning = entry.KeyDescription->GetPartitions(); + + auto forceTraversalTable = Self->ForceTraversalTable(Self->NavigateAnalyzeOperationId, Self->NavigatePathId); + Y_ABORT_UNLESS(forceTraversalTable); + + for (auto& part : partitioning) { + if (part.Range) { + forceTraversalTable->AnalyzedShards.push_back({ + .ShardTabletId = part.ShardId, + .Status = TAnalyzedShard::EStatus::None + }); + } + } + + SA_LOG_D("[" << Self->TabletID() << "] TTxResolve::ExecuteAnalyze. Table OperationId " << Self->NavigateAnalyzeOperationId << ", PathId " << Self->NavigatePathId + << ", AnalyzedShards " << forceTraversalTable->AnalyzedShards.size()); + + Self->UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::AnalyzeStarted, Self->NavigateAnalyzeOperationId, *forceTraversalTable, db); + return true; + } + bool ExecuteTraversal(const NSchemeCache::TSchemeCacheRequest::TEntry& entry, NIceDb::TNiceDb& db) { if (entry.Status != NSchemeCache::TSchemeCacheRequest::EStatus::OkData) { Cancelled = true; @@ -61,27 +91,58 @@ struct TStatisticsAggregator::TTxResolve : public TTxBase { if (Self->TraversalIsColumnTable && Self->TabletsForReqDistribution.empty()) { Self->FinishTraversal(db); - DoSend = false; + StartColumnShardEventDistribution = false; } return true; } - void Complete(const TActorContext& ctx) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxResolve::Complete"); + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxResolve::Execute"); + + NIceDb::TNiceDb db(txc.DB); + + Y_ABORT_UNLESS(Request->ResultSet.size() == 1); + const auto& entry = Request->ResultSet.front(); + + switch (Self->NavigateType) { + case ENavigateType::Analyze: + return ExecuteAnalyze(entry, db); + case ENavigateType::Traversal: + return ExecuteTraversal(entry, db); + }; + + } + + void CompleteTraversal(const TActorContext& ctx) { if (Cancelled) { return; } if (Self->TraversalIsColumnTable) { - if (DoSend) { + if (StartColumnShardEventDistribution) { ctx.Send(Self->SelfId(), new TEvPrivate::TEvRequestDistribution); } } else { Self->ScanNextDatashardRange(); } } + + void Complete(const TActorContext& ctx) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxResolve::Complete"); + + switch (Self->NavigateType) { + case ENavigateType::Analyze: + break; + case ENavigateType::Traversal: + CompleteTraversal(ctx); + break; + }; + + Self->NavigateAnalyzeOperationId.clear(); + Self->NavigatePathId = {}; + } }; void TStatisticsAggregator::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { diff --git a/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp b/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp index 341b31964907..abe80134ee3c 100644 --- a/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp +++ b/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp @@ -8,7 +8,7 @@ namespace NKikimr::NStat { struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { - NKikimrHive::TEvResponseTabletDistribution Record; + const NKikimrHive::TEvResponseTabletDistribution HiveRecord; enum class EAction : ui8 { None, @@ -18,30 +18,47 @@ struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { }; EAction Action = EAction::None; - std::unique_ptr Request; + std::unique_ptr AggregateStatisticsRequest; - TTxResponseTabletDistribution(TSelf* self, NKikimrHive::TEvResponseTabletDistribution&& record) + TTxResponseTabletDistribution(TSelf* self, NKikimrHive::TEvResponseTabletDistribution&& hiveRecord) : TTxBase(self) - , Record(std::move(record)) + , HiveRecord(std::move(hiveRecord)) {} TTxType GetTxType() const override { return TXTYPE_RESPONSE_TABLET_DISTRIBUTION; } - bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_D("[" << Self->TabletID() << "] TTxResponseTabletDistribution::Execute"); - + bool ExecuteStartForceTraversal(TTransactionContext& txc) { + ++Self->TraversalRound; + ++Self->GlobalTraversalRound; + NIceDb::TNiceDb db(txc.DB); + Self->PersistGlobalTraversalRound(db); - Request = std::make_unique(); - auto& outRecord = Request->Record; + AggregateStatisticsRequest = std::make_unique(); + auto& outRecord = AggregateStatisticsRequest->Record; + outRecord.SetRound(Self->GlobalTraversalRound); + PathIdFromPathId(Self->TraversalPathId, outRecord.MutablePathId()); - PathIdFromPathId(Self->TraversalTableId.PathId, outRecord.MutablePathId()); - - TVector columnTags = Scan(SplitString(Self->ForceTraversalColumnTags, ",")); - outRecord.MutableColumnTags()->Add(columnTags.begin(), columnTags.end()); + const auto forceTraversalTable = Self->CurrentForceTraversalTable(); + if (forceTraversalTable) { + TVector columnTags = Scan(SplitString(forceTraversalTable->ColumnTags, ",")); + outRecord.MutableColumnTags()->Add(columnTags.begin(), columnTags.end()); + } + + for (auto& inNode : HiveRecord.GetNodes()) { + auto& outNode = *outRecord.AddNodes(); + outNode.SetNodeId(inNode.GetNodeId()); + outNode.MutableTabletIds()->CopyFrom(inNode.GetTabletIds()); + } + + return true; + } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + SA_LOG_D("[" << Self->TabletID() << "] TTxResponseTabletDistribution::Execute"); auto distribution = Self->TabletsForReqDistribution; - for (auto& inNode : Record.GetNodes()) { + for (auto& inNode : HiveRecord.GetNodes()) { if (inNode.GetNodeId() == 0) { // these tablets are probably in Hive boot queue if (Self->HiveRequestRound < Self->MaxHiveRequestRoundCount) { @@ -49,11 +66,7 @@ struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { } continue; } - auto& outNode = *outRecord.AddNodes(); - outNode.SetNodeId(inNode.GetNodeId()); - outNode.MutableTabletIds()->Reserve(inNode.TabletIdsSize()); for (auto tabletId : inNode.GetTabletIds()) { - outNode.AddTabletIds(tabletId); distribution.erase(tabletId); } } @@ -64,17 +77,13 @@ struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { if (!distribution.empty() && Self->ResolveRound < Self->MaxResolveRoundCount) { // these tablets do not exist in Hive anymore + Self->NavigatePathId = Self->TraversalPathId; Action = EAction::ScheduleResolve; return true; } - ++Self->TraversalRound; - ++Self->GlobalTraversalRound; - Self->PersistGlobalTraversalRound(db); - outRecord.SetRound(Self->GlobalTraversalRound); Action = EAction::SendAggregate; - - return true; + return ExecuteStartForceTraversal(txc); } void Complete(const TActorContext& ctx) override { @@ -90,7 +99,7 @@ struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { break; case EAction::SendAggregate: - ctx.Send(MakeStatServiceID(Self->SelfId().NodeId()), Request.release()); + ctx.Send(MakeStatServiceID(Self->SelfId().NodeId()), AggregateStatisticsRequest.release()); ctx.Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++Self->KeepAliveSeqNo)); break; diff --git a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp index 0eb09bbd67af..2b2d81628a76 100644 --- a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp +++ b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp @@ -14,23 +14,40 @@ struct TStatisticsAggregator::TTxScheduleTrasersal : public TTxBase { bool Execute(TTransactionContext& txc, const TActorContext&) override { SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Execute"); - Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); - if (!Self->EnableColumnStatistics) { return true; } - if (Self->TraversalTableId.PathId) { - return true; // traverse is in progress + if (Self->TraversalPathId) { + SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Execute. Traverse is in progress. PathId " << Self->TraversalPathId); + return true; + } + + if (!Self->IsSchemeshardSeen) { + SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal. No info from schemeshard"); + return true; } NIceDb::TNiceDb db(txc.DB); - Self->ScheduleNextTraversal(db); + + switch (Self->NavigateType) { + case ENavigateType::Analyze: + Self->NavigateType = ENavigateType::Traversal; + Self->ScheduleNextTraversal(db); + break; + case ENavigateType::Traversal: + Self->NavigateType = ENavigateType::Analyze; + Self->ScheduleNextAnalyze(db); + break; + } + return true; } void Complete(const TActorContext&) override { SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Complete"); + + Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); } }; diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp index d491aad97420..bfcecafa635c 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp @@ -94,7 +94,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { Analyze(runtime, saTabletId, {pathId}); - runtime.SimulateSleep(TDuration::Seconds(60)); + runtime.SimulateSleep(TDuration::Seconds(10)); ValidateCountMinAbsense(runtime, pathId); } diff --git a/ydb/core/statistics/aggregator/ya.make b/ydb/core/statistics/aggregator/ya.make index f4003bcdade1..8898d92bfc59 100644 --- a/ydb/core/statistics/aggregator/ya.make +++ b/ydb/core/statistics/aggregator/ya.make @@ -9,7 +9,9 @@ SRCS( schema.cpp tx_ack_timeout.cpp tx_aggr_stat_response.cpp - tx_analyze_table.cpp + tx_analyze.cpp + tx_analyze_table_request.cpp + tx_analyze_table_response.cpp tx_configure.cpp tx_datashard_scan_response.cpp tx_finish_trasersal.cpp diff --git a/ydb/core/statistics/service/service_impl.cpp b/ydb/core/statistics/service/service_impl.cpp index 113e4466bbe2..c77b03c7ddee 100644 --- a/ydb/core/statistics/service/service_impl.cpp +++ b/ydb/core/statistics/service/service_impl.cpp @@ -69,7 +69,7 @@ struct TAggregationStatistics { size_t NextTablet{ 0 }; ui32 InFlight{ 0 }; std::vector Ids; - std::unordered_set FinishedTablets; + std::unordered_map TabletsPipes; }; struct ColumnStatistics { @@ -195,8 +195,6 @@ class TStatService : public TActorBootstrapped { hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); hFunc(TEvPrivate::TEvDispatchKeepAlive, Handle); hFunc(TEvPrivate::TEvKeepAliveTimeout, Handle); - hFunc(TEvPipeCache::TEvGetTabletNodeResult, Handle); - hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); hFunc(TEvStatistics::TEvStatisticsResponse, Handle); hFunc(TEvStatistics::TEvAggregateStatisticsResponse, Handle); @@ -238,29 +236,9 @@ class TStatService : public TActorBootstrapped { return false; } - // - // returns true if the tablet processing has not been completed yet - // - bool OnTabletFinished(ui64 tabletId) { - auto& localTablets = AggregationStatistics.LocalTablets; - auto isFinished = !localTablets.FinishedTablets.emplace(tabletId).second; - if (isFinished) { - LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "OnTabletFinished: table " << tabletId << " has already been processed"); - return false; - } - - --localTablets.InFlight; - return true; - } - void OnAggregateStatisticsFinished() { SendAggregateStatisticsResponse(); - - Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); - - TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); - std::swap(AggregationStatistics, aggregationStatistics); + ResetAggregationStatistics(); } void SendRequestToNextTablet() { @@ -272,80 +250,22 @@ class TStatService : public TActorBootstrapped { const auto tabletId = localTablets.Ids[localTablets.NextTablet]; ++localTablets.NextTablet; ++localTablets.InFlight; - Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvGetTabletNode(tabletId), 0, AggregationStatistics.Round); - } - - void Handle(TEvPipeCache::TEvGetTabletNodeResult::TPtr& ev) { - const auto msg = ev->Get(); - LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "Received TEvGetTabletNodeResult NodeId: " << msg->NodeId << ", TabletId: " << msg->TabletId); - - const auto round = ev->Cookie; - if (IsNotCurrentRound(round)) { - return; - } - - const auto currentNodeId = ev->Recipient.NodeId(); - - // there is no need for retries, as the tablet must be local - // no problems are expected in resolving - if (currentNodeId != msg->NodeId) { - const auto tabletFinished = OnTabletFinished(msg->TabletId); - Y_ABORT_UNLESS(tabletFinished); - - LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "Tablet is not local. Table node: " << msg->NodeId << ", current node: " << ev->Recipient.NodeId()); - - const auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; - AggregationStatistics.FailedTablets.emplace_back(msg->TabletId, msg->NodeId, error); - - SendRequestToNextTablet(); - - if (AggregationStatistics.IsCompleted()) { - OnAggregateStatisticsFinished(); - } - - return; - } - auto request = std::make_unique(); - auto& record = request->Record; - record.MutableTypes()->Add(NKikimr::NStat::COUNT_MIN_SKETCH); - - auto* path = record.MutableTable()->MutablePathId(); - path->SetOwnerId(AggregationStatistics.PathId.OwnerId); - path->SetLocalId(AggregationStatistics.PathId.LocalPathId); - - auto* columnTags = record.MutableTable()->MutableColumnTags(); - for (const auto& tag : AggregationStatistics.ColumnTags) { - columnTags->Add(tag); - } - - Send(MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward(request.release(), msg->TabletId, true, round), - IEventHandle::FlagTrackDelivery, round); + auto policy = NTabletPipe::TClientRetryPolicy::WithRetries(); + policy.RetryLimitCount = 2; + NTabletPipe::TClientConfig pipeConfig{policy}; + pipeConfig.ForceLocal = true; + localTablets.TabletsPipes[tabletId] = Register(NTabletPipe::CreateClient(SelfId(), tabletId, pipeConfig)); } - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "Received TEvDeliveryProblem TabletId: " << ev->Get()->TabletId); - - const auto round = ev->Cookie; - const auto tabletId = ev->Get()->TabletId; - if (IsNotCurrentRound(round) - || !OnTabletFinished(tabletId)) { - return; + void ResetAggregationStatistics() { + const auto& tabletsPipes = AggregationStatistics.LocalTablets.TabletsPipes; + for (auto it = tabletsPipes.begin(); it != tabletsPipes.end(); ++it) { + NTabletPipe::CloseClient(SelfId(), it->second); } - const auto nodeId = ev->Recipient.NodeId(); - const auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; - AggregationStatistics.FailedTablets.emplace_back(tabletId, nodeId, error); - - SendRequestToNextTablet(); - - if (AggregationStatistics.IsCompleted()) { - OnAggregateStatisticsFinished(); - } + TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); + std::swap(AggregationStatistics, aggregationStatistics); } void AggregateStatistics(const TAggregationStatistics::TColumnsStatistics& columnsStatistics) { @@ -372,17 +292,25 @@ class TStatService : public TActorBootstrapped { } void Handle(TEvStatistics::TEvStatisticsResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + const auto tabletId = record.GetShardTabletId(); + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "Received TEvStatisticsResponse TabletId: " << ev->Get()->Record.GetShardTabletId()); + "Received TEvStatisticsResponse TabletId: " << tabletId); const auto round = ev->Cookie; - const auto& record = ev->Get()->Record; - if (IsNotCurrentRound(round) - || !OnTabletFinished(record.GetShardTabletId())) { + if (IsNotCurrentRound(round)) { return; } + auto tabletPipe = AggregationStatistics.LocalTablets.TabletsPipes.find(tabletId); + if (tabletPipe != AggregationStatistics.LocalTablets.TabletsPipes.end()) { + NTabletPipe::CloseClient(SelfId(), tabletPipe->second); + AggregationStatistics.LocalTablets.TabletsPipes.erase(tabletPipe); + } + AggregateStatistics(record.GetColumns()); + --AggregationStatistics.LocalTablets.InFlight; SendRequestToNextTablet(); @@ -394,6 +322,7 @@ class TStatService : public TActorBootstrapped { void Handle(TEvStatistics::TEvAggregateKeepAliveAck::TPtr& ev) { const auto& record = ev->Get()->Record; const auto round = record.GetRound(); + if (IsNotCurrentRound(round)) { LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "Skip TEvAggregateKeepAliveAck"); @@ -425,10 +354,8 @@ class TStatService : public TActorBootstrapped { LOG_INFO_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "Parent node " << AggregationStatistics.ParentNode.NodeId() << " is unavailable"); - TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); - std::swap(AggregationStatistics, aggregationStatistics); - Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); + ResetAggregationStatistics(); } void Handle(TEvPrivate::TEvDispatchKeepAlive::TPtr& ev) { @@ -658,8 +585,7 @@ class TStatService : public TActorBootstrapped { // reset previous state if (AggregationStatistics.Round != 0) { - TAggregationStatistics aggregationStatistics(Settings.FanOutFactor); - std::swap(AggregationStatistics, aggregationStatistics); + ResetAggregationStatistics(); } AggregationStatistics.Round = round; @@ -684,8 +610,6 @@ class TStatService : public TActorBootstrapped { // forming the right and left child nodes size_t k = 0; for (const auto& node : nodes) { - ++k; - if (node.GetNodeId() == currentNodeId) { AggregationStatistics.LocalTablets.Ids.reserve(node.GetTabletIds().size()); @@ -703,6 +627,7 @@ class TStatService : public TActorBootstrapped { } AggregationStatistics.Nodes[k % Settings.FanOutFactor].Tablets.push_back(std::move(nodeTablets)); + ++k; } for (auto& node : AggregationStatistics.Nodes) { @@ -969,31 +894,104 @@ class TStatService : public TActorBootstrapped { } } + void SendStatisticsRequest(const TActorId& clientId) { + auto request = std::make_unique(); + auto& record = request->Record; + record.MutableTypes()->Add(NKikimr::NStat::COUNT_MIN_SKETCH); + + auto* path = record.MutableTable()->MutablePathId(); + path->SetOwnerId(AggregationStatistics.PathId.OwnerId); + path->SetLocalId(AggregationStatistics.PathId.LocalPathId); + + auto* columnTags = record.MutableTable()->MutableColumnTags(); + for (const auto& tag : AggregationStatistics.ColumnTags) { + columnTags->Add(tag); + } + + NTabletPipe::SendData(SelfId(), clientId, request.release(), AggregationStatistics.Round); + } + + void OnTabletError(ui64 tabletId) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Tablet " << tabletId << " is not local."); + + constexpr auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; + AggregationStatistics.FailedTablets.emplace_back(tabletId, 0, error); + + AggregationStatistics.LocalTablets.TabletsPipes.erase(tabletId); + --AggregationStatistics.LocalTablets.InFlight; + SendRequestToNextTablet(); + + if (AggregationStatistics.IsCompleted()) { + OnAggregateStatisticsFinished(); + } + } + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + const auto& clientId = ev->Get()->ClientId; + const auto& tabletId = ev->Get()->TabletId; + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "EvClientConnected" << ", node id = " << ev->Get()->ClientId.NodeId() - << ", client id = " << ev->Get()->ClientId + << ", client id = " << clientId << ", server id = " << ev->Get()->ServerId + << ", tablet id = " << tabletId << ", status = " << ev->Get()->Status); - if (ev->Get()->Status != NKikimrProto::OK) { - SAPipeClientId = TActorId(); - ConnectToSA(); - SyncNode(); + if (clientId == SAPipeClientId) { + if (ev->Get()->Status != NKikimrProto::OK) { + SAPipeClientId = TActorId(); + ConnectToSA(); + SyncNode(); + } + return; + } + + const auto& tabletsPipes = AggregationStatistics.LocalTablets.TabletsPipes; + auto tabletPipe = tabletsPipes.find(tabletId); + + if (tabletPipe != tabletsPipes.end() && clientId == tabletPipe->second) { + if (ev->Get()->Status == NKikimrProto::OK) { + SendStatisticsRequest(clientId); + } else { + OnTabletError(tabletId); + } + return; } + + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip EvClientConnected"); } void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { + const auto& clientId = ev->Get()->ClientId; + const auto& tabletId = ev->Get()->TabletId; + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "EvClientDestroyed" << ", node id = " << ev->Get()->ClientId.NodeId() - << ", client id = " << ev->Get()->ClientId - << ", server id = " << ev->Get()->ServerId); + << ", client id = " << clientId + << ", server id = " << ev->Get()->ServerId + << ", tablet id = " << tabletId); - SAPipeClientId = TActorId(); - ConnectToSA(); - SyncNode(); + if (clientId == SAPipeClientId) { + SAPipeClientId = TActorId(); + ConnectToSA(); + SyncNode(); + return; + } + + const auto& tabletsPipes = AggregationStatistics.LocalTablets.TabletsPipes; + auto tabletPipe = tabletsPipes.find(tabletId); + + if (tabletPipe != tabletsPipes.end() && clientId == tabletPipe->second) { + OnTabletError(tabletId); + return; + } + + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip EvClientDestroyed"); } void Handle(TEvStatistics::TEvStatisticsIsDisabled::TPtr&) { diff --git a/ydb/core/statistics/service/ut/ut_service.cpp b/ydb/core/statistics/service/ut/ut_service.cpp index 84d3c2827987..a885c1b06838 100644 --- a/ydb/core/statistics/service/ut/ut_service.cpp +++ b/ydb/core/statistics/service/ut/ut_service.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include @@ -139,7 +139,7 @@ TStatServiceSettings GetDefaultSettings() { return settings; } -std::unordered_map InitializeRuntime(TTestActorRuntime& runtime, ui32 nodesCount, +std::unordered_map InitializeRuntime(TTestActorRuntime& runtime, ui32 nodesCount, const TStatServiceSettings& settings = GetDefaultSettings()) { runtime.SetLogPriority(NKikimrServices::STATISTICS, NLog::EPriority::PRI_DEBUG); runtime.SetScheduledEventFilter([](TTestActorRuntimeBase&, TAutoPtr&, TDuration, TInstant&){ @@ -154,235 +154,382 @@ std::unordered_map InitializeRuntime(TTestActorRuntime& runtime, ui3 } auto nameserviceActor = GetNameserviceActorId(); - auto pipeActor = MakePipePerNodeCacheID(false); - auto pipeConfig = MakeIntrusive(); - - std::unordered_map indexToNodeId; + auto resolverActor = MakeTabletResolverID(); + auto resolverConfig = MakeIntrusive(); + std::unordered_map indexToActor; for (ui32 i = 0; i < nodesCount; ++i) { ui32 nodeId = runtime.GetNodeId(i); - indexToNodeId.emplace(i, nodeId); - auto actorId = NStat::MakeStatServiceID(nodeId); + indexToActor.emplace(i, actorId); + runtime.AddLocalService(actorId, TActorSetupCmd(NStat::CreateStatService(settings).Release(), TMailboxType::HTSwap, 0), i); - runtime.AddLocalService(pipeActor, TActorSetupCmd(CreatePipePerNodeCache(pipeConfig), TMailboxType::HTSwap, 0), i); + runtime.AddLocalService(resolverActor, TActorSetupCmd(CreateTabletResolver(resolverConfig), TMailboxType::Simple, 0), i); runtime.AddLocalService(nameserviceActor, TActorSetupCmd(CreateNameserverTable(nameserverTable), TMailboxType::Simple, 0), i); } TTestActorRuntime::TEgg egg{ new TAppData(0, 0, 0, 0, { }, nullptr, nullptr, nullptr, nullptr), nullptr, nullptr, {} }; runtime.Initialize(egg); - return indexToNodeId; + return indexToActor; } -std::unordered_map ReverseMap(const std::unordered_map& map) { +std::unordered_map GetNodeIdToIndexMap(const std::unordered_map& map) { std::unordered_map res; for (auto it = map.begin(); it != map.end(); ++it) { - res.emplace(it->second, it->first); + res.emplace(it->second.NodeId(), it->first); } return res; } - Y_UNIT_TEST_SUITE(StatisticsService) { - Y_UNIT_TEST(ShouldBeVisitEveryNodeAndTablet) { - size_t nodeCount = 10; + Y_UNIT_TEST(ShouldBeCorrectlyAggregateStatisticsFromAllNodes) { + size_t nodeCount = 4; auto runtime = TTestActorRuntime(nodeCount, 1, false); - auto indexToNodeIdMap = InitializeRuntime(runtime, nodeCount); - auto nodeIdToIndexMap = ReverseMap(indexToNodeIdMap); - std::vector nodesTablets = {{.NodeId = indexToNodeIdMap[0], .Ids{0}}, - {.NodeId = indexToNodeIdMap[1], .Ids{1}}, {.NodeId = indexToNodeIdMap[2], .Ids{2}}, - {.NodeId = indexToNodeIdMap[3], .Ids{3}}, {.NodeId = indexToNodeIdMap[4], .Ids{4}}, - {.NodeId = indexToNodeIdMap[5], .Ids{5}}, {.NodeId = indexToNodeIdMap[6], .Ids{6}}, - {.NodeId = indexToNodeIdMap[7], .Ids{7}}, {.NodeId = indexToNodeIdMap[8], .Ids{8}}, - {.NodeId = indexToNodeIdMap[9], .Ids{9}}}; - - std::unordered_map nodes; - std::unordered_map tablets; + auto indexToActorMap = InitializeRuntime(runtime, nodeCount); + auto nodeIdToIndexMap = GetNodeIdToIndexMap(indexToActorMap); + std::vector localTabletsIds = {1, 2, 3}; + std::vector nodesTablets = {{.NodeId = indexToActorMap[0].NodeId(), .Ids{localTabletsIds}}, + {.NodeId = indexToActorMap[1].NodeId(), .Ids{4}}, {.NodeId = indexToActorMap[2].NodeId(), .Ids{5}}, + {.NodeId = indexToActorMap[3].NodeId(), .Ids{6}}}; + std::unordered_map pipeToTablet; + std::vector observers; - observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvAggregateStatistics::TPtr& ev) { - if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { - ++nodes[ev->Recipient.NodeId()]; - } + observers.emplace_back(runtime.AddObserver([&](TEvTabletResolver::TEvForward::TPtr& ev) { + auto tabletId = ev->Get()->TabletID; + auto recipient = indexToActorMap[nodeIdToIndexMap[ev->Sender.NodeId()]]; + pipeToTablet[ev->Sender] = tabletId; + + runtime.Send(new IEventHandle(recipient, ev->Sender, + new TEvTabletPipe::TEvClientConnected(tabletId, NKikimrProto::OK, ev->Sender, ev->Sender, + true, false, 0), 0, ev->Cookie), nodeIdToIndexMap[ev->Sender.NodeId()], true); + ev.Reset(); })); - observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNode::TPtr& ev) { - if (nodeIdToIndexMap.find(ev->Sender.NodeId()) != nodeIdToIndexMap.end()) { - ++tablets[ev->Get()->TabletId]; - ev.Reset(); + observers.emplace_back(runtime.AddObserver([&](TAutoPtr& ev) { + switch (ev->GetTypeRewrite()) { + case TEvTabletPipe::EvSend: + auto msg = ev->Get(); + if (msg != nullptr) { + auto tabletId = pipeToTablet[ev->Recipient]; + auto senderNodeIndex = nodeIdToIndexMap[ev->Sender.NodeId()]; + + if (tabletId <= localTabletsIds.back()) { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = tabletId, + .Columns{ + TColumnItem{.Tag = 1, .Cells{"1", "2"}}, + TColumnItem{.Tag = 2, .Cells{"3"}} + }, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), senderNodeIndex, true); + } else { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = tabletId, + .Columns{ + TColumnItem{.Tag = 2, .Cells{"3"}} + }, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), senderNodeIndex, true); + } + } + break; } })); auto sender = runtime.AllocateEdgeActor(); - runtime.Send(NStat::MakeStatServiceID(indexToNodeIdMap[0]), sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + runtime.Send(indexToActorMap[0], sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ .Round = 1, .PathId{3, 3}, .Nodes{ nodesTablets }, - .ColumnTags{1} + .ColumnTags{1, 2} }).release()); - runtime.DispatchEvents(TDispatchOptions{ - .CustomFinalCondition = [&]() { - return nodes.size() >= 10 && tablets.size() >= 10; - } - }); + auto res = runtime.GrabEdgeEvent(sender); + const auto& record = res->Get()->Record; + + std::unordered_map> expected = { + {1, {{"1", localTabletsIds.size()}, {"2", localTabletsIds.size()}}}, + {2, {{"3", nodesTablets.size() - 1 + localTabletsIds.size()}}}, + }; - for (auto it = nodes.begin(); it != nodes.end(); ++it) { - UNIT_ASSERT_VALUES_EQUAL(it->second, 1); - } - for (auto it = tablets.begin(); it != tablets.end(); ++it) { - UNIT_ASSERT_VALUES_EQUAL(it->second, 1); + const auto& columns = record.GetColumns(); + for (const auto& column : columns) { + const auto tag = column.GetTag(); + + for (auto& statistic : column.GetStatistics()) { + if (statistic.GetType() == NKikimr::NStat::COUNT_MIN_SKETCH) { + auto data = statistic.GetData().Data(); + auto sketch = reinterpret_cast(data); + + const auto& cells = expected[tag]; + for (auto it = cells.begin(); it != cells.end(); ++it) { + UNIT_ASSERT_VALUES_EQUAL(it->second, sketch->Probe(it->first.data(), it->first.size())); + } + } + } } - UNIT_ASSERT_VALUES_EQUAL(nodesTablets.size(), nodes.size()); - UNIT_ASSERT_VALUES_EQUAL(nodesTablets.size(), tablets.size()); } - Y_UNIT_TEST(ShouldBeAtMostMaxInFlightTabletRequests) { - size_t nodeCount = 1; + Y_UNIT_TEST(ShouldBePings) { + size_t nodeCount = 2; auto runtime = TTestActorRuntime(nodeCount, 1, false); - auto settings = GetDefaultSettings() - .SetMaxInFlightTabletRequests(3); - auto indexToNodeIdMap = InitializeRuntime(runtime, nodeCount, settings); - auto nodeIdToIndexMap = ReverseMap(indexToNodeIdMap); - std::vector localTabletsIds = {1, 2, 3, 4, 5, 6, 7, 8}; - std::vector nodesTablets = {{.NodeId = indexToNodeIdMap[0], .Ids{localTabletsIds}}}; - - size_t inFlight = 0; - size_t maxInFlight = 0; - size_t tabletsCount = 0; + auto indexToActorMap = InitializeRuntime(runtime, nodeCount, + GetDefaultSettings() + .SetAggregateKeepAlivePeriod(TDuration::MilliSeconds(10)) + .SetAggregateKeepAliveTimeout(TDuration::Seconds(3)) + .SetAggregateKeepAliveAckTimeout(TDuration::Seconds(3))); + auto nodeIdToIndexMap = GetNodeIdToIndexMap(indexToActorMap); + std::vector nodesTablets = {{.NodeId = indexToActorMap[0].NodeId(), .Ids{1}}, + {.NodeId = indexToActorMap[1].NodeId(), .Ids{2}}}; + + std::vector ping(3); + std::vector pong(3); + auto sender = runtime.AllocateEdgeActor(); std::vector observers; - observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNode::TPtr& ev) { - auto it = nodeIdToIndexMap.find(ev->Sender.NodeId()); - if (it == nodeIdToIndexMap.end()) { + observers.emplace_back(runtime.AddObserver([&](TEvTabletResolver::TEvForward::TPtr& ev) { + ev.Reset(); + })); + observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvAggregateKeepAlive::TPtr& ev) { + if (ev->Recipient == sender) { + ++ping[0]; + ev.Reset(); return; } - auto senderNodeIndex = it->second; - ++inFlight; - ++tabletsCount; - maxInFlight = std::max(maxInFlight, inFlight); - - auto tabletId = ev->Get()->TabletId; - - if (tabletId % 3 == 1) { // non local - runtime.Send(new IEventHandle(ev->Sender, ev->Sender, - new TEvPipeCache::TEvGetTabletNodeResult(tabletId, 10), 0, ev->Cookie), senderNodeIndex, true); - } else if(tabletId % 3 == 2) { // delivery problem - runtime.Send(new IEventHandle(ev->Sender, ev->Sender, - new TEvPipeCache::TEvDeliveryProblem(tabletId, true), 0, ev->Cookie), senderNodeIndex, true); - } else { - runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + + auto it = nodeIdToIndexMap.find(ev->Recipient.NodeId()); + if (it != nodeIdToIndexMap.end()) { + ++ping[it->second + 1]; + } + })); + observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvAggregateKeepAliveAck::TPtr& ev) { + auto it = nodeIdToIndexMap.find(ev->Recipient.NodeId()); + if (it != nodeIdToIndexMap.end()) { + ++pong[it->second + 1]; + } + })); + runtime.Send(indexToActorMap[0], sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + .Round = 1, + .PathId{3, 3}, + .Nodes{ nodesTablets }, + .ColumnTags{1, 2} + }).release()); + + runtime.DispatchEvents(TDispatchOptions{ + .CustomFinalCondition = [&]() { + return ping[0] >= 10 && ping[1] >= 10 && pong[2] >= 10; + } + }); + + for (const auto& node : nodesTablets) { + for (auto tabletId : node.Ids) { + auto actorId = NStat::MakeStatServiceID(node.NodeId); + runtime.Send(new IEventHandle(actorId, actorId, CreateStatisticsResponse(TStatisticsResponse{ .TabletId = tabletId, .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS - }).release(), 0, ev->Cookie), senderNodeIndex, true); + }).release(), 0, 1), nodeIdToIndexMap[node.NodeId], true); } - ev.Reset(); - })); - observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { - --inFlight; + } + + auto res = runtime.GrabEdgeEvent(sender); + const auto& record = res->Get()->Record; + UNIT_ASSERT(record.GetFailedTablets().empty()); + } + + Y_UNIT_TEST(RootNodeShouldBeInvalidateByTimeout) { + size_t nodeCount = 4; + auto runtime = TTestActorRuntime(nodeCount, 1, false); + auto indexToActorMap = InitializeRuntime(runtime, nodeCount, + GetDefaultSettings() + .SetAggregateKeepAlivePeriod(TDuration::MilliSeconds(5)) + .SetAggregateKeepAliveTimeout(TDuration::MilliSeconds(10)) + .SetAggregateKeepAliveAckTimeout(TDuration::MilliSeconds(10))); + auto nodeIdToIndexMap = GetNodeIdToIndexMap(indexToActorMap); + std::vector nodesTablets = {{.NodeId = indexToActorMap[0].NodeId(), .Ids{1}}, + {.NodeId = indexToActorMap[1].NodeId(), .Ids{2}}, {.NodeId = indexToActorMap[2].NodeId(), .Ids{3}}, + {.NodeId = indexToActorMap[3].NodeId(), .Ids{4}}}; + std::unordered_map pipeToTablet; + + std::vector observers; + observers.emplace_back(runtime.AddObserver([&](TEvTabletResolver::TEvForward::TPtr& ev) { + auto tabletId = ev->Get()->TabletID; + pipeToTablet[ev->Sender] = tabletId; + + if (tabletId == 2) { + ev.Reset(); + return; } + + auto recipient = indexToActorMap[nodeIdToIndexMap[ev->Sender.NodeId()]]; + runtime.Send(new IEventHandle(recipient, ev->Sender, + new TEvTabletPipe::TEvClientConnected(tabletId, NKikimrProto::OK, ev->Sender, ev->Sender, + true, false, 0), 0, ev->Cookie), nodeIdToIndexMap[ev->Sender.NodeId()], true); + ev.Reset(); })); - observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNodeResult::TPtr& ev) { - if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { - --inFlight; + observers.emplace_back(runtime.AddObserver([&](TAutoPtr& ev) { + switch (ev->GetTypeRewrite()) { + case TEvTabletPipe::EvSend: + auto msg = ev->Get(); + if (msg != nullptr) { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = pipeToTablet[ev->Recipient], + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), nodeIdToIndexMap[ev->Sender.NodeId()], true); + } + break; } })); - observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvStatisticsResponse::TPtr& ev) { - if (nodeIdToIndexMap.find(ev->Recipient.NodeId()) != nodeIdToIndexMap.end()) { - --inFlight; + observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvAggregateKeepAliveAck::TPtr& ev) { + if (ev->Sender == indexToActorMap[1]) { + ev.Reset(); + return; } })); auto sender = runtime.AllocateEdgeActor(); - runtime.Send(NStat::MakeStatServiceID(indexToNodeIdMap[0]), sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + runtime.Send(indexToActorMap[0], sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ .Round = 1, .PathId{3, 3}, .Nodes{ nodesTablets }, - .ColumnTags{1} + .ColumnTags{1, 2} }).release()); - runtime.DispatchEvents(TDispatchOptions{ - .CustomFinalCondition = [&]() { - return tabletsCount >= localTabletsIds.size(); - } - }); - UNIT_ASSERT_LT(maxInFlight, settings.MaxInFlightTabletRequests + 1); + auto res = runtime.GrabEdgeEvent(sender); + const auto& record = res->Get()->Record; + size_t expectedFailedTabletsCount = 2; + UNIT_ASSERT_VALUES_EQUAL(expectedFailedTabletsCount, record.GetFailedTablets().size()); + + ui32 expectedError = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_UNAVAILABLE_NODE; + for (const auto& fail : record.GetFailedTablets()) { + ui32 actualError = fail.GetError(); + UNIT_ASSERT_VALUES_EQUAL(expectedError, actualError); + } } - Y_UNIT_TEST(ShouldBeCorrectlyAggregateStatisticsFromAllNodes) { + Y_UNIT_TEST(ChildNodesShouldBeInvalidateByTimeout) { size_t nodeCount = 4; auto runtime = TTestActorRuntime(nodeCount, 1, false); - auto indexToNodeIdMap = InitializeRuntime(runtime, nodeCount); - auto nodeIdToIndexMap = ReverseMap(indexToNodeIdMap); - std::vector localTabletsIds = {1, 2, 3}; - std::vector nodesTablets = {{.NodeId = indexToNodeIdMap[0], .Ids{localTabletsIds}}, - {.NodeId = indexToNodeIdMap[1], .Ids{4}}, {.NodeId = indexToNodeIdMap[2], .Ids{5}}, - {.NodeId = indexToNodeIdMap[3], .Ids{6}}}; + auto indexToActorMap = InitializeRuntime(runtime, nodeCount, + GetDefaultSettings() + .SetAggregateKeepAlivePeriod(TDuration::MilliSeconds(5)) + .SetAggregateKeepAliveTimeout(TDuration::MilliSeconds(10))); + auto nodeIdToIndexMap = GetNodeIdToIndexMap(indexToActorMap); + std::vector nodesTablets = {{.NodeId = indexToActorMap[0].NodeId(), .Ids{1}}, + {.NodeId = indexToActorMap[1].NodeId(), .Ids{2}}, {.NodeId = indexToActorMap[2].NodeId(), .Ids{3}}, + {.NodeId = indexToActorMap[3].NodeId(), .Ids{4}}}; + std::unordered_map pipeToTablet; std::vector observers; - observers.emplace_back(runtime.AddObserver([&](TEvPipeCache::TEvGetTabletNode::TPtr& ev) { - auto tabletId = ev->Get()->TabletId; - auto senderNodeId = ev->Sender.NodeId(); - auto senderNodeIndex = nodeIdToIndexMap.find(senderNodeId); + observers.emplace_back(runtime.AddObserver([&](TEvTabletResolver::TEvForward::TPtr& ev) { + auto tabletId = ev->Get()->TabletID; + pipeToTablet[ev->Sender] = tabletId; - if (senderNodeIndex == nodeIdToIndexMap.end()) { + if (tabletId == 2) { + ev.Reset(); return; } - if (tabletId <= localTabletsIds.back()) { - runtime.Send(new IEventHandle(ev->Sender, ev->Sender, - CreateStatisticsResponse(TStatisticsResponse{ - .TabletId = ev->Get()->TabletId, - .Columns{ - TColumnItem{.Tag = 1, .Cells{"1", "2"}}, - TColumnItem{.Tag = 2, .Cells{"3"}} - }, - .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS - }).release(), 0, ev->Cookie), senderNodeIndex->second, true); - } else { - runtime.Send(new IEventHandle(ev->Sender, ev->Sender, - CreateStatisticsResponse(TStatisticsResponse{ - .TabletId = ev->Get()->TabletId, - .Columns{ - TColumnItem{.Tag = 2, .Cells{"3"}} - }, - .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS - }).release(), 0, ev->Cookie), senderNodeIndex->second, true); - } - + auto recipient = indexToActorMap[nodeIdToIndexMap[ev->Sender.NodeId()]]; + runtime.Send(new IEventHandle(recipient, ev->Sender, + new TEvTabletPipe::TEvClientConnected(tabletId, NKikimrProto::OK, ev->Sender, ev->Sender, + true, false, 0), 0, ev->Cookie), nodeIdToIndexMap[ev->Sender.NodeId()], true); ev.Reset(); })); + observers.emplace_back(runtime.AddObserver([&](TAutoPtr& ev) { + switch (ev->GetTypeRewrite()) { + case TEvTabletPipe::EvSend: + auto msg = ev->Get(); + if (msg != nullptr) { + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = pipeToTablet[ev->Recipient], + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), nodeIdToIndexMap[ev->Sender.NodeId()], true); + } + break; + } + })); + observers.emplace_back(runtime.AddObserver([&](TEvStatistics::TEvAggregateKeepAlive::TPtr& ev) { + if (ev->Sender == indexToActorMap[1]) { + ev.Reset(); + return; + } + })); auto sender = runtime.AllocateEdgeActor(); - runtime.Send(NStat::MakeStatServiceID(indexToNodeIdMap[0]), sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + runtime.Send(indexToActorMap[0], sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ .Round = 1, .PathId{3, 3}, .Nodes{ nodesTablets }, .ColumnTags{1, 2} }).release()); + auto res = runtime.GrabEdgeEvent(sender); const auto& record = res->Get()->Record; + size_t expectedFailedTabletsCount = 2; + UNIT_ASSERT_VALUES_EQUAL(expectedFailedTabletsCount, record.GetFailedTablets().size()); - std::unordered_map> expected = { - {1, {{"1", localTabletsIds.size()}, {"2", localTabletsIds.size()}}}, - {2, {{"3", nodesTablets.size() - 1 + localTabletsIds.size()}}}, - }; - - const auto& columns = record.GetColumns(); - for (const auto& column : columns) { - const auto tag = column.GetTag(); + ui32 expectedError = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_UNAVAILABLE_NODE; + for (const auto& fail : record.GetFailedTablets()) { + ui32 actualError = fail.GetError(); + UNIT_ASSERT_VALUES_EQUAL(expectedError, actualError); + } + } - for (auto& statistic : column.GetStatistics()) { - if (statistic.GetType() == NKikimr::NStat::COUNT_MIN_SKETCH) { - auto data = statistic.GetData().Data(); - auto sketch = reinterpret_cast(data); + Y_UNIT_TEST(ShouldBeCcorrectProcessingOfLocalTablets) { + size_t nodeCount = 1; + auto runtime = TTestActorRuntime(nodeCount, 1, false); + auto settings = GetDefaultSettings() + .SetMaxInFlightTabletRequests(3); + auto indexToActorMap = InitializeRuntime(runtime, nodeCount, settings); + auto nodeIdToIndexMap = GetNodeIdToIndexMap(indexToActorMap); + std::vector localTabletsIds = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector nodesTablets = {{.NodeId = indexToActorMap[0].NodeId(), .Ids{localTabletsIds}}}; - const auto& cells = expected[tag]; - for (auto it = cells.begin(); it != cells.end(); ++it) { - UNIT_ASSERT_VALUES_EQUAL(it->second, sketch->Probe(it->first.data(), it->first.size())); - } - } + std::vector observers; + observers.emplace_back(runtime.AddObserver([&](TEvTabletResolver::TEvForward::TPtr& ev) { + auto tabletId = ev->Get()->TabletID; + auto senderNodeIndex = nodeIdToIndexMap[ev->Sender.NodeId()]; + auto recipient = indexToActorMap[senderNodeIndex]; + + if (tabletId % 3 == 1) { + runtime.Send(new IEventHandle(recipient, ev->Sender, + new TEvTabletPipe::TEvClientConnected(tabletId, NKikimrProto::ERROR, ev->Sender, ev->Sender, + true, false, 0), 0, ev->Cookie), senderNodeIndex, true); + } else if(tabletId % 3 == 2) { + runtime.Send(new IEventHandle(recipient, ev->Sender, + new TEvTabletPipe::TEvClientDestroyed(tabletId, ev->Sender, ev->Sender), 0, ev->Cookie), senderNodeIndex, true); + } else { + auto actor = indexToActorMap[senderNodeIndex]; + runtime.Send(new IEventHandle(actor, actor, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = tabletId, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, 1), senderNodeIndex, true); } + ev.Reset(); + })); + + auto sender = runtime.AllocateEdgeActor(); + runtime.Send(indexToActorMap[0], sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + .Round = 1, + .PathId{3, 3}, + .Nodes{ nodesTablets }, + .ColumnTags{1} + }).release()); + + auto res = runtime.GrabEdgeEvent(sender); + const auto& record = res->Get()->Record; + size_t expectedFailedTabletsCount = 6; + UNIT_ASSERT_VALUES_EQUAL(expectedFailedTabletsCount, record.GetFailedTablets().size()); + + ui32 expectedError = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; + for (const auto& fail : record.GetFailedTablets()) { + ui32 actualError = fail.GetError(); + UNIT_ASSERT_VALUES_EQUAL(expectedError, actualError); } } } diff --git a/ydb/core/testlib/actors/block_events.cpp b/ydb/core/testlib/actors/block_events.cpp new file mode 100644 index 000000000000..801f1c5bf126 --- /dev/null +++ b/ydb/core/testlib/actors/block_events.cpp @@ -0,0 +1 @@ +#include "block_events.h" diff --git a/ydb/core/testlib/actors/block_events.h b/ydb/core/testlib/actors/block_events.h new file mode 100644 index 000000000000..2b76acf54eca --- /dev/null +++ b/ydb/core/testlib/actors/block_events.h @@ -0,0 +1,82 @@ +#include "test_runtime.h" + +#include +#include + +namespace NActors { + + /** + * Easy blocking for events under the test actor runtime + * + * Matching events are blocked just before they are processed and stashed + * into a deque. + */ + template + class TBlockEvents : public std::deque { + public: + TBlockEvents(TTestActorRuntime& runtime, std::function condition = {}) + : Runtime(runtime) + , Condition(std::move(condition)) + , Holder(Runtime.AddObserver( + [this](typename TEvType::TPtr& ev) { + this->Process(ev); + })) + {} + + /** + * Unblocks up to count events at the front of the deque, allowing them + * to be handled by the destination actor. + */ + TBlockEvents& Unblock(size_t count = -1) { + while (!this->empty() && count > 0) { + auto& ev = this->front(); + if (!Stopped) { + IEventHandle* ptr = ev.Get(); + UnblockedOnce.insert(ptr); + } + ui32 nodeId = ev->GetRecipientRewrite().NodeId(); + ui32 nodeIdx = nodeId - Runtime.GetFirstNodeId(); + Runtime.Send(ev.Release(), nodeIdx, /* viaActorSystem */ true); + this->pop_front(); + --count; + } + return *this; + } + + /** + * Stops blocking any new events. Events currently in the deque are + * not unblocked, but may be unblocked at a later time if needed. + */ + TBlockEvents& Stop() { + UnblockedOnce.clear(); + Holder.Remove(); + Stopped = true; + return *this; + } + + private: + void Process(typename TEvType::TPtr& ev) { + IEventHandle* ptr = ev.Get(); + auto it = UnblockedOnce.find(ptr); + if (it != UnblockedOnce.end()) { + UnblockedOnce.erase(it); + return; + } + + if (Condition && !Condition(ev)) { + return; + } + + this->emplace_back(std::move(ev)); + } + + private: + TTestActorRuntime& Runtime; + std::function Condition; + TTestActorRuntime::TEventObserverHolder Holder; + THashSet UnblockedOnce; + bool Stopped = false; + }; + + +} // namespace NActors diff --git a/ydb/core/testlib/actors/test_runtime.h b/ydb/core/testlib/actors/test_runtime.h index 3016bc441417..d57fcada8d28 100644 --- a/ydb/core/testlib/actors/test_runtime.h +++ b/ydb/core/testlib/actors/test_runtime.h @@ -68,22 +68,42 @@ namespace NActors { void SimulateSleep(TDuration duration); template - inline TResult WaitFuture(NThreading::TFuture f) { + inline TResult WaitFuture(NThreading::TFuture f, TDuration simTimeout = TDuration::Max()) { if (!f.HasValue() && !f.HasException()) { TDispatchOptions options; options.CustomFinalCondition = [&]() { return f.HasValue() || f.HasException(); }; - options.FinalEvents.emplace_back([&](IEventHandle&) { - return f.HasValue() || f.HasException(); - }); + // Quirk: non-empty FinalEvents enables full simulation + options.FinalEvents.emplace_back([](IEventHandle&) { return false; }); - this->DispatchEvents(options); + this->DispatchEvents(options, simTimeout); Y_ABORT_UNLESS(f.HasValue() || f.HasException()); } - return f.ExtractValue(); + if constexpr (!std::is_same_v) { + return f.ExtractValue(); + } else { + return f.GetValue(); + } + } + + template + inline void WaitFor(const TString& description, const TCondition& condition, TDuration simTimeout = TDuration::Max()) { + if (!condition()) { + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + return condition(); + }; + // Quirk: non-empty FinalEvents enables full simulation + options.FinalEvents.emplace_back([](IEventHandle&) { return false; }); + + Cerr << "... waiting for " << description << Endl; + this->DispatchEvents(options, simTimeout); + + Y_ABORT_UNLESS(condition(), "Timeout while waiting for %s", description.c_str()); + } } TIntrusivePtr GetMemObserver(ui32 nodeIndex = 0) { diff --git a/ydb/core/testlib/actors/test_runtime_ut.cpp b/ydb/core/testlib/actors/test_runtime_ut.cpp index d649df72fc89..e36b842df65a 100644 --- a/ydb/core/testlib/actors/test_runtime_ut.cpp +++ b/ydb/core/testlib/actors/test_runtime_ut.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -622,6 +623,209 @@ Y_UNIT_TEST_SUITE(TActorTest) { UNIT_ASSERT_VALUES_EQUAL(event->Get()->Index, 12u); } } -}; + + Y_UNIT_TEST(TestWaitFuture) { + enum EEv { + EvTrigger = EventSpaceBegin(TEvents::ES_PRIVATE) + }; + + struct TEvTrigger : public TEventLocal { + TEvTrigger() = default; + }; + + class TTriggerActor : public TActorBootstrapped { + public: + TTriggerActor(NThreading::TPromise promise) + : Promise(std::move(promise)) + {} + + void Bootstrap() { + Schedule(TDuration::Seconds(1), new TEvTrigger); + Become(&TThis::StateWork); + } + + private: + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTrigger, Handle); + } + } + + void Handle(TEvTrigger::TPtr&) { + Promise.SetValue(); + PassAway(); + } + + private: + NThreading::TPromise Promise; + }; + + TTestActorRuntime runtime; + runtime.Initialize(MakeEgg()); + + NThreading::TPromise promise = NThreading::NewPromise(); + NThreading::TFuture future = promise.GetFuture(); + + auto actor = runtime.Register(new TTriggerActor(std::move(promise))); + runtime.EnableScheduleForActor(actor); + + runtime.WaitFuture(std::move(future)); + } + + Y_UNIT_TEST(TestWaitFor) { + enum EEv { + EvTrigger = EventSpaceBegin(TEvents::ES_PRIVATE) + }; + + struct TEvTrigger : public TEventLocal { + TEvTrigger() = default; + }; + + class TTriggerActor : public TActorBootstrapped { + public: + TTriggerActor(int* ptr) + : Ptr(ptr) + {} + + void Bootstrap() { + Schedule(TDuration::Seconds(1), new TEvTrigger); + Become(&TThis::StateWork); + } + + private: + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTrigger, Handle); + } + } + + void Handle(TEvTrigger::TPtr&) { + *Ptr = 42; + PassAway(); + } + + private: + int* Ptr; + }; + + TTestActorRuntime runtime; + runtime.Initialize(MakeEgg()); + + int value = 0; + auto actor = runtime.Register(new TTriggerActor(&value)); + runtime.EnableScheduleForActor(actor); + + runtime.WaitFor("value = 42", [&]{ return value == 42; }); + UNIT_ASSERT_VALUES_EQUAL(value, 42); + } + + Y_UNIT_TEST(TestBlockEvents) { + enum EEv { + EvTrigger = EventSpaceBegin(TEvents::ES_PRIVATE) + }; + + struct TEvTrigger : public TEventLocal { + int Value; + + TEvTrigger(int value) + : Value(value) + {} + }; + + class TTargetActor : public TActorBootstrapped { + public: + TTargetActor(std::vector* ptr) + : Ptr(ptr) + {} + + void Bootstrap() { + Become(&TThis::StateWork); + } + + private: + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTrigger, Handle); + } + } + + void Handle(TEvTrigger::TPtr& ev) { + Ptr->push_back(ev->Get()->Value); + } + + private: + std::vector* Ptr; + }; + + class TSourceActor : public TActorBootstrapped { + public: + TSourceActor(const TActorId& target) + : Target(target) + {} + + void Bootstrap() { + Become(&TThis::StateWork); + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); + } + + private: + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvents::TEvWakeup, Handle); + } + } + + void Handle(TEvents::TEvWakeup::TPtr&) { + Send(Target, new TEvTrigger(++Counter)); + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup); + } + + private: + TActorId Target; + int Counter = 0; + }; + + TTestActorRuntime runtime(2); + runtime.Initialize(MakeEgg()); + + std::vector values; + auto target = runtime.Register(new TTargetActor(&values), /* nodeIdx */ 1); + auto source = runtime.Register(new TSourceActor(target), /* nodeIdx */ 1); + runtime.EnableScheduleForActor(source); + + TBlockEvents block(runtime, [&](TEvTrigger::TPtr& ev){ return ev->GetRecipientRewrite() == target; }); + runtime.WaitFor("blocked 3 events", [&]{ return block.size() >= 3; }); + UNIT_ASSERT_VALUES_EQUAL(block.size(), 3u); + UNIT_ASSERT_VALUES_EQUAL(values.size(), 0u); + + block.Unblock(2); + UNIT_ASSERT_VALUES_EQUAL(block.size(), 1u); + UNIT_ASSERT_VALUES_EQUAL(values.size(), 0u); + + runtime.WaitFor("blocked 1 more event", [&]{ return block.size() >= 2; }); + UNIT_ASSERT_VALUES_EQUAL(block.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(values.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(values.at(0), 1); + UNIT_ASSERT_VALUES_EQUAL(values.at(1), 2); + values.clear(); + + block.Stop(); + runtime.WaitFor("processed 2 more events", [&]{ return values.size() >= 2; }); + UNIT_ASSERT_VALUES_EQUAL(block.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(values.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(values.at(0), 5); + UNIT_ASSERT_VALUES_EQUAL(values.at(1), 6); + values.clear(); + + block.Unblock(); + UNIT_ASSERT_VALUES_EQUAL(block.size(), 0u); + UNIT_ASSERT_VALUES_EQUAL(values.size(), 0u); + runtime.WaitFor("processed 3 more events", [&]{ return values.size() >= 3; }); + UNIT_ASSERT_VALUES_EQUAL(values.size(), 3u); + UNIT_ASSERT_VALUES_EQUAL(values.at(0), 3); + UNIT_ASSERT_VALUES_EQUAL(values.at(1), 4); + UNIT_ASSERT_VALUES_EQUAL(values.at(2), 7); + } +} } diff --git a/ydb/core/testlib/actors/ya.make b/ydb/core/testlib/actors/ya.make index 25f814605794..9c7caacf445d 100644 --- a/ydb/core/testlib/actors/ya.make +++ b/ydb/core/testlib/actors/ya.make @@ -1,7 +1,10 @@ LIBRARY() SRCS( + block_events.cpp + block_events.h test_runtime.cpp + test_runtime.h ) PEERDIR( diff --git a/ydb/core/tx/columnshard/columnshard__statistics.cpp b/ydb/core/tx/columnshard/columnshard__statistics.cpp index 152445ce1ed4..86fbb2482cf8 100644 --- a/ydb/core/tx/columnshard/columnshard__statistics.cpp +++ b/ydb/core/tx/columnshard/columnshard__statistics.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NColumnShard { void TColumnShard::Handle(NStat::TEvStatistics::TEvAnalyzeTable::TPtr& ev, const TActorContext&) { - + auto& requestRecord = ev->Get()->Record; // TODO Start a potentially long analysis process. // ... @@ -14,6 +14,10 @@ void TColumnShard::Handle(NStat::TEvStatistics::TEvAnalyzeTable::TPtr& ev, const // Return the response when the analysis is completed auto response = std::make_unique(); + auto& responseRecord = response->Record; + responseRecord.SetOperationId(requestRecord.GetOperationId()); + responseRecord.MutablePathId()->CopyFrom(requestRecord.GetTable().GetPathId()); + responseRecord.SetShardTabletId(TabletID()); Send(ev->Sender, response.release(), 0, ev->Cookie); } diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp index 2710e08e9b17..14153b3e8912 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp @@ -4,6 +4,7 @@ #include "read_iterator.h" #include +#include #include #include #include @@ -4670,58 +4671,6 @@ Y_UNIT_TEST_SUITE(DataShardReadIteratorConsistency) { "result2: " << result2); } - template - class TBlockEvents : public std::deque { - public: - TBlockEvents(TTestActorRuntime& runtime, std::function condition = {}) - : Runtime(runtime) - , Condition(std::move(condition)) - , Holder(Runtime.AddObserver( - [this](typename TEvType::TPtr& ev) { - this->Process(ev); - })) - {} - - TBlockEvents& Unblock(size_t count = -1) { - while (!this->empty() && count > 0) { - auto& ev = this->front(); - IEventHandle* ptr = ev.Get(); - UnblockedOnce.insert(ptr); - Runtime.Send(ev.Release(), 0, /* viaActorSystem */ true); - this->pop_front(); - --count; - } - return *this; - } - - void Stop() { - UnblockedOnce.clear(); - Holder.Remove(); - } - - private: - void Process(typename TEvType::TPtr& ev) { - IEventHandle* ptr = ev.Get(); - auto it = UnblockedOnce.find(ptr); - if (it != UnblockedOnce.end()) { - UnblockedOnce.erase(it); - return; - } - - if (Condition && !Condition(ev)) { - return; - } - - this->emplace_back(std::move(ev)); - } - - private: - TTestActorRuntime& Runtime; - std::function Condition; - TTestActorRuntime::TEventObserverHolder Holder; - THashSet UnblockedOnce; - }; - Y_UNIT_TEST(Bug_7674_IteratorDuplicateRows) { TPortManager pm; TServerSettings serverSettings(pm.GetPort(2134)); diff --git a/ydb/core/tx/datashard/datashard_ut_volatile.cpp b/ydb/core/tx/datashard/datashard_ut_volatile.cpp index 1ca101657cb6..c0df06dff7a4 100644 --- a/ydb/core/tx/datashard/datashard_ut_volatile.cpp +++ b/ydb/core/tx/datashard/datashard_ut_volatile.cpp @@ -1226,8 +1226,12 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { "value: [\n" " 2\n" " ]\n"); + } + + SimulateSleep(runtime, TDuration::MilliSeconds(0)); - msg = readResults.back()->Get(); + { + auto* msg = readResults.back()->Get(); UNIT_ASSERT_VALUES_EQUAL(msg->Record.GetStatus().GetCode(), Ydb::StatusIds::SUCCESS); UNIT_ASSERT_VALUES_EQUAL(msg->Record.GetFinished(), true); } diff --git a/ydb/library/actors/testlib/test_runtime.cpp b/ydb/library/actors/testlib/test_runtime.cpp index 12c63197c658..d448287d5c04 100644 --- a/ydb/library/actors/testlib/test_runtime.cpp +++ b/ydb/library/actors/testlib/test_runtime.cpp @@ -1166,6 +1166,26 @@ namespace NActors { tempEdgeEventsCaptor.Reset(new TTempEdgeEventsCaptor(*this)); } + auto checkStopConditions = [&](bool perMessage = false) -> bool { + // Note: too many tests expect unrelated messages to be + // processed before simulation is stopped. + if (!perMessage) { + if (localContext.FinalEventFound) { + return true; + } + + if (!localContext.FoundNonEmptyMailboxes.empty()) { + return true; + } + } + + if (options.CustomFinalCondition && options.CustomFinalCondition()) { + return true; + } + + return false; + }; + TEventMailBoxList& currentMailboxes = useRestrictedMailboxes ? restrictedMailboxes : Mailboxes; while (!currentMailboxes.empty()) { bool hasProgress = true; @@ -1195,7 +1215,8 @@ namespace NActors { isEmpty = true; auto mboxIt = startWithMboxIt; TDeque suspectedBoxes; - while (true) { + bool stopCondition = false; + while (!stopCondition) { auto& mbox = *mboxIt; bool isIgnored = true; if (!mbox.second->IsEmpty()) { @@ -1264,6 +1285,9 @@ namespace NActors { case EEventAction::PROCESS: UpdateFinalEventsStatsForEachContext(*ev); SendInternal(ev.Release(), mbox.first.NodeId - FirstNodeId, false); + if (checkStopConditions(/* perMessage */ true)) { + stopCondition = true; + } break; case EEventAction::DROP: // do nothing @@ -1305,19 +1329,17 @@ namespace NActors { currentMailboxes.erase(it); } } + + if (stopCondition) { + return true; + } } } - if (localContext.FinalEventFound) { + if (checkStopConditions()) { return true; } - if (!localContext.FoundNonEmptyMailboxes.empty()) - return true; - - if (options.CustomFinalCondition && options.CustomFinalCondition()) - return true; - if (options.FinalEvents.empty()) { for (auto& mbox : currentMailboxes) { if (!mbox.second->IsActive(TInstant::MicroSeconds(CurrentTimestamp))) diff --git a/ydb/services/persqueue_v1/ut/partition_writer_cache_actor_fixture.cpp b/ydb/services/persqueue_v1/ut/partition_writer_cache_actor_fixture.cpp index d4e973a3a3c9..dde8e647a9bf 100644 --- a/ydb/services/persqueue_v1/ut/partition_writer_cache_actor_fixture.cpp +++ b/ydb/services/persqueue_v1/ut/partition_writer_cache_actor_fixture.cpp @@ -188,6 +188,9 @@ void TPartitionWriterCacheActorFixture::WaitForPartitionWriterOps(const TWaitFor }; Ctx->Runtime->DispatchEvents(options); + + // Tests rely on unrelated events processed after the condition is satisfied + Ctx->Runtime->DispatchEvents({}, TInstant::Zero()); } void TPartitionWriterCacheActorFixture::AdvanceCurrentTime(TDuration d) From dfe8109d465a13373860d5bd781066265c4cc5ec Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:40:02 +0000 Subject: [PATCH 04/17] TSerializedCellVec operator bool (#7981) --- ydb/core/scheme/scheme_tablecell.h | 8 ++++++++ ydb/core/scheme/scheme_tablecell_ut.cpp | 3 +++ 2 files changed, 11 insertions(+) diff --git a/ydb/core/scheme/scheme_tablecell.h b/ydb/core/scheme/scheme_tablecell.h index d2abd9f5d548..ed5f32fe1c9f 100644 --- a/ydb/core/scheme/scheme_tablecell.h +++ b/ydb/core/scheme/scheme_tablecell.h @@ -541,6 +541,14 @@ class TSerializedCellVec { return Cells; } + explicit operator bool() const + { + return !Cells.empty(); + } + + // read headers, assuming the buf is correct and append additional cells at the end + static bool UnsafeAppendCells(TConstArrayRef cells, TString& serializedCellVec); + static void Serialize(TString& res, TConstArrayRef cells); static TString Serialize(TConstArrayRef cells); diff --git a/ydb/core/scheme/scheme_tablecell_ut.cpp b/ydb/core/scheme/scheme_tablecell_ut.cpp index ae780b8cfc5f..a323a3e5d405 100644 --- a/ydb/core/scheme/scheme_tablecell_ut.cpp +++ b/ydb/core/scheme/scheme_tablecell_ut.cpp @@ -135,12 +135,15 @@ Y_UNIT_TEST_SUITE(Scheme) { 0); TSerializedCellVec vec3; + UNIT_ASSERT(!vec3); UNIT_ASSERT(vec3.GetCells().empty()); UNIT_ASSERT(vec3.GetBuffer().empty()); TString buf = vec.GetBuffer(); UNIT_ASSERT(buf.size() > cells.size()*2); vec3.Parse(buf); + UNIT_ASSERT(vec3); + UNIT_ASSERT_VALUES_EQUAL(CompareTypedCellVectors(vec3.GetCells().data(), cells.data(), types.data(), From 4333e32d0810e5a21a2ac9351e34647665af3ab9 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:45:27 +0000 Subject: [PATCH 05/17] Statistics: Traverse tests refactoring (#8110) --- ydb/core/statistics/aggregator/tx_init.cpp | 11 +- .../aggregator/ut/ut_analyze_columnshard.cpp | 185 ++++++++++--- .../aggregator/ut/ut_analyze_datashard.cpp | 8 +- .../aggregator/ut/ut_traverse_columnshard.cpp | 249 ++++-------------- .../aggregator/ut/ut_traverse_datashard.cpp | 16 +- ydb/core/statistics/ut_common/ut_common.cpp | 43 ++- ydb/core/statistics/ut_common/ut_common.h | 16 +- 7 files changed, 269 insertions(+), 259 deletions(-) diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index 0281115f9f26..de0bc44a0bca 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -227,14 +227,19 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { ui64 ownerId = rowset.GetValue(); ui64 localPathId = rowset.GetValue(); TString columnTags = rowset.GetValue(); - ui64 status = rowset.GetValue(); + TForceTraversalTable::EStatus status = (TForceTraversalTable::EStatus)rowset.GetValue(); + + if (status == TForceTraversalTable::EStatus::AnalyzeStarted) { + // Resent TEvAnalyzeTable to shards + status = TForceTraversalTable::EStatus::None; + } auto pathId = TPathId(ownerId, localPathId); TForceTraversalTable operationTable { .PathId = pathId, .ColumnTags = columnTags, - .Status = (TForceTraversalTable::EStatus)status, + .Status = status, }; auto forceTraversalOperation = Self->ForceTraversalOperation(operationId); if (!forceTraversalOperation) { @@ -270,7 +275,7 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { Self->InitializeStatisticsTable(); - if (Self->TraversalPathId) { + if (Self->TraversalPathId && Self->TraversalStartKey) { Self->NavigateType = ENavigateType::Traversal; Self->NavigatePathId = Self->TraversalPathId; Self->Navigate(); diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 5b90aa0362f8..0c4f1e5615b0 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -2,49 +2,20 @@ #include +#include + #include namespace NKikimr { namespace NStat { -struct TTableInfo { - std::vector ShardIds; - ui64 SaTabletId; - TPathId DomainKey; - TPathId PathId; -}; - -std::vector CreateDatabaseTables(TTestEnv& env, ui8 tableCount, ui8 shardCount) { - auto init = [&] () { - CreateDatabase(env, "Database"); - for (ui8 tableId = 1; tableId <= tableCount; tableId++) { - CreateColumnStoreTable(env, "Database", Sprintf("Table%u", tableId), shardCount); - } - }; - std::thread initThread(init); - - auto& runtime = *env.GetServer().GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - - runtime.SimulateSleep(TDuration::Seconds(10)); - initThread.join(); - - std::vector ret; - for (ui8 tableId = 1; tableId <= tableCount; tableId++) { - TTableInfo tableInfo; - const TString path = Sprintf("/Root/Database/Table%u", tableId); - tableInfo.ShardIds = GetColumnTableShards(runtime, sender, path); - tableInfo.PathId = ResolvePathId(runtime, path, &tableInfo.DomainKey, &tableInfo.SaTabletId); - ret.emplace_back(tableInfo); - } - return ret; -} + Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { Y_UNIT_TEST(AnalyzeOneColumnTable) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); - auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; AnalyzeTable(runtime, tableInfo.ShardIds[0], tableInfo.PathId); @@ -54,7 +25,7 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { Y_UNIT_TEST(AnalyzeAnalyzeOneColumnTableSpecificColumns) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); - auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; Analyze(runtime, tableInfo.SaTabletId, {{tableInfo.PathId, {1, 2}}}); } @@ -62,7 +33,7 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { Y_UNIT_TEST(AnalyzeTwoColumnTables) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); - auto tableInfos = CreateDatabaseTables(env, 2, 1); + auto tableInfos = CreateDatabaseColumnTables(env, 2, 1); Analyze(runtime, tableInfos[0].SaTabletId, {tableInfos[0].PathId, tableInfos[1].PathId}); } @@ -76,7 +47,7 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { ev.Reset(); }); - auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; const TString operationId = "operationId"; @@ -98,8 +69,7 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { Y_UNIT_TEST(AnalyzeSameOperationId) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); - auto tableInfo = CreateDatabaseTables(env, 1, 1)[0]; - + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; auto sender = runtime.AllocateEdgeActor(); const TString operationId = "operationId"; @@ -117,6 +87,145 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { UNIT_ASSERT(!response2); } + Y_UNIT_TEST(AnalyzeRebootSaBeforeAnalyzeTableResponse) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&) { + eventSeen = true; + }); + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + runtime.GrabEdgeEventRethrow(sender); + } + + Y_UNIT_TEST(AnalyzeRebootSaBeforeResolve) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + int observerCount = 0; + auto observer = runtime.AddObserver([&](auto&){ + observerCount++; + }); + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + runtime.WaitFor("TEvResolveKeySetResult", [&]{ return observerCount == 3; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + runtime.GrabEdgeEventRethrow(sender); + } + + Y_UNIT_TEST(AnalyzeRebootSaBeforeReqDistribution) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&) { + eventSeen = true; + }); + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + runtime.WaitFor("TEvRequestTabletDistribution", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + runtime.GrabEdgeEventRethrow(sender); + } + + Y_UNIT_TEST(AnalyzeRebootSaBeforeAggregate) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&){ + eventSeen = true; + }); + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + runtime.WaitFor("TEvAggregateStatistics", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + runtime.GrabEdgeEventRethrow(sender); + } + + Y_UNIT_TEST(AnalyzeRebootSaBeforeSave) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&){ + eventSeen = true; + }); + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + runtime.WaitFor("TEvAggregateStatisticsResponse", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + runtime.GrabEdgeEventRethrow(sender); + } + + // + Y_UNIT_TEST(AnalyzeRebootSaInAggregate) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; + auto sender = runtime.AllocateEdgeActor(); + + int observerCount = 0; + auto observer = runtime.AddObserver([&](auto&) { + observerCount++; + }); + + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + + runtime.WaitFor("5th TEvStatisticsRequest", [&]{ return observerCount == 5; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + + runtime.GrabEdgeEventRethrow(sender); + } + } } // NStat diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp index bfcecafa635c..766777482291 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp @@ -38,7 +38,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { Analyze(runtime, saTabletId, {{pathId}}); - ValidateCountMin(runtime, pathId); + ValidateCountMinDatashard(runtime, pathId); } Y_UNIT_TEST(AnalyzeTwoTables) { @@ -64,8 +64,8 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { Analyze(runtime, saTabletId1, {pathId1, pathId2}); - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); + ValidateCountMinDatashard(runtime, pathId1); + ValidateCountMinDatashard(runtime, pathId2); } @@ -96,7 +96,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { runtime.SimulateSleep(TDuration::Seconds(10)); - ValidateCountMinAbsense(runtime, pathId); + ValidateCountMinDatashardAbsense(runtime, pathId); } } diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp index ad7c926b78a0..ffe9e03889f4 100644 --- a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp @@ -7,8 +7,6 @@ #include #include -#include - namespace NKikimr { namespace NStat { @@ -16,211 +14,103 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { Y_UNIT_TEST(TraverseColumnTable) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(30)); - initThread.join(); - - auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; runtime.SimulateSleep(TDuration::Seconds(30)); - auto countMin = ExtractCountMin(runtime, pathId); - - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeResolve) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 saTabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); - + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; auto sender = runtime.AllocateEdgeActor(); - int observerCount = 0; - auto observer = runtime.AddObserver( - [&](TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) - { - if (observerCount++ == 2) { - RebootTablet(runtime, saTabletId, sender); - } - }); - runtime.SimulateSleep(TDuration::Seconds(30)); + int eventCount = 0; + auto observer = runtime.AddObserver([&](auto&) { + eventCount++; + }); - auto countMin = ExtractCountMin(runtime, pathId); + runtime.WaitFor("TEvResolveKeySetResult", [&]{ return eventCount == 3; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeReqDistribution) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 saTabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); - + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; auto sender = runtime.AllocateEdgeActor(); - bool observerFirstExec = true; - auto observer = runtime.AddObserver( - [&](TEvHive::TEvRequestTabletDistribution::TPtr& ev) - { - if (observerFirstExec) { - observerFirstExec = false; - RebootTablet(runtime, saTabletId, sender); - } - }); - runtime.SimulateSleep(TDuration::Seconds(30)); + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&){ + eventSeen = true; + }); - auto countMin = ExtractCountMin(runtime, pathId); + runtime.WaitFor("TEvRequestTabletDistribution", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeAggregate) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 saTabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); - + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; auto sender = runtime.AllocateEdgeActor(); - bool observerFirstExec = true; - auto observer = runtime.AddObserver( - [&](TEvStatistics::TEvAggregateStatistics::TPtr& ev) - { - if (observerFirstExec) { - observerFirstExec = false; - RebootTablet(runtime, saTabletId, sender); - } - }); - runtime.SimulateSleep(TDuration::Seconds(30)); + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&){ + eventSeen = true; + }); - auto countMin = ExtractCountMin(runtime, pathId); + runtime.WaitFor("TEvAggregateStatistics", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeSave) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 saTabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); - + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; auto sender = runtime.AllocateEdgeActor(); - bool observerFirstExec = true; - auto observer = runtime.AddObserver( - [&](TEvStatistics::TEvAggregateStatisticsResponse::TPtr& ev) - { - if (observerFirstExec) { - observerFirstExec = false; - RebootTablet(runtime, saTabletId, sender); - } - }); - runtime.SimulateSleep(TDuration::Seconds(30)); + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto&){ + eventSeen = true; + }); - auto countMin = ExtractCountMin(runtime, pathId); + runtime.WaitFor("TEvAggregateStatisticsResponse", [&]{ return eventSeen; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletInAggregate) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); - - ui64 saTabletId = 0; - auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); - + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; auto sender = runtime.AllocateEdgeActor(); + int observerCount = 0; - auto observer = runtime.AddObserver( - [&](TEvStatistics::TEvStatisticsRequest::TPtr& ev) - { - if (observerCount++ == 5) { - RebootTablet(runtime, saTabletId, sender); - } + auto observer = runtime.AddObserver([&](auto&){ + observerCount++; }); - runtime.SimulateSleep(TDuration::Seconds(30)); - - auto countMin = ExtractCountMin(runtime, pathId); + runtime.WaitFor("5th TEvStatisticsRequest", [&]{ return observerCount == 5; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableHiveDistributionZeroNodes) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; bool observerFirstExec = true; auto observer = runtime.AddObserver( @@ -258,25 +148,13 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); - auto countMin = ExtractCountMin(runtime, pathId); - - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableHiveDistributionAbsentNodes) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; bool observerFirstExec = true; auto observer = runtime.AddObserver( @@ -306,25 +184,13 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); - auto countMin = ExtractCountMin(runtime, pathId); - - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 10); + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); } Y_UNIT_TEST(TraverseColumnTableAggrStatUnavailableNode) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; bool observerFirstExec = true; auto observer = runtime.AddObserver( @@ -349,25 +215,13 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); - auto countMin = ExtractCountMin(runtime, pathId); - - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 11); // 10 for first round, 1 for second + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 11); // 10 for first round, 1 for second } Y_UNIT_TEST(TraverseColumnTableAggrStatNonLocalTablet) { TTestEnv env(1, 1); - auto init = [&] () { - CreateDatabase(env, "Database"); - CreateColumnStoreTable(env, "Database", "Table", 10); - }; - std::thread initThread(init); - auto& runtime = *env.GetServer().GetRuntime(); - runtime.SimulateSleep(TDuration::Seconds(5)); - initThread.join(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; bool observerFirstExec = true; auto observer = runtime.AddObserver( @@ -392,12 +246,7 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(60)); - auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); - auto countMin = ExtractCountMin(runtime, pathId); - - ui32 value = 1; - auto probe = countMin->Probe((const char *)&value, sizeof(value)); - UNIT_ASSERT_VALUES_EQUAL(probe, 11); // 10 for first round, 1 for second + ValidateCountMinColumnshard(runtime, tableInfo.PathId, 11); // 10 for first round, 1 for second } } diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp index 747826dcac84..cab2ef533c13 100644 --- a/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp @@ -34,7 +34,7 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { runtime.SimulateSleep(TDuration::Seconds(60)); auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); - ValidateCountMin(runtime, pathId); + ValidateCountMinDatashard(runtime, pathId); } Y_UNIT_TEST(TraverseTwoTables) { @@ -54,8 +54,8 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1"); auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2"); - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); + ValidateCountMinDatashard(runtime, pathId1); + ValidateCountMinDatashard(runtime, pathId2); } Y_UNIT_TEST(TraverseOneTableServerless) { @@ -85,7 +85,7 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { runtime.SimulateSleep(TDuration::Seconds(60)); auto pathId = ResolvePathId(runtime, "/Root/Serverless/Table"); - ValidateCountMin(runtime, pathId); + ValidateCountMinDatashard(runtime, pathId); } Y_UNIT_TEST(TraverseTwoTablesServerless) { @@ -117,8 +117,8 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Serverless/Table1"); auto pathId2 = ResolvePathId(runtime, "/Root/Serverless/Table2"); - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); + ValidateCountMinDatashard(runtime, pathId1); + ValidateCountMinDatashard(runtime, pathId2); } Y_UNIT_TEST(TraverseTwoTablesTwoServerlessDbs) { @@ -151,8 +151,8 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Serverless1/Table1"); auto pathId2 = ResolvePathId(runtime, "/Root/Serverless2/Table2"); - ValidateCountMin(runtime, pathId1); - ValidateCountMin(runtime, pathId2); + ValidateCountMinDatashard(runtime, pathId1); + ValidateCountMinDatashard(runtime, pathId2); } } diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index a95e14037e0f..ec4787cba99a 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -11,6 +11,9 @@ #include #include +// TODO remove thread +#include + using namespace NYdb; using namespace NYdb::NTable; using namespace NYdb::NScheme; @@ -252,6 +255,32 @@ void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TS UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } +std::vector CreateDatabaseColumnTables(TTestEnv& env, ui8 tableCount, ui8 shardCount) { + auto init = [&] () { + CreateDatabase(env, "Database"); + for (ui8 tableId = 1; tableId <= tableCount; tableId++) { + CreateColumnStoreTable(env, "Database", Sprintf("Table%u", tableId), shardCount); + } + }; + std::thread initThread(init); + + auto& runtime = *env.GetServer().GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SimulateSleep(TDuration::Seconds(10)); + initThread.join(); + + std::vector ret; + for (ui8 tableId = 1; tableId <= tableCount; tableId++) { + TTableInfo tableInfo; + const TString path = Sprintf("/Root/Database/Table%u", tableId); + tableInfo.ShardIds = GetColumnTableShards(runtime, sender, path); + tableInfo.PathId = ResolvePathId(runtime, path, &tableInfo.DomainKey, &tableInfo.SaTabletId); + ret.emplace_back(tableInfo); + } + return ret; +} + void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableName) { TTableClient client(env.GetDriver()); auto session = client.CreateSession().GetValueSync().GetSession(); @@ -262,7 +291,7 @@ void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableN UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } -std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPathId pathId) { +std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, const TPathId& pathId, ui64 columnTag) { auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); NStat::TRequest req; @@ -289,7 +318,15 @@ std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPa return stat.CountMin; } -void ValidateCountMin(TTestActorRuntime& runtime, TPathId pathId) { +void ValidateCountMinColumnshard(TTestActorRuntime& runtime, const TPathId& pathId, ui64 expectedProbe) { + auto countMin = ExtractCountMin(runtime, pathId); + + ui32 value = 1; + auto actualProbe = countMin->Probe((const char *)&value, sizeof(value)); + UNIT_ASSERT_VALUES_EQUAL(actualProbe, expectedProbe); +} + +void ValidateCountMinDatashard(TTestActorRuntime& runtime, TPathId pathId) { auto countMin = ExtractCountMin(runtime, pathId); for (ui32 i = 0; i < 4; ++i) { @@ -299,7 +336,7 @@ void ValidateCountMin(TTestActorRuntime& runtime, TPathId pathId) { } } -void ValidateCountMinAbsense(TTestActorRuntime& runtime, TPathId pathId) { +void ValidateCountMinDatashardAbsense(TTestActorRuntime& runtime, TPathId pathId) { auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); NStat::TRequest req; diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index 20126bd5de29..8a1a46c63371 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -66,6 +66,14 @@ void CreateDatabase(TTestEnv& env, const TString& databaseName, size_t nodeCount void CreateServerlessDatabase(TTestEnv& env, const TString& databaseName, TPathId resourcesDomainKey); +struct TTableInfo { + std::vector ShardIds; + ui64 SaTabletId; + TPathId DomainKey; + TPathId PathId; +}; +std::vector CreateDatabaseColumnTables(TTestEnv& env, ui8 tableCount, ui8 shardCount); + TPathId ResolvePathId(TTestActorRuntime& runtime, const TString& path, TPathId* domainKey = nullptr, ui64* saTabletId = nullptr); @@ -76,9 +84,11 @@ void CreateUniformTable(TTestEnv& env, const TString& databaseName, const TStrin void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TString& tableName, int shardCount); void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableName); -std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPathId pathId); -void ValidateCountMin(TTestActorRuntime& runtime, TPathId pathId); -void ValidateCountMinAbsense(TTestActorRuntime& runtime, TPathId pathId); +std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, const TPathId& pathId, ui64 columnTag = 1); +void ValidateCountMinColumnshard(TTestActorRuntime& runtime, const TPathId& pathId, ui64 expectedProbe); + +void ValidateCountMinDatashard(TTestActorRuntime& runtime, TPathId pathId); +void ValidateCountMinDatashardAbsense(TTestActorRuntime& runtime, TPathId pathId); struct TAnalyzedTable { TPathId PathId; From 61db185d06e50575617db375ed7e2617cd4d7582 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:46:19 +0000 Subject: [PATCH 06/17] Show a nicer demangled name in block/unblock messages (#8138) --- ydb/core/statistics/aggregator/aggregator_impl.cpp | 3 +++ .../statistics/aggregator/ut/ut_analyze_columnshard.cpp | 6 +++--- ydb/core/statistics/ut_common/ut_common.cpp | 3 +-- ydb/core/statistics/ut_common/ut_common.h | 2 +- ydb/core/testlib/actors/block_events.h | 8 ++++++++ 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 2a087684293f..98dde3778cc3 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -450,6 +450,9 @@ void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyzeStatus::TPtr& ev) { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); } } + + SA_LOG_D("[" << TabletID() << "] Send TEvStatistics::TEvAnalyzeStatusResponse. Status " << outRecord.GetStatus()); + Send(ev->Sender, response.release(), 0, ev->Cookie); } diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 0c4f1e5615b0..6b5beea5ace7 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -51,19 +51,19 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { const TString operationId = "operationId"; - AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); + AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); auto analyzeRequest = MakeAnalyzeRequest({{tableInfo.PathId, {1, 2}}}, operationId); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest.release()); - AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); + AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); schemeShardStatsBlocker.Remove(); auto analyzeResonse = runtime.GrabEdgeEventRethrow(sender); UNIT_ASSERT_VALUES_EQUAL(analyzeResonse->Get()->Record.GetOperationId(), operationId); - AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); + AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); } Y_UNIT_TEST(AnalyzeSameOperationId) { diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index ec4787cba99a..daea04ea379e 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -404,10 +404,9 @@ void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyze runtime.GrabEdgeEventRethrow(sender); } -void AnalyzeStatus(TTestActorRuntime& runtime, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus) { +void AnalyzeStatus(TTestActorRuntime& runtime, TActorId sender, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus) { auto analyzeStatusRequest = std::make_unique(); analyzeStatusRequest->Record.SetOperationId(operationId); - auto sender = runtime.AllocateEdgeActor(); runtime.SendToPipe(saTabletId, sender, analyzeStatusRequest.release()); auto analyzeStatusResponse = runtime.GrabEdgeEventRethrow(sender); diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index 8a1a46c63371..dc69f56bb78e 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -103,7 +103,7 @@ std::unique_ptr MakeAnalyzeRequest(const std::vector< void Analyze(TTestActorRuntime& runtime, ui64 saTabletId, const std::vector& table, const TString operationId = "operationId"); void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyzedTable& table); -void AnalyzeStatus(TTestActorRuntime& runtime, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus); +void AnalyzeStatus(TTestActorRuntime& runtime, TActorId sender, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus); } // namespace NStat diff --git a/ydb/core/testlib/actors/block_events.h b/ydb/core/testlib/actors/block_events.h index 2b76acf54eca..27eb3ba8e195 100644 --- a/ydb/core/testlib/actors/block_events.h +++ b/ydb/core/testlib/actors/block_events.h @@ -36,6 +36,10 @@ namespace NActors { } ui32 nodeId = ev->GetRecipientRewrite().NodeId(); ui32 nodeIdx = nodeId - Runtime.GetFirstNodeId(); + Cerr << "... unblocking " << (ev->HasEvent() ? TypeName(*ev->GetBase()) : TypeName()) + << " from " << Runtime.FindActorName(ev->Sender) + << " to " << Runtime.FindActorName(ev->GetRecipientRewrite()) + << Endl; Runtime.Send(ev.Release(), nodeIdx, /* viaActorSystem */ true); this->pop_front(); --count; @@ -67,6 +71,10 @@ namespace NActors { return; } + Cerr << "... blocking " << (ev->HasEvent() ? TypeName(*ev->GetBase()) : TypeName()) + << " from " << Runtime.FindActorName(ev->Sender) + << " to " << Runtime.FindActorName(ev->GetRecipientRewrite()) + << Endl; this->emplace_back(std::move(ev)); } From 089d3424f8c451d133e1e516965c8a87c38f150e Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:47:20 +0000 Subject: [PATCH 07/17] fix tests (#8323) --- ydb/core/formats/arrow/protos/ssa.proto | 4 + ydb/core/kqp/ut/olap/indexes_ut.cpp | 141 +++++++++++++++++- .../counters_statistics_aggregator.proto | 2 + ydb/core/protos/flat_scheme_op.proto | 11 ++ ydb/core/protos/out/out.cpp | 4 + ydb/core/protos/statistics.proto | 11 +- .../statistics/aggregator/aggregator_impl.cpp | 15 +- .../statistics/aggregator/aggregator_impl.h | 16 +- ydb/core/statistics/aggregator/schema.h | 4 +- ydb/core/statistics/aggregator/tx_analyze.cpp | 11 +- .../aggregator/tx_analyze_deadline.cpp | 65 ++++++++ .../tx_analyze_table_delivery_problem.cpp | 46 ++++++ ydb/core/statistics/aggregator/tx_init.cpp | 16 +- .../aggregator/tx_schedule_traversal.cpp | 5 +- .../aggregator/tx_schemeshard_stats.cpp | 2 - .../aggregator/ut/ut_analyze_columnshard.cpp | 79 ++++++++-- .../aggregator/ut/ut_traverse_columnshard.cpp | 98 +++++++++--- ydb/core/statistics/aggregator/ya.make | 2 + ydb/core/statistics/database/database.cpp | 7 +- ydb/core/statistics/service/service_impl.cpp | 19 ++- ydb/core/statistics/ut_common/ut_common.cpp | 24 ++- ydb/core/statistics/ut_common/ut_common.h | 15 +- ydb/core/statistics/ut_common/ya.make | 2 +- ydb/core/testlib/actors/test_runtime.h | 1 + .../columnshard/columnshard__statistics.cpp | 79 ++++++++-- .../columnshard/engines/scheme/index_info.cpp | 18 ++- .../columnshard/engines/scheme/index_info.h | 7 +- .../storage/actualizer/tiering/tiering.cpp | 2 +- .../engines/storage/indexes/bloom/meta.cpp | 2 +- .../engines/storage/indexes/bloom/meta.h | 2 +- .../indexes/count_min_sketch/checker.cpp | 22 +++ .../indexes/count_min_sketch/checker.h | 32 ++++ .../indexes/count_min_sketch/constructor.cpp | 63 ++++++++ .../indexes/count_min_sketch/constructor.h | 31 ++++ .../storage/indexes/count_min_sketch/meta.cpp | 56 +++++++ .../storage/indexes/count_min_sketch/meta.h | 63 ++++++++ .../storage/indexes/count_min_sketch/ya.make | 15 ++ .../engines/storage/indexes/portions/meta.cpp | 2 +- .../engines/storage/indexes/portions/meta.h | 4 +- .../engines/storage/indexes/ya.make | 1 + ydb/core/tx/columnshard/splitter/chunks.h | 2 +- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 55 +++++-- ydb/core/tx/schemeshard/schemeshard_impl.h | 2 +- ydb/library/minsketch/count_min_sketch.h | 2 - ydb/library/table_creator/table_creator.cpp | 16 +- ydb/library/table_creator/table_creator.h | 3 +- 46 files changed, 967 insertions(+), 112 deletions(-) create mode 100644 ydb/core/statistics/aggregator/tx_analyze_deadline.cpp create mode 100644 ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.cpp create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.cpp create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.h create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.cpp create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h create mode 100644 ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/ya.make diff --git a/ydb/core/formats/arrow/protos/ssa.proto b/ydb/core/formats/arrow/protos/ssa.proto index 5ffbf067b33d..193c759a3a80 100644 --- a/ydb/core/formats/arrow/protos/ssa.proto +++ b/ydb/core/formats/arrow/protos/ssa.proto @@ -45,6 +45,9 @@ message TProgram { repeated uint64 HashValues = 1; } + message TCountMinSketchChecker { + } + message TOlapIndexChecker { optional uint32 IndexId = 1; optional string ClassName = 2; @@ -56,6 +59,7 @@ message TProgram { oneof Implementation { TBloomFilterChecker BloomFilter = 40; TCompositeChecker Composite = 41; + TCountMinSketchChecker CountMinSketch = 42; } } diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp index feff93803fc5..1ef9943fd540 100644 --- a/ydb/core/kqp/ut/olap/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include namespace NKikimr::NKqp { @@ -59,7 +61,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { { auto alterQuery = TStringBuilder() << - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["uid"], "false_positive_probability" : 0.05}`); )"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); @@ -68,7 +70,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { } { auto alterQuery = TStringBuilder() << - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_resource_id, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_resource_id, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["resource_id", "level"], "false_positive_probability" : 0.05}`); )"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); @@ -105,6 +107,129 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { } } + Y_UNIT_TEST(CountMinSketchIndex) { + auto settings = TKikimrSettings() + .SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); + + TLocalHelper(kikimr).CreateTestOlapTable(); + auto tableClient = kikimr.GetTableClient(); + + Tests::NCommon::TLoggerInit(kikimr).SetComponents({NKikimrServices::TX_COLUMNSHARD}, "CS").SetPriority(NActors::NLog::PRI_DEBUG).Initialize(); + + { + auto alterQuery = TStringBuilder() << + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=cms_ts, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['timestamp']}`); + )"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + + { + auto alterQuery = TStringBuilder() << + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=cms_res_id, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['resource_id']}`); + )"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + + { + auto alterQuery = TStringBuilder() << + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=cms_uid, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['uid']}`); + )"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + + { + auto alterQuery = TStringBuilder() << + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=cms_level, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['level']}`); + )"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + + { + auto alterQuery = TStringBuilder() << + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=cms_message, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['message']}`); + )"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + + WriteTestData(kikimr, "/Root/olapStore/olapTable", 1000000, 300000000, 10000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 1100000, 300100000, 10000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 1200000, 300200000, 10000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 1300000, 300300000, 10000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 1400000, 300400000, 10000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 2000000, 200000000, 70000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 3000000, 100000000, 110000); + + csController->WaitActualization(TDuration::Seconds(10)); + { + auto runtime = kikimr.GetTestServer().GetRuntime(); + auto sender = runtime->AllocateEdgeActor(); + + TAutoPtr handle; + + size_t shard = 0; + std::set pathids; + for (auto&& i : csController->GetShardActualIds()) { + Cerr << ">>> shard actual id: " << i << Endl; + for (auto&& j : csController->GetPathIds(i)) { + Cerr << ">>> path id: " << j << Endl; + pathids.insert(j); + } + if (++shard == 3) + break; + } + + UNIT_ASSERT(pathids.size() == 1); + ui64 pathId = *pathids.begin(); + + shard = 0; + for (auto&& i : csController->GetShardActualIds()) { + auto request = std::make_unique(); + request->Record.MutableTable()->MutablePathId()->SetLocalId(pathId); + + runtime->Send(MakePipePerNodeCacheID(false), sender, new TEvPipeCache::TEvForward( + request.release(), i, false)); + if (++shard == 3) + break; + } + + auto sketch = std::unique_ptr(TCountMinSketch::Create()); + for (size_t shard = 0; shard < 3; ++shard) { + auto event = runtime->GrabEdgeEvent(handle); + UNIT_ASSERT(event); + + auto& response = event->Record; + // Cerr << response << Endl; + UNIT_ASSERT_VALUES_EQUAL(response.GetStatus(), NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS); + UNIT_ASSERT(response.ColumnsSize() == 5); + TString someData = response.GetColumns(0).GetStatistics(0).GetData(); + *sketch += *std::unique_ptr(TCountMinSketch::FromString(someData.data(), someData.size())); + Cerr << ">>> sketch.GetElementCount() = " << sketch->GetElementCount() << Endl; + UNIT_ASSERT(sketch->GetElementCount() > 0); + } + } + } + Y_UNIT_TEST(SchemeActualizationOnceOnStart) { auto settings = TKikimrSettings() .SetWithSampleTables(false); @@ -194,7 +319,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { { auto alterQuery = TStringBuilder() << Sprintf( - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["uid"], "false_positive_probability" : 0.05, "storage_id" : "%s"}`); )", StorageId.data()); auto session = tableClient.CreateSession().GetValueSync().GetSession(); @@ -203,7 +328,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { } { auto alterQuery = TStringBuilder() << Sprintf( - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_resource_id, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_resource_id, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["resource_id", "level"], "false_positive_probability" : 0.05, "storage_id" : "%s"}`); )", StorageId.data() ); @@ -347,7 +472,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { { auto alterQuery = TStringBuilder() << - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["uid"], "false_positive_probability" : 0.05}`); )"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); @@ -357,7 +482,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { { auto alterQuery = TStringBuilder() << - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["uid", "resource_id"], "false_positive_probability" : 0.05}`); )"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); @@ -367,7 +492,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { { auto alterQuery = TStringBuilder() << - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["uid"], "false_positive_probability" : 0.005}`); )"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); @@ -377,7 +502,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { { auto alterQuery = TStringBuilder() << - R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=BLOOM_FILTER, FEATURES=`{"column_names" : ["uid"], "false_positive_probability" : 0.01}`); )"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/protos/counters_statistics_aggregator.proto b/ydb/core/protos/counters_statistics_aggregator.proto index eb263bab214c..80cf9cbcbb4e 100644 --- a/ydb/core/protos/counters_statistics_aggregator.proto +++ b/ydb/core/protos/counters_statistics_aggregator.proto @@ -22,4 +22,6 @@ enum ETxTypes { TXTYPE_ACK_TIMEOUT = 12 [(TxTypeOpts) = {Name: "TxAckTimeout"}]; TXTYPE_ANALYZE_TABLE_REQUEST = 13 [(TxTypeOpts) = {Name: "TxAnalyzeTableRequest"}]; TXTYPE_ANALYZE_TABLE_RESPONSE = 14 [(TxTypeOpts) = {Name: "TxAnalyzeTableResponse"}]; + TXTYPE_ANALYZE_TABLE_DELIVERY_PROBLEM = 15 [(TxTypeOpts) = {Name: "TTxAnalyzeTableDeliveryProblem"}]; + TXTYPE_ANALYZE_DEADLINE = 16 [(TxTypeOpts) = {Name: "TTxAnalyzeDeadline"}]; } diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 1d385713a62a..22e35f6c12ec 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -452,6 +452,11 @@ message TRequestedMaxIndex { optional string ColumnName = 1; } +message TRequestedCountMinSketch { + // sketch built on the combined data from the set of columns + repeated string ColumnNames = 1; +} + message TOlapIndexRequested { optional string Name = 1; optional TCompressionOptions Compression = 3; @@ -461,6 +466,7 @@ message TOlapIndexRequested { oneof Implementation { TRequestedBloomFilter BloomFilter = 40; TRequestedMaxIndex MaxIndex = 41; + TRequestedCountMinSketch CountMinSketch = 42; } } @@ -474,6 +480,10 @@ message TMaxIndex { optional uint32 ColumnId = 1; } +message TCountMinSketch { + repeated uint32 ColumnIds = 1; +} + message TOlapIndexDescription { // This id is auto-generated by schemeshard optional uint32 Id = 1; @@ -487,6 +497,7 @@ message TOlapIndexDescription { oneof Implementation { TBloomFilter BloomFilter = 41; TMaxIndex MaxIndex = 42; + TCountMinSketch CountMinSketch = 43; } } diff --git a/ydb/core/protos/out/out.cpp b/ydb/core/protos/out/out.cpp index 735ac9b34a44..f85c5b9c0256 100644 --- a/ydb/core/protos/out/out.cpp +++ b/ydb/core/protos/out/out.cpp @@ -243,3 +243,7 @@ Y_DECLARE_OUT_SPEC(, NKikimrDataEvents::TEvWrite::ETxMode, stream, value) { Y_DECLARE_OUT_SPEC(, NKikimrStat::TEvAnalyzeStatusResponse_EStatus, stream, value) { stream << NKikimrStat::TEvAnalyzeStatusResponse_EStatus_Name(value); } + +Y_DECLARE_OUT_SPEC(, NKikimrStat::TEvStatisticsResponse::EStatus, stream, value) { + stream << NKikimrStat::TEvStatisticsResponse::EStatus_Name(value); +} diff --git a/ydb/core/protos/statistics.proto b/ydb/core/protos/statistics.proto index 08701e5ba138..838bc9f64e38 100644 --- a/ydb/core/protos/statistics.proto +++ b/ydb/core/protos/statistics.proto @@ -118,9 +118,9 @@ message TEvAnalyzeTable { // Shard -> SA message TEvAnalyzeTableResponse { - optional bytes OperationId = 1; - optional NKikimrProto.TPathID PathId = 2; - optional fixed64 ShardTabletId = 3; + optional bytes OperationId = 1; + optional NKikimrProto.TPathID PathId = 2; + optional fixed64 ShardTabletId = 3; } @@ -148,9 +148,8 @@ message TEvStatisticsResponse { enum EStatus { STATUS_UNSPECIFIED = 0; STATUS_SUCCESS = 1; - STATUS_PROCESSING = 2; - STATUS_ABORTED = 3; - STATUS_ERROR = 4; + STATUS_ABORTED = 2; + STATUS_ERROR = 3; } optional EStatus Status = 2; optional fixed64 ShardTabletId = 3; diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 98dde3778cc3..03777ef39b09 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -404,11 +404,24 @@ void TStatisticsAggregator::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { auto tabletId = ev->Get()->TabletId; if (TraversalIsColumnTable) { if (tabletId == HiveId) { + SA_LOG_E("[" << TabletID() << "] TEvDeliveryProblem with HiveId=" << tabletId); Schedule(HiveRetryInterval, new TEvPrivate::TEvRequestDistribution); } else { + for (TForceTraversalOperation& operation : ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + for (TAnalyzedShard& shard : operationTable.AnalyzedShards) { + if (shard.ShardTabletId == tabletId) { + SA_LOG_E("[" << TabletID() << "] TEvDeliveryProblem with ColumnShard=" << tabletId); + shard.Status = TAnalyzedShard::EStatus::DeliveryProblem; + return; + } + } + } + } SA_LOG_CRIT("[" << TabletID() << "] TEvDeliveryProblem with unexpected tablet " << tabletId); } } else { + SA_LOG_E("[" << TabletID() << "] TEvDeliveryProblem with DataShard=" << tabletId); if (DatashardRanges.empty()) { return; } @@ -745,8 +758,6 @@ TStatisticsAggregator::TForceTraversalOperation* TStatisticsAggregator::ForceTra } std::optional TStatisticsAggregator::IsColumnTable(const TPathId& pathId) const { - Y_ABORT_UNLESS(IsSchemeshardSeen); - auto itPath = ScheduleTraversals.find(pathId); if (itPath != ScheduleTraversals.end()) { bool ret = itPath->second.IsColumnTable; diff --git a/ydb/core/statistics/aggregator/aggregator_impl.h b/ydb/core/statistics/aggregator/aggregator_impl.h index e6b63b79e37f..3c24c27462b9 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.h +++ b/ydb/core/statistics/aggregator/aggregator_impl.h @@ -48,6 +48,8 @@ class TStatisticsAggregator : public TActor, public NTabl struct TTxAnalyze; struct TTxAnalyzeTableRequest; struct TTxAnalyzeTableResponse; + struct TTxAnalyzeTableDeliveryProblem; + struct TTxAnalyzeDeadline; struct TTxNavigate; struct TTxResolve; struct TTxDatashardScanResponse; @@ -68,6 +70,8 @@ class TStatisticsAggregator : public TActor, public NTabl EvResolve, EvAckTimeout, EvSendAnalyze, + EvAnalyzeDeliveryProblem, + EvAnalyzeDeadline, EvEnd }; @@ -80,6 +84,8 @@ class TStatisticsAggregator : public TActor, public NTabl struct TEvRequestDistribution : public TEventLocal {}; struct TEvResolve : public TEventLocal {}; struct TEvSendAnalyze : public TEventLocal {}; + struct TEvAnalyzeDeliveryProblem : public TEventLocal {}; + struct TEvAnalyzeDeadline : public TEventLocal {}; struct TEvAckTimeout : public TEventLocal { size_t SeqNo = 0; @@ -142,6 +148,8 @@ class TStatisticsAggregator : public TActor, public NTabl void Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev); void Handle(TEvPrivate::TEvAckTimeout::TPtr& ev); void Handle(TEvPrivate::TEvSendAnalyze::TPtr& ev); + void Handle(TEvPrivate::TEvAnalyzeDeliveryProblem::TPtr& ev); + void Handle(TEvPrivate::TEvAnalyzeDeadline::TPtr& ev); void InitializeStatisticsTable(); void Navigate(); @@ -204,6 +212,8 @@ class TStatisticsAggregator : public TActor, public NTabl hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); hFunc(TEvPrivate::TEvAckTimeout, Handle); hFunc(TEvPrivate::TEvSendAnalyze, Handle); + hFunc(TEvPrivate::TEvAnalyzeDeliveryProblem, Handle); + hFunc(TEvPrivate::TEvAnalyzeDeadline, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -251,7 +261,6 @@ class TStatisticsAggregator : public TActor, public NTabl std::queue PendingRequests; bool ProcessUrgentInFlight = false; - bool IsSchemeshardSeen = false; bool IsStatisticsTableCreated = false; bool PendingSaveStatistics = false; bool PendingDeleteStatistics = false; @@ -311,6 +320,9 @@ class TStatisticsAggregator : public TActor, public NTabl static constexpr size_t SendAnalyzeCount = 100; static constexpr TDuration SendAnalyzePeriod = TDuration::Seconds(1); + static constexpr TDuration AnalyzeDeliveryProblemPeriod = TDuration::Seconds(1); + static constexpr TDuration AnalyzeDeadline = TDuration::Days(1); + static constexpr TDuration AnalyzeDeadlinePeriod = TDuration::Seconds(1); enum ENavigateType { Analyze, @@ -348,6 +360,7 @@ class TStatisticsAggregator : public TActor, public NTabl enum class EStatus : ui8 { None, + DeliveryProblem, AnalyzeStarted, AnalyzeFinished, }; @@ -373,6 +386,7 @@ class TStatisticsAggregator : public TActor, public NTabl std::vector Tables; TString Types; TActorId ReplyToActorId; + TInstant CreatedAt; }; std::list ForceTraversals; diff --git a/ydb/core/statistics/aggregator/schema.h b/ydb/core/statistics/aggregator/schema.h index d897dc8cd8b5..592564ede6e3 100644 --- a/ydb/core/statistics/aggregator/schema.h +++ b/ydb/core/statistics/aggregator/schema.h @@ -51,11 +51,13 @@ struct TAggregatorSchema : NIceDb::Schema { struct ForceTraversalOperations : Table<6> { struct OperationId : Column<1, NScheme::NTypeIds::String> {}; struct Types : Column<2, NScheme::NTypeIds::String> {}; + struct CreatedAt : Column<3, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns< OperationId, - Types + Types, + CreatedAt >; }; diff --git a/ydb/core/statistics/aggregator/tx_analyze.cpp b/ydb/core/statistics/aggregator/tx_analyze.cpp index 87043ce0f02b..67868409ab21 100644 --- a/ydb/core/statistics/aggregator/tx_analyze.cpp +++ b/ydb/core/statistics/aggregator/tx_analyze.cpp @@ -18,7 +18,7 @@ struct TStatisticsAggregator::TTxAnalyze : public TTxBase { TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE; } - bool Execute(TTransactionContext& txc, const TActorContext&) override { + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyze::Execute. ReplyToActorId " << ReplyToActorId << " , Record " << Record); if (!Self->EnableColumnStatistics) { @@ -48,11 +48,13 @@ struct TStatisticsAggregator::TTxAnalyze : public TTxBase { const TString types = JoinVectorIntoString(TVector(Record.GetTypes().begin(), Record.GetTypes().end()), ","); // create new force trasersal + auto createdAt = ctx.Now(); TForceTraversalOperation operation { .OperationId = operationId, .Tables = {}, .Types = types, - .ReplyToActorId = ReplyToActorId + .ReplyToActorId = ReplyToActorId, + .CreatedAt = createdAt }; for (const auto& table : Record.GetTables()) { @@ -79,11 +81,12 @@ struct TStatisticsAggregator::TTxAnalyze : public TTxBase { ); } - Self->ForceTraversals.emplace_back(operation); + Self->ForceTraversals.emplace_back(operation); db.Table().Key(operationId).Update( NIceDb::TUpdate(operationId), - NIceDb::TUpdate(types) + NIceDb::TUpdate(types), + NIceDb::TUpdate(createdAt.GetValue()) ); return true; diff --git a/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp b/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp new file mode 100644 index 000000000000..67730beadb57 --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp @@ -0,0 +1,65 @@ +#include "aggregator_impl.h" + +#include +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyzeDeadline : public TTxBase { + TString OperationId; + TActorId ReplyToActorId; + + TTxAnalyzeDeadline(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_DEADLINE; } + + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeDeadline::Execute"); + + NIceDb::TNiceDb db(txc.DB); + auto now = ctx.Now(); + + for (TForceTraversalOperation& operation : Self->ForceTraversals) { + if (operation.CreatedAt + Self->AnalyzeDeadline < now) { + SA_LOG_E("[" << Self->TabletID() << "] Delete long analyze operation, OperationId=" << operation.OperationId); + + OperationId = operation.OperationId; + ReplyToActorId = operation.ReplyToActorId; + Self->DeleteForceTraversalOperation(operation.OperationId, db); + break; + } + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeDeadline::Complete"); + + if (OperationId) { + if (ReplyToActorId) { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeDeadline::Complete. " << + "Send TEvAnalyzeResponse for deleted operation, OperationId=" << OperationId << ", ActorId=" << ReplyToActorId); + auto response = std::make_unique(); + response->Record.SetOperationId(OperationId); + ctx.Send(ReplyToActorId, response.release()); + } else { + SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeDeadline::Complete. No ActorId to send reply. OperationId=" << OperationId); + } + ctx.Send(Self->SelfId(), new TEvPrivate::TEvAnalyzeDeadline()); + } else { + ctx.Schedule(AnalyzeDeadlinePeriod, new TEvPrivate::TEvAnalyzeDeadline()); + } + } +}; + +void TStatisticsAggregator::Handle(TEvPrivate::TEvAnalyzeDeadline::TPtr&) { + Execute(new TTxAnalyzeDeadline(this), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp b/ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp new file mode 100644 index 000000000000..6e04a23e339e --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp @@ -0,0 +1,46 @@ +#include "aggregator_impl.h" + +#include +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyzeTableDeliveryProblem : public TTxBase { + TTxAnalyzeTableDeliveryProblem(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE_DELIVERY_PROBLEM; } + + bool Execute(TTransactionContext&, const TActorContext&) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeTableDeliveryProblem::Execute"); + + for (TForceTraversalOperation& operation : Self->ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + for(TAnalyzedShard& analyzedShard : operationTable.AnalyzedShards) { + if (analyzedShard.Status == TAnalyzedShard::EStatus::DeliveryProblem) { + SA_LOG_D("[" << Self->TabletID() << "] Reset DeliveryProblem to ColumnShard=" << analyzedShard.ShardTabletId); + analyzedShard.Status = TAnalyzedShard::EStatus::None; + } + } + } + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeTableDeliveryProblem::Complete"); + + ctx.Schedule(AnalyzeDeliveryProblemPeriod, new TEvPrivate::TEvAnalyzeDeliveryProblem()); + } +}; + +void TStatisticsAggregator::Handle(TEvPrivate::TEvAnalyzeDeliveryProblem::TPtr&) { + Execute(new TTxAnalyzeTableDeliveryProblem(this), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index de0bc44a0bca..55eaa4a13148 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -194,12 +194,14 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { while (!rowset.EndOfSet()) { TString operationId = rowset.GetValue(); TString types = rowset.GetValue(); + ui64 createdAt = rowset.GetValue(); TForceTraversalOperation operation { .OperationId = operationId, .Tables = {}, .Types = types, - .ReplyToActorId = {} + .ReplyToActorId = {}, + .CreatedAt = TInstant::FromValue(createdAt) }; Self->ForceTraversals.emplace_back(operation); @@ -270,12 +272,20 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { Self->SubscribeForConfigChanges(ctx); Self->Schedule(Self->PropagateInterval, new TEvPrivate::TEvPropagate()); - Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); - Self->Schedule(Self->SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); + + if (Self->EnableColumnStatistics) { + Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); + Self->Schedule(Self->SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); + Self->Schedule(Self->AnalyzeDeliveryProblemPeriod, new TEvPrivate::TEvAnalyzeDeliveryProblem()); + Self->Schedule(Self->AnalyzeDeadlinePeriod, new TEvPrivate::TEvAnalyzeDeadline()); + } else { + SA_LOG_W("[" << Self->TabletID() << "] TTxInit::Complete. EnableColumnStatistics=false"); + } Self->InitializeStatisticsTable(); if (Self->TraversalPathId && Self->TraversalStartKey) { + SA_LOG_D("[" << Self->TabletID() << "] TTxInit::Complete. Start navigate. PathId " << Self->TraversalPathId); Self->NavigateType = ENavigateType::Traversal; Self->NavigatePathId = Self->TraversalPathId; Self->Navigate(); diff --git a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp index 2b2d81628a76..6300fb751c8b 100644 --- a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp +++ b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp @@ -12,7 +12,6 @@ struct TStatisticsAggregator::TTxScheduleTrasersal : public TTxBase { TTxType GetTxType() const override { return TXTYPE_SCHEDULE_TRAVERSAL; } bool Execute(TTransactionContext& txc, const TActorContext&) override { - SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Execute"); if (!Self->EnableColumnStatistics) { return true; @@ -23,11 +22,13 @@ struct TStatisticsAggregator::TTxScheduleTrasersal : public TTxBase { return true; } - if (!Self->IsSchemeshardSeen) { + if (Self->ScheduleTraversals.empty()) { SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal. No info from schemeshard"); return true; } + SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Execute"); + NIceDb::TNiceDb db(txc.DB); switch (Self->NavigateType) { diff --git a/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp b/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp index f81a99b4ac83..61efd9a2c743 100644 --- a/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp +++ b/ydb/core/statistics/aggregator/tx_schemeshard_stats.cpp @@ -76,8 +76,6 @@ struct TStatisticsAggregator::TTxSchemeShardStats : public TTxBase { void Complete(const TActorContext&) override { SA_LOG_D("[" << Self->TabletID() << "] TTxSchemeShardStats::Complete"); - - Self->IsSchemeshardSeen = true; } }; diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 6b5beea5ace7..4931963f4ab6 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -4,7 +4,7 @@ #include -#include +#include namespace NKikimr { namespace NStat { @@ -94,14 +94,16 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&) { + auto observer = runtime.AddObserver([&](auto& ev) { eventSeen = true; + ev.Reset(); }); auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); @@ -116,16 +118,21 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; auto sender = runtime.AllocateEdgeActor(); - int observerCount = 0; - auto observer = runtime.AddObserver([&](auto&){ - observerCount++; - }); + TBlockEvents block(runtime); auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); - - runtime.WaitFor("TEvResolveKeySetResult", [&]{ return observerCount == 3; }); + + runtime.WaitFor("1st TEvResolveKeySetResult", [&]{ return block.size() >= 1; }); + block.Unblock(1); + runtime.WaitFor("2nd TEvResolveKeySetResult", [&]{ return block.size() >= 1; }); + block.Unblock(1); + runtime.WaitFor("3rd TEvResolveKeySetResult", [&]{ return block.size() >= 1; }); + RebootTablet(runtime, tableInfo.SaTabletId, sender); + + block.Unblock(); + block.Stop(); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); @@ -140,14 +147,16 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&) { + auto observer = runtime.AddObserver([&](auto& ev) { eventSeen = true; + ev.Reset(); }); auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); runtime.WaitFor("TEvRequestTabletDistribution", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); @@ -163,14 +172,16 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&){ + auto observer = runtime.AddObserver([&](auto& ev){ eventSeen = true; + ev.Reset(); }); auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); runtime.WaitFor("TEvAggregateStatistics", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); @@ -186,14 +197,16 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&){ + auto observer = runtime.AddObserver([&](auto& ev){ eventSeen = true; + ev.Reset(); }); auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); runtime.WaitFor("TEvAggregateStatisticsResponse", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); @@ -202,7 +215,6 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { runtime.GrabEdgeEventRethrow(sender); } - // Y_UNIT_TEST(AnalyzeRebootSaInAggregate) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); @@ -210,14 +222,17 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto sender = runtime.AllocateEdgeActor(); int observerCount = 0; - auto observer = runtime.AddObserver([&](auto&) { - observerCount++; + auto observer = runtime.AddObserver([&](auto& ev) { + if (++observerCount >= 5) { + ev.Reset(); + } }); auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); - runtime.WaitFor("5th TEvStatisticsRequest", [&]{ return observerCount == 5; }); + runtime.WaitFor("5th TEvStatisticsRequest", [&]{ return observerCount >= 5; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}); @@ -226,6 +241,40 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { runtime.GrabEdgeEventRethrow(sender); } + Y_UNIT_TEST(AnalyzeRebootColumnShard) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + TBlockEvents block(runtime); + + auto analyzeRequest = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest.release()); + + runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return block.size(); }); + block.Stop(); + RebootTablet(runtime, tableInfo.ShardIds[0], sender); + + runtime.GrabEdgeEventRethrow(sender); + } + + Y_UNIT_TEST(AnalyzeDeadline) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + TBlockEvents block(runtime); + + auto analyzeRequest = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest.release()); + + runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return block.size(); }); + runtime.AdvanceCurrentTime(TDuration::Days(2)); + + runtime.GrabEdgeEventRethrow(sender); + } } } // NStat diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp index ffe9e03889f4..3c110788fd93 100644 --- a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp @@ -6,10 +6,29 @@ #include #include #include +#include namespace NKikimr { namespace NStat { +// TODO: check for arbitrary set of values of type T (including frequent duplicates) +// numbers (1..N) were count as a sketch. Check sketch properties +bool CheckCountMinSketch(const std::shared_ptr& sketch, const ui32 N) { + UNIT_ASSERT(sketch->GetElementCount() == N); + const double eps = 1. / sketch->GetWidth(); + const double delta = 1. / (1 << sketch->GetDepth()); + size_t failedEstimatesCount = 0; + for (ui32 i = 0; i < N; ++i) { + const ui32 trueCount = 1; // true count of value i + auto probe = sketch->Probe((const char *)&i, sizeof(i)); + if (probe > trueCount + eps * N) { + failedEstimatesCount++; + } + } + Cerr << ">>> failedEstimatesCount = " << failedEstimatesCount << Endl; + return failedEstimatesCount < delta * N; +} + Y_UNIT_TEST_SUITE(TraverseColumnShard) { Y_UNIT_TEST(TraverseColumnTable) { @@ -19,7 +38,9 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeResolve) { @@ -28,15 +49,23 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; auto sender = runtime.AllocateEdgeActor(); - int eventCount = 0; - auto observer = runtime.AddObserver([&](auto&) { - eventCount++; - }); + TBlockEvents block(runtime); + + runtime.WaitFor("1st TEvResolveKeySetResult", [&]{ return block.size() >= 1; }); + block.Unblock(1); + runtime.WaitFor("2nd TEvResolveKeySetResult", [&]{ return block.size() >= 1; }); + block.Unblock(1); + runtime.WaitFor("3rd TEvResolveKeySetResult", [&]{ return block.size() >= 1; }); - runtime.WaitFor("TEvResolveKeySetResult", [&]{ return eventCount == 3; }); RebootTablet(runtime, tableInfo.SaTabletId, sender); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + block.Unblock(); + block.Stop(); + + runtime.SimulateSleep(TDuration::Seconds(10)); + + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeReqDistribution) { @@ -46,14 +75,17 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&){ + auto observer = runtime.AddObserver([&](auto& ev){ eventSeen = true; + ev.Reset(); }); runtime.WaitFor("TEvRequestTabletDistribution", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeAggregate) { @@ -63,14 +95,17 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&){ + auto observer = runtime.AddObserver([&](auto& ev){ eventSeen = true; + ev.Reset(); }); runtime.WaitFor("TEvAggregateStatistics", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeSave) { @@ -80,14 +115,17 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { auto sender = runtime.AllocateEdgeActor(); bool eventSeen = false; - auto observer = runtime.AddObserver([&](auto&){ + auto observer = runtime.AddObserver([&](auto& ev){ eventSeen = true; + ev.Reset(); }); runtime.WaitFor("TEvAggregateStatisticsResponse", [&]{ return eventSeen; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableRebootSaTabletInAggregate) { @@ -97,14 +135,18 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { auto sender = runtime.AllocateEdgeActor(); int observerCount = 0; - auto observer = runtime.AddObserver([&](auto&){ - observerCount++; + auto observer = runtime.AddObserver([&](auto& ev){ + if (++observerCount >= 5) { + ev.Reset(); + } }); - runtime.WaitFor("5th TEvStatisticsRequest", [&]{ return observerCount == 5; }); + runtime.WaitFor("5th TEvStatisticsRequest", [&]{ return observerCount >= 5; }); + observer.Remove(); RebootTablet(runtime, tableInfo.SaTabletId, sender); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableHiveDistributionZeroNodes) { @@ -148,7 +190,8 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableHiveDistributionAbsentNodes) { @@ -184,7 +227,8 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 10); + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } Y_UNIT_TEST(TraverseColumnTableAggrStatUnavailableNode) { @@ -215,7 +259,13 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(30)); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 11); // 10 for first round, 1 for second + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + Cerr << "probe = " << probe << Endl; + const double eps = 1. / countMin->GetWidth(); + UNIT_ASSERT(probe <= 1 + eps * ColumnTableRowsNumber * 1.1); // 10 for first round, 1 for second } Y_UNIT_TEST(TraverseColumnTableAggrStatNonLocalTablet) { @@ -246,7 +296,13 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { runtime.SimulateSleep(TDuration::Seconds(60)); - ValidateCountMinColumnshard(runtime, tableInfo.PathId, 11); // 10 for first round, 1 for second + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + + ui32 value = 1; + auto probe = countMin->Probe((const char *)&value, sizeof(value)); + Cerr << "probe = " << probe << Endl; + const double eps = 1. / countMin->GetWidth(); + UNIT_ASSERT(probe <= 1 + eps * ColumnTableRowsNumber * 1.1); // 10 for first round, 1 for second } } diff --git a/ydb/core/statistics/aggregator/ya.make b/ydb/core/statistics/aggregator/ya.make index 8898d92bfc59..bab460ef0055 100644 --- a/ydb/core/statistics/aggregator/ya.make +++ b/ydb/core/statistics/aggregator/ya.make @@ -10,6 +10,8 @@ SRCS( tx_ack_timeout.cpp tx_aggr_stat_response.cpp tx_analyze.cpp + tx_analyze_deadline.cpp + tx_analyze_table_delivery_problem.cpp tx_analyze_table_request.cpp tx_analyze_table_response.cpp tx_configure.cpp diff --git a/ydb/core/statistics/database/database.cpp b/ydb/core/statistics/database/database.cpp index cb3e9dab7707..8b216f7da9d5 100644 --- a/ydb/core/statistics/database/database.cpp +++ b/ydb/core/statistics/database/database.cpp @@ -21,6 +21,10 @@ class TStatisticsTableCreator : public TActorBootstrapped { return; } + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Handle TEvStatistics::TEvGetStatistics, request id = " << requestId + << ", ReplyToActorId = " << request.ReplyToActorId + << ", StatRequests.size() = " << request.StatRequests.size()); + if (request.StatType == EStatType::COUNT_MIN_SKETCH) { request.StatResponses.reserve(request.StatRequests.size()); ui32 reqIndex = 0; @@ -694,6 +699,9 @@ class TStatService : public TActorBootstrapped { auto cookie = navigate->Cookie; + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult, request id = " << cookie); + if (cookie == ResolveSACookie) { Y_ABORT_UNLESS(navigate->ResultSet.size() == 1); auto& entry = navigate->ResultSet.back(); @@ -897,7 +905,7 @@ class TStatService : public TActorBootstrapped { void SendStatisticsRequest(const TActorId& clientId) { auto request = std::make_unique(); auto& record = request->Record; - record.MutableTypes()->Add(NKikimr::NStat::COUNT_MIN_SKETCH); + record.MutableTypes()->Add(NKikimrStat::TYPE_COUNT_MIN_SKETCH); auto* path = record.MutableTable()->MutablePathId(); path->SetOwnerId(AggregationStatistics.PathId.OwnerId); @@ -909,6 +917,11 @@ class TStatService : public TActorBootstrapped { } NTabletPipe::SendData(SelfId(), clientId, request.release(), AggregationStatistics.Round); + + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "TEvStatisticsRequest send" + << ", client id = " << clientId + << ", path = " << *path); } void OnTabletError(ui64 tabletId) { @@ -1108,7 +1121,9 @@ class TStatService : public TActorBootstrapped { auto& request = itRequest->second; LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, - "ReplySuccess(), request id = " << requestId); + "ReplySuccess(), request id = " << requestId + << ", ReplyToActorId = " << request.ReplyToActorId + << ", StatRequests.size() = " << request.StatRequests.size()); auto itStatistics = Statistics.find(request.SchemeShardId); if (itStatistics == Statistics.end()) { diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index daea04ea379e..efdc2eb37b92 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -43,7 +43,8 @@ NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString return subdomain; } -TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools) { +TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool useRealThreads) + : CSController(NYDBTest::TControllers::RegisterCSControllerGuard()) { auto mbusPort = PortManager.GetPort(); auto grpcPort = PortManager.GetPort(); @@ -77,6 +78,10 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools) { DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint); Driver = MakeHolder(DriverConfig); + CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + CSController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); + CSController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); + Server->GetRuntime()->SetLogPriority(NKikimrServices::STATISTICS, NActors::NLog::PRI_DEBUG); } @@ -238,9 +243,21 @@ void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TS )", fullTableName.c_str(), shardCount)).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + result = session.ExecuteSchemeQuery(Sprintf(R"( + ALTER OBJECT `%s` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, NAME=cms_key, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['Key']}`); + )", fullTableName.c_str())).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + result = session.ExecuteSchemeQuery(Sprintf(R"( + ALTER OBJECT `%s` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, NAME=cms_value, TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['Value']}`); + )", fullTableName.c_str())).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + NYdb::TValueBuilder rows; rows.BeginList(); - for (size_t i = 0; i < 100; ++i) { + for (size_t i = 0; i < ColumnTableRowsNumber; ++i) { auto key = TValueBuilder().Uint64(i).Build(); auto value = TValueBuilder().OptionalUint64(i).Build(); rows.AddListItem(); @@ -250,9 +267,10 @@ void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TS rows.EndStruct(); } rows.EndList(); - result = client.BulkUpsert(fullTableName, rows.Build()).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + env.GetController()->WaitActualization(TDuration::Seconds(1)); } std::vector CreateDatabaseColumnTables(TTestEnv& env, ui8 tableCount, ui8 shardCount) { diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index dc69f56bb78e..1caf652aa43d 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -2,6 +2,8 @@ #include +#include + #include #include @@ -12,12 +14,14 @@ namespace NKikimrStat { namespace NKikimr { namespace NStat { +static constexpr ui32 ColumnTableRowsNumber = 1000; + NKikimrSubDomains::TSubDomainSettings GetSubDomainDeclareSettings( const TString &name, const TStoragePools &pools = {}); NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings( const TString &name, const TStoragePools &pools = {}); - + class TTestEnv { public: TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 1, ui32 storagePools = 1); @@ -49,6 +53,10 @@ class TTestEnv { TStoragePools GetPools() const; + auto& GetController() { + return CSController; + } + private: TPortManager PortManager; @@ -60,6 +68,7 @@ class TTestEnv { TString Endpoint; NYdb::TDriverConfig DriverConfig; THolder Driver; + NYDBTest::TControllers::TGuard CSController; }; void CreateDatabase(TTestEnv& env, const TString& databaseName, size_t nodeCount = 1); @@ -68,7 +77,7 @@ void CreateServerlessDatabase(TTestEnv& env, const TString& databaseName, TPathI struct TTableInfo { std::vector ShardIds; - ui64 SaTabletId; + ui64 SaTabletId; TPathId DomainKey; TPathId PathId; }; @@ -93,7 +102,7 @@ void ValidateCountMinDatashardAbsense(TTestActorRuntime& runtime, TPathId pathId struct TAnalyzedTable { TPathId PathId; std::vector ColumnTags; - + TAnalyzedTable(const TPathId& pathId); TAnalyzedTable(const TPathId& pathId, const std::vector& columnTags); void ToProto(NKikimrStat::TTable& tableProto) const; diff --git a/ydb/core/statistics/ut_common/ya.make b/ydb/core/statistics/ut_common/ya.make index 38e6f63e24ed..94514bd506fe 100644 --- a/ydb/core/statistics/ut_common/ya.make +++ b/ydb/core/statistics/ut_common/ya.make @@ -6,10 +6,10 @@ SRCS( ) PEERDIR( + ydb/core/tx/columnshard/hooks/testing ydb/core/testlib ) YQL_LAST_ABI_VERSION() END() - diff --git a/ydb/core/testlib/actors/test_runtime.h b/ydb/core/testlib/actors/test_runtime.h index d57fcada8d28..f3a403dcdf3d 100644 --- a/ydb/core/testlib/actors/test_runtime.h +++ b/ydb/core/testlib/actors/test_runtime.h @@ -103,6 +103,7 @@ namespace NActors { this->DispatchEvents(options, simTimeout); Y_ABORT_UNLESS(condition(), "Timeout while waiting for %s", description.c_str()); + Cerr << "... waiting for " << description << " (done)" << Endl; } } diff --git a/ydb/core/tx/columnshard/columnshard__statistics.cpp b/ydb/core/tx/columnshard/columnshard__statistics.cpp index 86fbb2482cf8..84b1a89982b8 100644 --- a/ydb/core/tx/columnshard/columnshard__statistics.cpp +++ b/ydb/core/tx/columnshard/columnshard__statistics.cpp @@ -1,7 +1,12 @@ #include "columnshard.h" #include "columnshard_impl.h" +#include "ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h" #include +#include + +#include + namespace NKikimr::NColumnShard { @@ -22,23 +27,73 @@ void TColumnShard::Handle(NStat::TEvStatistics::TEvAnalyzeTable::TPtr& ev, const } void TColumnShard::Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev, const TActorContext&) { + const auto& record = ev->Get()->Record; + auto response = std::make_unique(); - auto& record = response->Record; - record.SetShardTabletId(TabletID()); + auto& respRecord = response->Record; + respRecord.SetShardTabletId(TabletID()); + + if (record.TypesSize() > 0 && (record.TypesSize() > 1 || record.GetTypes(0) != NKikimrStat::TYPE_COUNT_MIN_SKETCH)) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("error", "Unsupported statistic type in statistics request"); + + respRecord.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_ERROR); + + Send(ev->Sender, response.release(), 0, ev->Cookie); + return; + } + + AFL_VERIFY(HasIndex()); + auto index = GetIndexAs(); + auto spg = index.GetGranuleOptional(record.GetTable().GetPathId().GetLocalId()); + AFL_VERIFY(spg); + + std::set columnTagsRequested; + for (ui32 tag : record.GetTable().GetColumnTags()) { + columnTagsRequested.insert(tag); + } + if (columnTagsRequested.empty()) { + auto schema = index.GetVersionedIndex().GetLastSchema(); + auto allColumnIds = schema->GetIndexInfo().GetColumnIds(false); + columnTagsRequested = std::set(allColumnIds.begin(), allColumnIds.end()); + } + + std::map> sketchesByColumns; + for (auto id : columnTagsRequested) { + sketchesByColumns.emplace(id, TCountMinSketch::Create()); + } + + for (const auto& [_, portionInfo] : spg->GetPortions()) { + if (portionInfo->IsVisible(GetMaxReadVersion())) { + std::shared_ptr portionSchema = portionInfo->GetSchema(index.GetVersionedIndex()); + for (ui32 columnId : columnTagsRequested) { + auto indexMeta = portionSchema->GetIndexInfo().GetIndexMetaCountMinSketch({columnId}); + + if (!indexMeta) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("error", "Missing countMinSketch index for columnId " + ToString(columnId)); + continue; + } + AFL_VERIFY(indexMeta->GetColumnIds().size() == 1); + + const std::vector data = portionInfo->GetIndexInplaceDataVerified(indexMeta->GetIndexId()); - record.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS); + for (const auto& sketchAsString : data) { + auto sketch = std::unique_ptr(TCountMinSketch::FromString(sketchAsString.data(), sketchAsString.size())); + *sketchesByColumns[columnId] += *sketch; + } + } + } + } - std::unique_ptr sketch(TCountMinSketch::Create()); - ui32 value = 1; - sketch->Count((const char*)&value, sizeof(value)); - TString strSketch(sketch->AsStringBuf()); + respRecord.SetStatus(NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS); - auto* column = record.AddColumns(); - column->SetTag(1); + for (ui32 columnTag : columnTagsRequested) { + auto* column = respRecord.AddColumns(); + column->SetTag(columnTag); - auto* statistic = column->AddStatistics(); - statistic->SetType(NStat::COUNT_MIN_SKETCH); - statistic->SetData(std::move(strSketch)); + auto* statistic = column->AddStatistics(); + statistic->SetType(NStat::COUNT_MIN_SKETCH); + statistic->SetData(TString(sketchesByColumns[columnTag]->AsStringBuf())); + } Send(ev->Sender, response.release(), 0, ev->Cookie); } diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 5bdfc2838eb9..835d1c71c2bd 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -336,7 +337,7 @@ void TIndexInfo::InitializeCaches(const std::shared_ptr& opera for (auto&& c : Columns) { AFL_VERIFY(ArrowColumnByColumnIdCache.emplace(c.first, GetColumnFieldVerified(c.first)).second); - AFL_VERIFY(ColumnFeatures.emplace(c.first, TColumnFeatures(c.first, GetColumnFieldVerified(c.first), DefaultSerializer, operators->GetDefaultOperator(), + AFL_VERIFY(ColumnFeatures.emplace(c.first, TColumnFeatures(c.first, GetColumnFieldVerified(c.first), DefaultSerializer, operators->GetDefaultOperator(), NArrow::IsPrimitiveYqlType(c.second.PType), c.first == GetPKFirstColumnId(), nullptr)).second); } for (auto&& cId : GetSystemColumnIds()) { @@ -409,7 +410,7 @@ NKikimr::TConclusionStatus TIndexInfo::AppendIndex(const THashMap TIndexInfo::GetIndexMax(const ui32 columnId) const { +std::shared_ptr TIndexInfo::GetIndexMetaMax(const ui32 columnId) const { for (auto&& i : Indexes) { if (i.second->GetClassName() != NIndexes::NMax::TIndexMeta::GetClassNameStatic()) { continue; @@ -422,6 +423,19 @@ std::shared_ptr TIndexInfo::GetIndexMax(const ui32 c return nullptr; } +std::shared_ptr TIndexInfo::GetIndexMetaCountMinSketch(const std::set& columnIds) const { + for (auto&& i : Indexes) { + if (i.second->GetClassName() != NIndexes::NCountMinSketch::TIndexMeta::GetClassNameStatic()) { + continue; + } + auto index = static_pointer_cast(i.second.GetObjectPtr()); + if (index->GetColumnIds() == columnIds) { + return index; + } + } + return nullptr; +} + std::vector TIndexInfo::GetEntityIds() const { auto result = GetColumnIds(true); for (auto&& i : Indexes) { diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index 622f8c741050..5a92fbc3d6c3 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -28,6 +28,10 @@ namespace NIndexes::NMax { class TIndexMeta; } +namespace NIndexes::NCountMinSketch { +class TIndexMeta; +} + namespace NStorageOptimizer { class IOptimizerPlannerConstructor; } @@ -226,7 +230,8 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { return result; } - std::shared_ptr GetIndexMax(const ui32 columnId) const; + std::shared_ptr GetIndexMetaMax(const ui32 columnId) const; + std::shared_ptr GetIndexMetaCountMinSketch(const std::set& columnIds) const; [[nodiscard]] TConclusionStatus AppendIndex(const THashMap>>& originalData, const ui32 indexId, const std::shared_ptr& operators, TSecondaryData& result) const; diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index 87fecc13e472..88a8ca36f031 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -26,7 +26,7 @@ std::optional TTieringActualizer::Bu if (Tiering) { AFL_VERIFY(TieringColumnId); - auto indexMeta = portionSchema->GetIndexInfo().GetIndexMax(*TieringColumnId); + auto indexMeta = portionSchema->GetIndexInfo().GetIndexMetaMax(*TieringColumnId); std::shared_ptr max; if (!indexMeta) { max = portion.MaxValue(*TieringColumnId); diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/bloom/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/bloom/meta.cpp index 19d283d7f9e5..fc89fb8b8ada 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/bloom/meta.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/bloom/meta.cpp @@ -66,4 +66,4 @@ void TBloomIndexMeta::DoFillIndexCheckers(const std::shared_ptr +#include +#include +#include + +namespace NKikimr::NOlap::NIndexes::NCountMinSketch { + +void TCountMinSketchChecker::DoSerializeToProtoImpl(NKikimrSSA::TProgram::TOlapIndexChecker& proto) const { + proto.MutableCountMinSketch(); +} + +bool TCountMinSketchChecker::DoCheckImpl(const std::vector& blobs) const { + Y_UNUSED(blobs); + return true; +} + +bool TCountMinSketchChecker::DoDeserializeFromProtoImpl(const NKikimrSSA::TProgram::TOlapIndexChecker& proto) { + return proto.HasCountMinSketch(); +} + +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h new file mode 100644 index 000000000000..ffa073b2e400 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h @@ -0,0 +1,32 @@ +#pragma once +#include + +namespace NKikimr::NOlap::NIndexes::NCountMinSketch { + +class TCountMinSketchChecker: public TSimpleIndexChecker { +public: + static TString GetClassNameStatic() { + return "COUNT_MIN_SKETCH"; + } +private: + using TBase = TSimpleIndexChecker; + static inline auto Registrator = TFactory::TRegistrator(GetClassNameStatic()); + +protected: + virtual bool DoDeserializeFromProtoImpl(const NKikimrSSA::TProgram::TOlapIndexChecker& proto) override; + virtual void DoSerializeToProtoImpl(NKikimrSSA::TProgram::TOlapIndexChecker& proto) const override; + + virtual bool DoCheckImpl(const std::vector& blobs) const override; + +public: + TCountMinSketchChecker() = default; + TCountMinSketchChecker(const ui32 indexId) + : TBase(indexId) + {} + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +}; + +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.cpp new file mode 100644 index 000000000000..7dcbaa9db476 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.cpp @@ -0,0 +1,63 @@ +#include "constructor.h" +#include "meta.h" + +#include + +namespace NKikimr::NOlap::NIndexes::NCountMinSketch { + +std::shared_ptr TCountMinSketchConstructor::DoCreateIndexMeta(const ui32 indexId, const TString& indexName, const NSchemeShard::TOlapSchema& currentSchema, NSchemeShard::IErrorCollector& errors) const { + std::set columnIds; + if (ColumnNames.empty()) { + for (const auto& [id, _] : currentSchema.GetColumns().GetColumns()) { + AFL_VERIFY(columnIds.emplace(id).second); + } + } + for (auto&& i : ColumnNames) { + auto* columnInfo = currentSchema.GetColumns().GetByName(i); + if (!columnInfo) { + errors.AddError("no column with name " + i); + return nullptr; + } + AFL_VERIFY(columnIds.emplace(columnInfo->GetId()).second); + } + return std::make_shared(indexId, indexName, GetStorageId().value_or(NBlobOperations::TGlobal::LocalMetadataStorageId), columnIds); +} + +NKikimr::TConclusionStatus TCountMinSketchConstructor::DoDeserializeFromJson(const NJson::TJsonValue& jsonInfo) { + if (!jsonInfo.Has("column_names")) { + return TConclusionStatus::Fail("column_names have to be in count min sketch features"); + } + const NJson::TJsonValue::TArray* columnNamesArray; + if (!jsonInfo["column_names"].GetArrayPointer(&columnNamesArray)) { + return TConclusionStatus::Fail("column_names have to be in count min sketch features as array ['column_name_1', ... , 'column_name_N']"); + } + for (auto&& i : *columnNamesArray) { + if (!i.IsString()) { + return TConclusionStatus::Fail("column_names have to be in count min sketch features as array of strings ['column_name_1', ... , 'column_name_N']"); + } + ColumnNames.emplace(i.GetString()); + } + return TConclusionStatus::Success(); +} + +NKikimr::TConclusionStatus TCountMinSketchConstructor::DoDeserializeFromProto(const NKikimrSchemeOp::TOlapIndexRequested& proto) { + if (!proto.HasCountMinSketch()) { + const TString errorMessage = "not found CountMinSketch section in proto: \"" + proto.DebugString() + "\""; + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("problem", errorMessage); + return TConclusionStatus::Fail(errorMessage); + } + auto& sketch = proto.GetCountMinSketch(); + for (auto&& i : sketch.GetColumnNames()) { + ColumnNames.emplace(i); + } + return TConclusionStatus::Success(); +} + +void TCountMinSketchConstructor::DoSerializeToProto(NKikimrSchemeOp::TOlapIndexRequested& proto) const { + auto* sketchProto = proto.MutableCountMinSketch(); + for (auto&& i : ColumnNames) { + sketchProto->AddColumnNames(i); + } +} + +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.h b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.h new file mode 100644 index 000000000000..86d7e34fa577 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/constructor.h @@ -0,0 +1,31 @@ +#pragma once +#include + +namespace NKikimr::NOlap::NIndexes::NCountMinSketch { + +class TCountMinSketchConstructor: public IIndexMetaConstructor { +public: + static TString GetClassNameStatic() { + return "COUNT_MIN_SKETCH"; + } +private: + std::set ColumnNames; + static inline auto Registrator = TFactory::TRegistrator(GetClassNameStatic()); + +protected: + virtual std::shared_ptr DoCreateIndexMeta(const ui32 indexId, const TString& indexName, const NSchemeShard::TOlapSchema& currentSchema, NSchemeShard::IErrorCollector& errors) const override; + + virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonInfo) override; + + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TOlapIndexRequested& proto) override; + virtual void DoSerializeToProto(NKikimrSchemeOp::TOlapIndexRequested& proto) const override; + +public: + TCountMinSketchConstructor() = default; + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +}; + +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.cpp new file mode 100644 index 000000000000..5e0465848b29 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.cpp @@ -0,0 +1,56 @@ +#include "meta.h" +#include "checker.h" +#include +#include +#include +#include +#include + +#include +#include + +namespace NKikimr::NOlap::NIndexes::NCountMinSketch { + +TString TIndexMeta::DoBuildIndexImpl(TChunkedBatchReader& reader) const { + auto sketch = std::unique_ptr(TCountMinSketch::Create()); + + for (auto& colReader : reader) { + for (colReader.Start(); colReader.IsCorrect(); colReader.ReadNextChunk()) { + auto array = colReader.GetCurrentChunk(); + + NArrow::SwitchType(array->type_id(), [&](const auto& type) { + using TWrap = std::decay_t; + using TArray = typename arrow::TypeTraits::ArrayType; + + const TArray& arrTyped = static_cast(*array); + if constexpr (arrow::has_c_type()) { + for (int64_t i = 0; i < arrTyped.length(); ++i) { + auto cell = TCell::Make(arrTyped.Value(i)); + sketch->Count(cell.Data(), cell.Size()); + } + return true; + } + if constexpr (arrow::has_string_view()) { + for (int64_t i = 0; i < arrTyped.length(); ++i) { + auto view = arrTyped.GetView(i); + sketch->Count(view.data(), view.size()); + } + return true; + } + AFL_VERIFY(false)("message", "Unsupported arrow type for building an index"); + return false; + }); + } + } + + TString result(sketch->AsStringBuf()); + return result; +} + +void TIndexMeta::DoFillIndexCheckers(const std::shared_ptr& info, const NSchemeShard::TOlapSchema& /*schema*/) const { + for (auto&& branch : info->GetBranches()) { + branch->MutableIndexes().emplace_back(std::make_shared(GetIndexId())); + } +} + +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h new file mode 100644 index 000000000000..2c23af1fefdb --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h @@ -0,0 +1,63 @@ +#pragma once +#include + +namespace NKikimr::NOlap::NIndexes::NCountMinSketch { + +class TIndexMeta: public TIndexByColumns { +public: + static TString GetClassNameStatic() { + return "COUNT_MIN_SKETCH"; + } + +private: + using TBase = TIndexByColumns; + + static inline auto Registrator = TFactory::TRegistrator(GetClassNameStatic()); + +protected: + virtual TConclusionStatus DoCheckModificationCompatibility(const IIndexMeta& newMeta) const override { + const auto* bMeta = dynamic_cast(&newMeta); + if (!bMeta) { + return TConclusionStatus::Fail("cannot read meta as appropriate class: " + GetClassName() + ". Meta said that class name is " + newMeta.GetClassName()); + } + return TBase::CheckSameColumnsForModification(newMeta); + } + + virtual void DoFillIndexCheckers(const std::shared_ptr& info, const NSchemeShard::TOlapSchema& schema) const override; + + virtual TString DoBuildIndexImpl(TChunkedBatchReader& reader) const override; + + virtual bool DoDeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto) override { + AFL_VERIFY(TBase::DoDeserializeFromProto(proto)); + AFL_VERIFY(proto.HasCountMinSketch()); + auto& sketch = proto.GetCountMinSketch(); + for (auto&& i : sketch.GetColumnIds()) { + ColumnIds.emplace(i); + } + return true; + } + + virtual void DoSerializeToProto(NKikimrSchemeOp::TOlapIndexDescription& proto) const override { + auto* sketchProto = proto.MutableCountMinSketch(); + for (auto&& i : ColumnIds) { + sketchProto->AddColumnIds(i); + } + } + +public: + TIndexMeta() = default; + TIndexMeta(const ui32 indexId, const TString& indexName, const TString& storageId, const std::set& columnIds) + : TBase(indexId, indexName, columnIds, storageId) { + } + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } + + const std::set& GetColumnIds() const { + return ColumnIds; + } + +}; + +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/ya.make b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/ya.make new file mode 100644 index 000000000000..bcba53e477ae --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +SRCS( + GLOBAL constructor.cpp + GLOBAL meta.cpp + GLOBAL checker.cpp +) + +PEERDIR( + ydb/core/protos + ydb/core/formats/arrow + ydb/core/tx/columnshard/engines/storage/indexes/portions +) + +END() diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.cpp index cd5f4f3c51cc..fd21cb5055cb 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.cpp @@ -63,4 +63,4 @@ NKikimr::TConclusionStatus TIndexByColumns::CheckSameColumnsForModification(cons return TConclusionStatus::Success(); } -} // namespace NKikimr::NOlap::NIndexes \ No newline at end of file +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.h b/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.h index 3f2f5dfb872f..e083c12fa927 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.h +++ b/ydb/core/tx/columnshard/engines/storage/indexes/portions/meta.h @@ -55,8 +55,10 @@ class TIndexByColumns: public IIndexMeta { private: using TBase = IIndexMeta; std::shared_ptr Serializer; + protected: std::set ColumnIds; + virtual TString DoBuildIndexImpl(TChunkedBatchReader& reader) const = 0; virtual std::shared_ptr DoBuildIndex(const THashMap>>& data, const TIndexInfo& indexInfo) const override final; @@ -69,4 +71,4 @@ class TIndexByColumns: public IIndexMeta { TIndexByColumns(const ui32 indexId, const TString& indexName, const std::set& columnIds, const TString& storageId); }; -} // namespace NKikimr::NOlap::NIndexes \ No newline at end of file +} // namespace NKikimr::NOlap::NIndexes diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/ya.make b/ydb/core/tx/columnshard/engines/storage/indexes/ya.make index 2edfa9332cd4..0459c906d836 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/ya.make +++ b/ydb/core/tx/columnshard/engines/storage/indexes/ya.make @@ -4,6 +4,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/storage/indexes/portions ydb/core/tx/columnshard/engines/storage/indexes/bloom ydb/core/tx/columnshard/engines/storage/indexes/max + ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch ) END() diff --git a/ydb/core/tx/columnshard/splitter/chunks.h b/ydb/core/tx/columnshard/splitter/chunks.h index 280f47d8c238..bb0bbb287ce7 100644 --- a/ydb/core/tx/columnshard/splitter/chunks.h +++ b/ydb/core/tx/columnshard/splitter/chunks.h @@ -58,7 +58,7 @@ class TChunkedColumnReader { ui32 CurrentRecordIndex = 0; public: TChunkedColumnReader(const std::vector>& chunks, const std::shared_ptr& loader) - : Chunks(chunks) + : Chunks(chunks) , Loader(loader) { Start(); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index a3ff33cb2cc8..d43c77e0ce7c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -7281,6 +7281,10 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvLogin::TPtr &ev, const TActorContex } void TSchemeShard::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext&) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ", at schemeshard: " << TabletID()); + using TNavigate = NSchemeCache::TSchemeCacheNavigate; std::unique_ptr request(ev->Get()->Request.Release()); if (request->ResultSet.size() != 1) { @@ -7293,15 +7297,19 @@ void TSchemeShard::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& if (entry.DomainInfo->Params.HasStatisticsAggregator()) { StatisticsAggregatorId = TTabletId(entry.DomainInfo->Params.GetStatisticsAggregator()); + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult, StatisticsAggregatorId=" << StatisticsAggregatorId + << ", at schemeshard: " << TabletID()); ConnectToSA(); } } void TSchemeShard::Handle(TEvPrivate::TEvSendBaseStatsToSA::TPtr&, const TActorContext& ctx) { - SendBaseStatsToSA(); - auto seconds = SendStatsIntervalMaxSeconds - SendStatsIntervalMinSeconds; - ctx.Schedule(TDuration::Seconds(SendStatsIntervalMinSeconds + RandomNumber(seconds)), - new TEvPrivate::TEvSendBaseStatsToSA()); + TDuration delta = SendBaseStatsToSA(); + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Schedule next SendBaseStatsToSA in " << delta + << ", at schemeshard: " << TabletID()); + ctx.Schedule(delta, new TEvPrivate::TEvSendBaseStatsToSA()); } void TSchemeShard::InitializeStatistics(const TActorContext& ctx) { @@ -7325,12 +7333,21 @@ void TSchemeShard::ResolveSA() { Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); } else { StatisticsAggregatorId = subDomainInfo->GetTenantStatisticsAggregatorID(); + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "ResolveSA(), StatisticsAggregatorId=" << StatisticsAggregatorId + << ", at schemeshard: " << TabletID()); ConnectToSA(); } } void TSchemeShard::ConnectToSA() { - if (!EnableStatistics || !StatisticsAggregatorId) { + if (!EnableStatistics) + return; + + if (!StatisticsAggregatorId) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "ConnectToSA(), no StatisticsAggregatorId" + << ", at schemeshard: " << TabletID()); return; } auto policy = NTabletPipe::TClientRetryPolicy::WithRetries(); @@ -7345,18 +7362,28 @@ void TSchemeShard::ConnectToSA() { LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "ConnectToSA()" << ", pipe client id: " << SAPipeClientId - << ", at schemeshard: " << TabletID()); + << ", at schemeshard: " << TabletID() + << ", StatisticsAggregatorId: " << StatisticsAggregatorId + << ", at schemeshard: " << TabletID() + ); } -void TSchemeShard::SendBaseStatsToSA() { +TDuration TSchemeShard::SendBaseStatsToSA() { if (!EnableStatistics) { - return; + return TDuration::Max(); } if (!SAPipeClientId) { ResolveSA(); if (!StatisticsAggregatorId) { - return; + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "SendBaseStatsToSA(), no StatisticsAggregatorId" + << ", at schemeshard: " << TabletID()); + return TDuration::Seconds(30); + } else { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "SendBaseStatsToSA(), StatisticsAggregatorId=" << StatisticsAggregatorId + << ", at schemeshard: " << TabletID()); } } @@ -7388,6 +7415,13 @@ void TSchemeShard::SendBaseStatsToSA() { ++count; } + if (!count) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "SendBaseStatsToSA() No tables to send" + << ", at schemeshard: " << TabletID()); + return TDuration::Seconds(30); + } + TString stats; stats.clear(); Y_PROTOBUF_SUPPRESS_NODISCARD record.SerializeToString(&stats); @@ -7402,6 +7436,9 @@ void TSchemeShard::SendBaseStatsToSA() { "SendBaseStatsToSA()" << ", path count: " << count << ", at schemeshard: " << TabletID()); + + return TDuration::Seconds(SendStatsIntervalMinSeconds + + RandomNumber(SendStatsIntervalMaxSeconds - SendStatsIntervalMinSeconds)); } } // namespace NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 56e4bb9cfbe9..53b328713d47 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1395,7 +1395,7 @@ class TSchemeShard void InitializeStatistics(const TActorContext& ctx); void ResolveSA(); void ConnectToSA(); - void SendBaseStatsToSA(); + TDuration SendBaseStatsToSA(); diff --git a/ydb/library/minsketch/count_min_sketch.h b/ydb/library/minsketch/count_min_sketch.h index c64b9250dd48..4b8866d3d921 100644 --- a/ydb/library/minsketch/count_min_sketch.h +++ b/ydb/library/minsketch/count_min_sketch.h @@ -3,8 +3,6 @@ #include #include -#include - namespace NKikimr { class TCountMinSketch { diff --git a/ydb/library/table_creator/table_creator.cpp b/ydb/library/table_creator/table_creator.cpp index f208b14c8d00..2f96883bff9a 100644 --- a/ydb/library/table_creator/table_creator.cpp +++ b/ydb/library/table_creator/table_creator.cpp @@ -35,13 +35,15 @@ using TTableCreatorRetryPolicy = IRetryPolicy; TVector keyColumns, NKikimrServices::EServiceKikimr logService, TMaybe ttlSettings = Nothing(), - bool isSystemUser = false) + bool isSystemUser = false, + TMaybe partitioningPolicy = Nothing()) : PathComponents(std::move(pathComponents)) , Columns(std::move(columns)) , KeyColumns(std::move(keyColumns)) , LogService(logService) , TtlSettings(std::move(ttlSettings)) , IsSystemUser(isSystemUser) + , PartitioningPolicy(std::move(partitioningPolicy)) , LogPrefix("Table " + TableName() + " updater. ") { Y_ABORT_UNLESS(!PathComponents.empty()); @@ -113,6 +115,11 @@ using TTableCreatorRetryPolicy = IRetryPolicy; if (IsSystemUser) { request->Record.SetUserToken(NACLib::TSystemUsers::Metadata().SerializeAsString()); } + if (PartitioningPolicy) { + auto* partitioningPolicy = tableDesc->MutablePartitionConfig()->MutablePartitioningPolicy(); + partitioningPolicy->CopyFrom(*PartitioningPolicy); + } + Send(MakeTxProxyID(), std::move(request)); } @@ -382,6 +389,7 @@ using TTableCreatorRetryPolicy = IRetryPolicy; NKikimrServices::EServiceKikimr LogService; const TMaybe TtlSettings; bool IsSystemUser = false; + const TMaybe PartitioningPolicy; NKikimrSchemeOp::EOperationType OperationType = NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable; NActors::TActorId Owner; NActors::TActorId SchemePipeActorId; @@ -480,10 +488,12 @@ NActors::IActor* CreateTableCreator( TVector keyColumns, NKikimrServices::EServiceKikimr logService, TMaybe ttlSettings, - bool isSystemUser) + bool isSystemUser, + TMaybe partitioningPolicy) { return new TTableCreator(std::move(pathComponents), std::move(columns), - std::move(keyColumns), logService, std::move(ttlSettings), isSystemUser); + std::move(keyColumns), logService, std::move(ttlSettings), isSystemUser, + std::move(partitioningPolicy)); } } // namespace NKikimr diff --git a/ydb/library/table_creator/table_creator.h b/ydb/library/table_creator/table_creator.h index 90d61e6ebc76..39e40d428016 100644 --- a/ydb/library/table_creator/table_creator.h +++ b/ydb/library/table_creator/table_creator.h @@ -77,6 +77,7 @@ NActors::IActor* CreateTableCreator( TVector keyColumns, NKikimrServices::EServiceKikimr logService, TMaybe ttlSettings = Nothing(), - bool isSystemUser = false); + bool isSystemUser = false, + TMaybe partitioningPolicy = Nothing()); } // namespace NKikimr From ccbd4ccbc61425caad5c0d307cc96a7f0dc54031 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Thu, 29 Aug 2024 15:47:52 +0000 Subject: [PATCH 08/17] Statistics: updated HTML page (#8471) --- .../counters_statistics_aggregator.proto | 19 ++ ydb/core/protos/out/out.cpp | 4 + ydb/core/protos/statistics.proto | 7 + .../statistics/aggregator/aggregator_impl.cpp | 163 ++++++++++++++---- .../statistics/aggregator/aggregator_impl.h | 10 ++ .../aggregator/tx_aggr_stat_response.cpp | 4 + ydb/core/statistics/aggregator/tx_analyze.cpp | 1 + .../aggregator/tx_analyze_deadline.cpp | 1 + .../aggregator/tx_finish_trasersal.cpp | 1 + ydb/core/statistics/aggregator/tx_init.cpp | 5 + .../tx_response_tablet_distribution.cpp | 1 + .../aggregator/tx_schedule_traversal.cpp | 8 +- .../aggregator/ut/ut_analyze_columnshard.cpp | 75 +++++++- .../aggregator/ut/ut_analyze_datashard.cpp | 6 +- .../aggregator/ut/ut_traverse_columnshard.cpp | 27 ++- .../aggregator/ut/ut_traverse_datashard.cpp | 16 +- ydb/core/statistics/database/database.cpp | 155 +++++++++++++++-- ydb/core/statistics/database/database.h | 10 +- .../statistics/database/ut/ut_database.cpp | 12 +- ydb/core/statistics/events.h | 10 ++ ydb/core/statistics/service/service.cpp | 2 + ydb/core/statistics/service/service.h | 6 + ydb/core/statistics/service/service_impl.cpp | 53 +++++- ydb/core/statistics/service/ut/ut_service.cpp | 61 +++++++ ydb/core/statistics/ut_common/ut_common.cpp | 15 +- ydb/core/statistics/ut_common/ut_common.h | 1 + 26 files changed, 581 insertions(+), 92 deletions(-) diff --git a/ydb/core/protos/counters_statistics_aggregator.proto b/ydb/core/protos/counters_statistics_aggregator.proto index 80cf9cbcbb4e..2df3f516c26e 100644 --- a/ydb/core/protos/counters_statistics_aggregator.proto +++ b/ydb/core/protos/counters_statistics_aggregator.proto @@ -25,3 +25,22 @@ enum ETxTypes { TXTYPE_ANALYZE_TABLE_DELIVERY_PROBLEM = 15 [(TxTypeOpts) = {Name: "TTxAnalyzeTableDeliveryProblem"}]; TXTYPE_ANALYZE_DEADLINE = 16 [(TxTypeOpts) = {Name: "TTxAnalyzeDeadline"}]; } + +enum ESimpleCounters { + COUNTER_FORCE_TRAVERSALS_INFLIGHT_SIZE = 0 [(CounterOpts) = {Name: "ForceTraversalsInflightSize"}]; + COUNTER_AGGREGATION_TIME = 1 [(CounterOpts) = {Name: "AggregationTime"}]; + COUNTER_FORCE_TRAVERSAL_INFLIGHT_MAX_TIME = 2 [(CounterOpts) = {Name: "ForceTraversalsInflightMaxTime"}]; +} + +enum ECumulativeCounters { + COUNTER_CUMULATIVE_IGNORE = 0; +} + +enum EPercentileCounters { + option (GlobalCounterOpts) = { + Ranges { Value: 0 Name: "0 ms" } + Ranges { Value: 1 Name: "1 ms" } + }; + + COUNTER_PERCENTILE_IGNORE = 0; +} diff --git a/ydb/core/protos/out/out.cpp b/ydb/core/protos/out/out.cpp index f85c5b9c0256..72bb4d1e22ba 100644 --- a/ydb/core/protos/out/out.cpp +++ b/ydb/core/protos/out/out.cpp @@ -240,6 +240,10 @@ Y_DECLARE_OUT_SPEC(, NKikimrDataEvents::TEvWrite::ETxMode, stream, value) { stream << NKikimrDataEvents::TEvWrite::ETxMode_Name(value); } +Y_DECLARE_OUT_SPEC(, NKikimrStat::TEvAnalyzeResponse_EStatus, stream, value) { + stream << NKikimrStat::TEvAnalyzeResponse_EStatus_Name(value); +} + Y_DECLARE_OUT_SPEC(, NKikimrStat::TEvAnalyzeStatusResponse_EStatus, stream, value) { stream << NKikimrStat::TEvAnalyzeStatusResponse_EStatus_Name(value); } diff --git a/ydb/core/protos/statistics.proto b/ydb/core/protos/statistics.proto index 838bc9f64e38..5ecebcaa32c2 100644 --- a/ydb/core/protos/statistics.proto +++ b/ydb/core/protos/statistics.proto @@ -89,6 +89,13 @@ message TEvAnalyze { // SA -> KQP message TEvAnalyzeResponse { optional bytes OperationId = 1; + + enum EStatus { + STATUS_UNSPECIFIED = 0; + STATUS_SUCCESS = 1; + STATUS_ERROR = 2; + } + optional EStatus Status = 2; } // KQP -> SA diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 03777ef39b09..ef16ee226551 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include @@ -21,6 +23,14 @@ TStatisticsAggregator::TStatisticsAggregator(const NActors::TActorId& tablet, TT auto seed = std::random_device{}(); RandomGenerator.seed(seed); + + TabletCountersPtr.Reset(new TProtobufTabletCounters< + ESimpleCounters_descriptor, + ECumulativeCounters_descriptor, + EPercentileCounters_descriptor, + ETxTypes_descriptor + >()); + TabletCounters = TabletCountersPtr.Get(); } void TStatisticsAggregator::OnDetach(const TActorContext& ctx) { @@ -34,6 +44,7 @@ void TStatisticsAggregator::OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const void TStatisticsAggregator::OnActivateExecutor(const TActorContext& ctx) { SA_LOG_I("[" << TabletID() << "] OnActivateExecutor"); + Executor()->RegisterExternalTabletCounters(TabletCountersPtr); Execute(CreateTxInitSchema(), ctx); } @@ -483,8 +494,15 @@ void TStatisticsAggregator::Handle(TEvPrivate::TEvRequestDistribution::TPtr&) { } void TStatisticsAggregator::Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev) { + const auto round = ev->Get()->Record.GetRound(); + if (round == GlobalTraversalRound && AggregationRequestBeginTime) { + TInstant now = AppData(TlsActivationContext->AsActorContext())->TimeProvider->Now(); + TDuration time = now - AggregationRequestBeginTime; + TabletCounters->Simple()[COUNTER_AGGREGATION_TIME].Set(time.MicroSeconds()); + } + auto ack = std::make_unique(); - ack->Record.SetRound(ev->Get()->Record.GetRound()); + ack->Record.SetRound(round); Send(ev->Sender, ack.release()); Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++KeepAliveSeqNo)); } @@ -532,6 +550,9 @@ void TStatisticsAggregator::Resolve() { } void TStatisticsAggregator::ScanNextDatashardRange() { + // Datashard traversal is temporary disabled + Y_FAIL(); + if (DatashardRanges.empty()) { SaveStatisticsToTable(); return; @@ -577,8 +598,8 @@ void TStatisticsAggregator::SaveStatisticsToTable() { data.push_back(strSketch); } - Register(CreateSaveStatisticsQuery(TraversalPathId, EStatType::COUNT_MIN_SKETCH, - std::move(columnTags), std::move(data))); + Register(CreateSaveStatisticsQuery(SelfId(), + TraversalPathId, EStatType::COUNT_MIN_SKETCH, std::move(columnTags), std::move(data))); } void TStatisticsAggregator::DeleteStatisticsFromTable() { @@ -589,7 +610,7 @@ void TStatisticsAggregator::DeleteStatisticsFromTable() { PendingDeleteStatistics = false; - Register(CreateDeleteStatisticsQuery(TraversalPathId)); + Register(CreateDeleteStatisticsQuery(SelfId(), TraversalPathId)); } void TStatisticsAggregator::ScheduleNextAnalyze(NIceDb::TNiceDb& db) { @@ -639,25 +660,30 @@ void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { if (!LastTraversalWasForce) { LastTraversalWasForce = true; - for (TForceTraversalOperation& operation : ForceTraversals) { - for (TForceTraversalTable& operationTable : operation.Tables) { - if (operationTable.Status == TForceTraversalTable::EStatus::AnalyzeFinished) { - UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::TraversalStarted, operation.OperationId, operationTable, db); - pathId = operationTable.PathId; - break; + if (ForceTraversals.empty()) { + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. No force traversals."); + } else { + for (TForceTraversalOperation& operation : ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + if (operationTable.Status == TForceTraversalTable::EStatus::AnalyzeFinished) { + UpdateForceTraversalTableStatus(TForceTraversalTable::EStatus::TraversalStarted, operation.OperationId, operationTable, db); + pathId = operationTable.PathId; + break; + } + } + + if (!pathId) { + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. All the force traversal tables sent the requests. OperationId=" << operation.OperationId); + continue; } + + ForceTraversalOperationId = operation.OperationId; + break; } - + if (!pathId) { - SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. All the force traversal tables sent the requests. OperationId=" << operation.OperationId); - continue; + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. All the force traversal operations sent the requests."); } - - ForceTraversalOperationId = operation.OperationId; - } - - if (!pathId) { - SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. All the force traversal operations sent the requests."); } } @@ -680,6 +706,7 @@ void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { } TraversalPathId = pathId; + TraversalStartTime = TInstant::Now(); std::optional isColumnTable = IsColumnTable(pathId); if (!isColumnTable){ @@ -687,6 +714,13 @@ void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { return; } + // Datashard traversal is temporary disabled + if (!*isColumnTable) { + SA_LOG_D("[" << TabletID() << "] ScheduleNextTraversal. Skip traversal for datashard table " << pathId); + DeleteStatisticsFromTable(); + return; + } + TraversalIsColumnTable = *isColumnTable; SA_LOG_D("[" << TabletID() << "] Start " @@ -697,7 +731,6 @@ void TStatisticsAggregator::ScheduleNextTraversal(NIceDb::TNiceDb& db) { } void TStatisticsAggregator::StartTraversal(NIceDb::TNiceDb& db) { - TraversalStartTime = TInstant::Now(); PersistTraversal(db); TraversalStartKey = TSerializedCellVec(); @@ -779,6 +812,7 @@ void TStatisticsAggregator::DeleteForceTraversalOperation(const TString& operati } ForceTraversals.remove_if([operationId](const TForceTraversalOperation& elem) { return elem.OperationId == operationId;}); + TabletCounters->Simple()[COUNTER_FORCE_TRAVERSALS_INFLIGHT_SIZE].Set(ForceTraversals.size()); } TStatisticsAggregator::TForceTraversalTable* TStatisticsAggregator::ForceTraversalTable(const TString& operationId, const TPathId& pathId) { @@ -853,9 +887,31 @@ void TStatisticsAggregator::ResetTraversalState(NIceDb::TNiceDb& db) { TraversalRound = 0; } -template -void PrintContainerStart(const T& container, size_t count, TStringStream& str, - std::function extractor) +TString TStatisticsAggregator::GetNavigateTypeString() const { + switch (NavigateType) { + case Analyze: + return "Analyze"; + case Traversal: + return "Traversal"; + } +} + +TString TStatisticsAggregator::TForceTraversalTable::GetStatusString() const { + switch (Status) { + case EStatus::None: + return "None"; + case EStatus::AnalyzeStarted: + return "AnalyzeStarted"; + case EStatus::AnalyzeFinished: + return "AnalyzeFinished"; + case EStatus::TraversalStarted: + return "TraversalStarted"; + case EStatus::TraversalFinished: + return "TraversalFinished"; + } +} + +void PrintContainerStart(const auto& container, size_t count, TStringStream& str, auto extractor) { if (container.empty()) { return; @@ -892,49 +948,46 @@ bool TStatisticsAggregator::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev str << "BaseStatistics: " << BaseStatistics.size() << Endl; str << "SchemeShards: " << SchemeShards.size() << Endl; { - std::function&)> extr = - [](const auto& x) { return x.first; }; + auto extr = [](const auto& x) { return x.first; }; PrintContainerStart(SchemeShards, 4, str, extr); } str << "Nodes: " << Nodes.size() << Endl; { - std::function&)> extr = - [](const auto& x) { return x.first; }; + auto extr = [](const auto& x) { return x.first; }; PrintContainerStart(Nodes, 8, str, extr); } str << "RequestedSchemeShards: " << RequestedSchemeShards.size() << Endl; { - std::function extr = [](const auto& x) { return x; }; + auto extr = [](const auto& x) { return x; }; PrintContainerStart(RequestedSchemeShards, 4, str, extr); } str << "FastCounter: " << FastCounter << Endl; str << "FastCheckInFlight: " << FastCheckInFlight << Endl; str << "FastSchemeShards: " << FastSchemeShards.size() << Endl; { - std::function extr = [](const auto& x) { return x; }; + auto extr = [](const auto& x) { return x; }; PrintContainerStart(FastSchemeShards, 4, str, extr); } str << "FastNodes: " << FastNodes.size() << Endl; { - std::function extr = [](const auto& x) { return x; }; + auto extr = [](const auto& x) { return x; }; PrintContainerStart(FastNodes, 8, str, extr); } str << "PropagationInFlight: " << PropagationInFlight << Endl; str << "PropagationSchemeShards: " << PropagationSchemeShards.size() << Endl; { - std::function extr = [](const auto& x) { return x; }; + auto extr = [](const auto& x) { return x; }; PrintContainerStart(PropagationSchemeShards, 4, str, extr); } str << "PropagationNodes: " << PropagationNodes.size() << Endl; { - std::function extr = [](const auto& x) { return x; }; + auto extr = [](const auto& x) { return x; }; PrintContainerStart(FastNodes, 8, str, extr); } str << "LastSSIndex: " << LastSSIndex << Endl; str << "PendingRequests: " << PendingRequests.size() << Endl; str << "ProcessUrgentInFlight: " << ProcessUrgentInFlight << Endl << Endl; - str << "TraversalPathId: " << TraversalPathId << Endl; str << "Columns: " << Columns.size() << Endl; str << "DatashardRanges: " << DatashardRanges.size() << Endl; str << "CountMinSketches: " << CountMinSketches.size() << Endl << Endl; @@ -943,16 +996,54 @@ bool TStatisticsAggregator::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev if (!ScheduleTraversalsByTime.Empty()) { auto* oldestTable = ScheduleTraversalsByTime.Top(); str << " oldest table: " << oldestTable->PathId - << ", ordest table update time: " << oldestTable->LastUpdateTime << Endl; + << ", update time: " << oldestTable->LastUpdateTime.ToStringUpToSeconds() << Endl; } str << "ScheduleTraversalsBySchemeShard: " << ScheduleTraversalsBySchemeShard.size() << Endl; if (!ScheduleTraversalsBySchemeShard.empty()) { str << " " << ScheduleTraversalsBySchemeShard.begin()->first << Endl; - std::function extr = [](const auto& x) { return x; }; + auto extr = [](const auto& x) { return x; }; PrintContainerStart(ScheduleTraversalsBySchemeShard.begin()->second, 2, str, extr); } - str << "TraversalStartTime: " << TraversalStartTime << Endl; + str << "ForceTraversals: " << ForceTraversals.size() << Endl; + if (!ForceTraversals.empty()) { + auto extr = [](const auto& x) { return x.CreatedAt.ToStringUpToSeconds(); }; + PrintContainerStart(ForceTraversals, 2, str, extr); + } + + str << Endl; + str << "NavigateType: " << GetNavigateTypeString() << Endl; + str << "NavigateAnalyzeOperationId: " << NavigateAnalyzeOperationId << Endl; + str << "NavigatePathId: " << NavigatePathId << Endl; + + str << Endl; + str << "ForceTraversalOperationId: " << ForceTraversalOperationId << Endl; + if (ForceTraversalOperationId) { + auto forceTraversal = CurrentForceTraversalOperation(); + str << " CreatedAt: " << forceTraversal->CreatedAt << Endl; + str << ", ReplyToActorId: " << forceTraversal->ReplyToActorId << Endl; + str << ", Types: " << forceTraversal->Types << Endl; + str << ", Tables size: " << forceTraversal->Tables.size() << Endl; + str << ", Tables: " << Endl; + + for (size_t i = 0; i < forceTraversal->Tables.size(); ++i) { + const TForceTraversalTable& table = forceTraversal->Tables[i]; + str << " Table[" << i << "] PathId: " << table.PathId << Endl; + str << " Status: " << table.GetStatusString() << Endl; + str << " AnalyzedShards size: " << table.AnalyzedShards.size() << Endl; + str << " ColumnTags: " << table.ColumnTags << Endl; + } + } + + str << Endl; + str << "TraversalStartTime: " << TraversalStartTime.ToStringUpToSeconds() << Endl; + str << "TraversalPathId: " << TraversalPathId << Endl; + str << "TraversalIsColumnTable: " << TraversalIsColumnTable << Endl; + str << "TraversalStartKey: " << TraversalStartKey.GetBuffer() << Endl; + str << Endl; + str << "GlobalTraversalRound: " << GlobalTraversalRound << Endl; + str << "TraversalRound: " << TraversalRound << Endl; + str << "HiveRequestRound: " << HiveRequestRound << Endl; } } diff --git a/ydb/core/statistics/aggregator/aggregator_impl.h b/ydb/core/statistics/aggregator/aggregator_impl.h index 3c24c27462b9..1d75e08b6c01 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.h +++ b/ydb/core/statistics/aggregator/aggregator_impl.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -228,6 +229,11 @@ class TStatisticsAggregator : public TActor, public NTabl std::mt19937_64 RandomGenerator; + TTabletCountersBase* TabletCounters; + TAutoPtr TabletCountersPtr; + + TInstant AggregationRequestBeginTime; + bool EnableStatistics = false; bool EnableColumnStatistics = false; @@ -329,6 +335,8 @@ class TStatisticsAggregator : public TActor, public NTabl Traversal }; ENavigateType NavigateType = Analyze; + TString GetNavigateTypeString() const; + TString NavigateAnalyzeOperationId; TPathId NavigatePathId; @@ -380,6 +388,8 @@ class TStatisticsAggregator : public TActor, public NTabl TraversalFinished, }; EStatus Status = EStatus::None; + + TString GetStatusString() const; }; struct TForceTraversalOperation { TString OperationId; diff --git a/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp b/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp index 89f01dc55763..d6622dc623d2 100644 --- a/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp +++ b/ydb/core/statistics/aggregator/tx_aggr_stat_response.cpp @@ -28,6 +28,9 @@ struct TStatisticsAggregator::TTxAggregateStatisticsResponse : public TTxBase { ++Self->KeepAliveSeqNo; // cancel timeout events + Self->TabletCounters->Simple()[COUNTER_AGGREGATION_TIME].Set(0); + Self->AggregationRequestBeginTime = TInstant::Zero(); + NIceDb::TNiceDb db(txc.DB); for (auto& column : Record.GetColumns()) { @@ -122,6 +125,7 @@ struct TStatisticsAggregator::TTxAggregateStatisticsResponse : public TTxBase { case EAction::SendAggregate: ctx.Send(MakeStatServiceID(Self->SelfId().NodeId()), Request.release()); ctx.Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++Self->KeepAliveSeqNo)); + Self->AggregationRequestBeginTime = AppData(ctx)->TimeProvider->Now(); break; default: diff --git a/ydb/core/statistics/aggregator/tx_analyze.cpp b/ydb/core/statistics/aggregator/tx_analyze.cpp index 67868409ab21..58548d41d47e 100644 --- a/ydb/core/statistics/aggregator/tx_analyze.cpp +++ b/ydb/core/statistics/aggregator/tx_analyze.cpp @@ -82,6 +82,7 @@ struct TStatisticsAggregator::TTxAnalyze : public TTxBase { } Self->ForceTraversals.emplace_back(operation); + Self->TabletCounters->Simple()[COUNTER_FORCE_TRAVERSALS_INFLIGHT_SIZE].Set(Self->ForceTraversals.size()); db.Table().Key(operationId).Update( NIceDb::TUpdate(operationId), diff --git a/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp b/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp index 67730beadb57..1143faca4390 100644 --- a/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp +++ b/ydb/core/statistics/aggregator/tx_analyze_deadline.cpp @@ -46,6 +46,7 @@ struct TStatisticsAggregator::TTxAnalyzeDeadline : public TTxBase { "Send TEvAnalyzeResponse for deleted operation, OperationId=" << OperationId << ", ActorId=" << ReplyToActorId); auto response = std::make_unique(); response->Record.SetOperationId(OperationId); + response->Record.SetStatus(NKikimrStat::TEvAnalyzeResponse::STATUS_ERROR); ctx.Send(ReplyToActorId, response.release()); } else { SA_LOG_D("[" << Self->TabletID() << "] TTxAnalyzeDeadline::Complete. No ActorId to send reply. OperationId=" << OperationId); diff --git a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp index a48dd94c2170..cb94e4eac7a8 100644 --- a/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp +++ b/ydb/core/statistics/aggregator/tx_finish_trasersal.cpp @@ -50,6 +50,7 @@ struct TStatisticsAggregator::TTxFinishTraversal : public TTxBase { "Send TEvAnalyzeResponse, OperationId=" << OperationId << ", ActorId=" << ReplyToActorId); auto response = std::make_unique(); response->Record.SetOperationId(OperationId); + response->Record.SetStatus(NKikimrStat::TEvAnalyzeResponse::STATUS_SUCCESS); ctx.Send(ReplyToActorId, response.release()); } } diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index 55eaa4a13148..ae8a8a355e00 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -210,6 +210,8 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { } } + Self->TabletCounters->Simple()[COUNTER_FORCE_TRAVERSALS_INFLIGHT_SIZE].Set(Self->ForceTraversals.size()); + SA_LOG_D("[" << Self->TabletID() << "] Loaded ForceTraversalOperations: " << "table count# " << Self->ForceTraversals.size()); } @@ -234,6 +236,9 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { if (status == TForceTraversalTable::EStatus::AnalyzeStarted) { // Resent TEvAnalyzeTable to shards status = TForceTraversalTable::EStatus::None; + } else if (status == TForceTraversalTable::EStatus::TraversalStarted) { + // Reset traversal + status = TForceTraversalTable::EStatus::AnalyzeFinished; } auto pathId = TPathId(ownerId, localPathId); diff --git a/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp b/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp index abe80134ee3c..5faf80918c8f 100644 --- a/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp +++ b/ydb/core/statistics/aggregator/tx_response_tablet_distribution.cpp @@ -101,6 +101,7 @@ struct TStatisticsAggregator::TTxResponseTabletDistribution : public TTxBase { case EAction::SendAggregate: ctx.Send(MakeStatServiceID(Self->SelfId().NodeId()), AggregateStatisticsRequest.release()); ctx.Schedule(KeepAliveTimeout, new TEvPrivate::TEvAckTimeout(++Self->KeepAliveSeqNo)); + Self->AggregationRequestBeginTime = AppData(ctx)->TimeProvider->Now(); break; default: diff --git a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp index 6300fb751c8b..0d07c6339c3c 100644 --- a/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp +++ b/ydb/core/statistics/aggregator/tx_schedule_traversal.cpp @@ -11,12 +11,18 @@ struct TStatisticsAggregator::TTxScheduleTrasersal : public TTxBase { TTxType GetTxType() const override { return TXTYPE_SCHEDULE_TRAVERSAL; } - bool Execute(TTransactionContext& txc, const TActorContext&) override { + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { if (!Self->EnableColumnStatistics) { return true; } + TDuration time = TDuration ::Zero(); + if (!Self->ForceTraversals.empty()) { + time = ctx.Now() - Self->ForceTraversals.front().CreatedAt; + } + Self->TabletCounters->Simple()[COUNTER_FORCE_TRAVERSAL_INFLIGHT_MAX_TIME].Set(time.MicroSeconds()); + if (Self->TraversalPathId) { SA_LOG_T("[" << Self->TabletID() << "] TTxScheduleTrasersal::Execute. Traverse is in progress. PathId " << Self->TraversalPathId); return true; diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 4931963f4ab6..4367cdc300be 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -12,12 +12,18 @@ namespace NStat { Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { - Y_UNIT_TEST(AnalyzeOneColumnTable) { + Y_UNIT_TEST(AnalyzeTable) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; AnalyzeTable(runtime, tableInfo.ShardIds[0], tableInfo.PathId); + } + + Y_UNIT_TEST(Analyze) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; Analyze(runtime, tableInfo.SaTabletId, {tableInfo.PathId}); } @@ -43,9 +49,7 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto& runtime = *env.GetServer().GetRuntime(); auto sender = runtime.AllocateEdgeActor(); - auto schemeShardStatsBlocker = runtime.AddObserver([&](auto& ev) { - ev.Reset(); - }); + TBlockEvents block(runtime); auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; @@ -58,7 +62,19 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); - schemeShardStatsBlocker.Remove(); + // Check EvRemoteHttpInfo + { + auto httpRequest = std::make_unique("/app?"); + runtime.SendToPipe(tableInfo.SaTabletId, sender, httpRequest.release(), 0, {}); + auto httpResponse = runtime.GrabEdgeEventRethrow(sender); + TString body = httpResponse->Get()->Html; + Cerr << body << Endl; + UNIT_ASSERT(body.Size() > 500); + UNIT_ASSERT(body.Contains("ForceTraversals: 1")); + } + + block.Unblock(); + block.Stop(); auto analyzeResonse = runtime.GrabEdgeEventRethrow(sender); UNIT_ASSERT_VALUES_EQUAL(analyzeResonse->Get()->Record.GetOperationId(), operationId); @@ -73,12 +89,21 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { auto sender = runtime.AllocateEdgeActor(); const TString operationId = "operationId"; + TBlockEvents block(runtime); + + auto tabletPipe = runtime.ConnectToPipe(tableInfo.SaTabletId, sender, 0, {}); + auto analyzeRequest1 = MakeAnalyzeRequest({tableInfo.PathId}, operationId); - runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest1.release()); + runtime.SendToPipe(tabletPipe, sender, analyzeRequest1.release()); + + runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return block.size(); }); auto analyzeRequest2 = MakeAnalyzeRequest({tableInfo.PathId}, operationId); - runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest2.release()); + runtime.SendToPipe(tabletPipe, sender, analyzeRequest2.release()); + block.Unblock(); + block.Stop(); + auto response1 = runtime.GrabEdgeEventRethrow(sender); UNIT_ASSERT(response1); UNIT_ASSERT_VALUES_EQUAL(response1->Get()->Record.GetOperationId(), operationId); @@ -87,6 +112,37 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { UNIT_ASSERT(!response2); } + Y_UNIT_TEST(AnalyzeMultiOperationId) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + auto GetOperationId = [] (size_t i) { return TStringBuilder() << "operationId" << i; }; + + TBlockEvents block(runtime); + + const size_t numEvents = 10; + + auto tabletPipe = runtime.ConnectToPipe(tableInfo.SaTabletId, sender, 0, {}); + + for (size_t i = 0; i < numEvents; ++i) { + auto analyzeRequest = MakeAnalyzeRequest({tableInfo.PathId}, GetOperationId(i)); + runtime.SendToPipe(tabletPipe, sender, analyzeRequest.release()); + } + + runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return block.size() == numEvents; }); + + block.Unblock(); + block.Stop(); + + for (size_t i = 0; i < numEvents; ++i) { + auto response = runtime.GrabEdgeEventRethrow(sender); + UNIT_ASSERT(response); + UNIT_ASSERT_VALUES_EQUAL(response->Get()->Record.GetOperationId(), GetOperationId(i)); + } + } + Y_UNIT_TEST(AnalyzeRebootSaBeforeAnalyzeTableResponse) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); @@ -273,7 +329,10 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return block.size(); }); runtime.AdvanceCurrentTime(TDuration::Days(2)); - runtime.GrabEdgeEventRethrow(sender); + auto analyzeResponse = runtime.GrabEdgeEventRethrow(sender); + const auto& record = analyzeResponse->Get()->Record; + UNIT_ASSERT_VALUES_EQUAL(record.GetOperationId(), "operationId"); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrStat::TEvAnalyzeResponse::STATUS_ERROR); } } diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp index 766777482291..4c1bf7a7d4e8 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_datashard.cpp @@ -38,7 +38,7 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { Analyze(runtime, saTabletId, {{pathId}}); - ValidateCountMinDatashard(runtime, pathId); + ValidateCountMinDatashardAbsense(runtime, pathId); } Y_UNIT_TEST(AnalyzeTwoTables) { @@ -64,8 +64,8 @@ Y_UNIT_TEST_SUITE(AnalyzeDatashard) { Analyze(runtime, saTabletId1, {pathId1, pathId2}); - ValidateCountMinDatashard(runtime, pathId1); - ValidateCountMinDatashard(runtime, pathId2); + ValidateCountMinDatashardAbsense(runtime, pathId1); + ValidateCountMinDatashardAbsense(runtime, pathId2); } diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp index 3c110788fd93..efd8c8015f85 100644 --- a/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_columnshard.cpp @@ -36,13 +36,28 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { auto& runtime = *env.GetServer().GetRuntime(); auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; - runtime.SimulateSleep(TDuration::Seconds(30)); + WaitForSavedStatistics(runtime, tableInfo.PathId); auto countMin = ExtractCountMin(runtime, tableInfo.PathId); UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); } + Y_UNIT_TEST(TraverseColumnTableRebootColumnshard) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 10)[0]; + auto sender = runtime.AllocateEdgeActor(); + + WaitForSavedStatistics(runtime, tableInfo.PathId); + + RebootTablet(runtime, tableInfo.ShardIds[0], sender); + + auto countMin = ExtractCountMin(runtime, tableInfo.PathId); + + UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); + } + Y_UNIT_TEST(TraverseColumnTableRebootSaTabletBeforeResolve) { TTestEnv env(1, 1); auto& runtime = *env.GetServer().GetRuntime(); @@ -62,7 +77,7 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { block.Unblock(); block.Stop(); - runtime.SimulateSleep(TDuration::Seconds(10)); + WaitForSavedStatistics(runtime, tableInfo.PathId); auto countMin = ExtractCountMin(runtime, tableInfo.PathId); UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); @@ -188,7 +203,7 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { } }); - runtime.SimulateSleep(TDuration::Seconds(30)); + WaitForSavedStatistics(runtime, tableInfo.PathId); auto countMin = ExtractCountMin(runtime, tableInfo.PathId); UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); @@ -225,7 +240,7 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { } }); - runtime.SimulateSleep(TDuration::Seconds(30)); + WaitForSavedStatistics(runtime, tableInfo.PathId); auto countMin = ExtractCountMin(runtime, tableInfo.PathId); UNIT_ASSERT(CheckCountMinSketch(countMin, ColumnTableRowsNumber)); @@ -257,7 +272,7 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { } }); - runtime.SimulateSleep(TDuration::Seconds(30)); + WaitForSavedStatistics(runtime, tableInfo.PathId); auto countMin = ExtractCountMin(runtime, tableInfo.PathId); @@ -294,7 +309,7 @@ Y_UNIT_TEST_SUITE(TraverseColumnShard) { } }); - runtime.SimulateSleep(TDuration::Seconds(60)); + WaitForSavedStatistics(runtime, tableInfo.PathId); auto countMin = ExtractCountMin(runtime, tableInfo.PathId); diff --git a/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp b/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp index cab2ef533c13..37bba99f4716 100644 --- a/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_traverse_datashard.cpp @@ -34,7 +34,7 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { runtime.SimulateSleep(TDuration::Seconds(60)); auto pathId = ResolvePathId(runtime, "/Root/Database/Table"); - ValidateCountMinDatashard(runtime, pathId); + ValidateCountMinDatashardAbsense(runtime, pathId); } Y_UNIT_TEST(TraverseTwoTables) { @@ -54,8 +54,8 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Database/Table1"); auto pathId2 = ResolvePathId(runtime, "/Root/Database/Table2"); - ValidateCountMinDatashard(runtime, pathId1); - ValidateCountMinDatashard(runtime, pathId2); + ValidateCountMinDatashardAbsense(runtime, pathId1); + ValidateCountMinDatashardAbsense(runtime, pathId2); } Y_UNIT_TEST(TraverseOneTableServerless) { @@ -85,7 +85,7 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { runtime.SimulateSleep(TDuration::Seconds(60)); auto pathId = ResolvePathId(runtime, "/Root/Serverless/Table"); - ValidateCountMinDatashard(runtime, pathId); + ValidateCountMinDatashardAbsense(runtime, pathId); } Y_UNIT_TEST(TraverseTwoTablesServerless) { @@ -117,8 +117,8 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Serverless/Table1"); auto pathId2 = ResolvePathId(runtime, "/Root/Serverless/Table2"); - ValidateCountMinDatashard(runtime, pathId1); - ValidateCountMinDatashard(runtime, pathId2); + ValidateCountMinDatashardAbsense(runtime, pathId1); + ValidateCountMinDatashardAbsense(runtime, pathId2); } Y_UNIT_TEST(TraverseTwoTablesTwoServerlessDbs) { @@ -151,8 +151,8 @@ Y_UNIT_TEST_SUITE(TraverseDatashard) { auto pathId1 = ResolvePathId(runtime, "/Root/Serverless1/Table1"); auto pathId2 = ResolvePathId(runtime, "/Root/Serverless2/Table2"); - ValidateCountMinDatashard(runtime, pathId1); - ValidateCountMinDatashard(runtime, pathId2); + ValidateCountMinDatashardAbsense(runtime, pathId1); + ValidateCountMinDatashardAbsense(runtime, pathId2); } } diff --git a/ydb/core/statistics/database/database.cpp b/ydb/core/statistics/database/database.cpp index 8b216f7da9d5..0a80147e24f1 100644 --- a/ydb/core/statistics/database/database.cpp +++ b/ydb/core/statistics/database/database.cpp @@ -81,14 +81,15 @@ class TSaveStatisticsQuery : public NKikimr::TQueryBase { const ui64 StatType; const std::vector ColumnTags; const std::vector Data; + public: TSaveStatisticsQuery(const TPathId& pathId, ui64 statType, - std::vector&& columnTags, std::vector&& data) + const std::vector& columnTags, const std::vector& data) : NKikimr::TQueryBase(NKikimrServices::STATISTICS, {}, {}, true) , PathId(pathId) , StatType(statType) - , ColumnTags(std::move(columnTags)) - , Data(std::move(data)) + , ColumnTags(columnTags) + , Data(data) { Y_ABORT_UNLESS(ColumnTags.size() == Data.size()); } @@ -148,15 +149,62 @@ class TSaveStatisticsQuery : public NKikimr::TQueryBase { void OnFinish(Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { Y_UNUSED(issues); auto response = std::make_unique(); + response->Status = status; + response->Issues = std::move(issues); response->Success = (status == Ydb::StatusIds::SUCCESS); + response->PathId = PathId; Send(Owner, response.release()); } }; -NActors::IActor* CreateSaveStatisticsQuery(const TPathId& pathId, ui64 statType, - std::vector&& columnTags, std::vector&& data) +class TSaveStatisticsRetryingQuery : public TActorBootstrapped { +private: + const NActors::TActorId ReplyActorId; + const TPathId PathId; + const ui64 StatType; + const std::vector ColumnTags; + const std::vector Data; + +public: + using TSaveRetryingQuery = TQueryRetryActor< + TSaveStatisticsQuery, TEvStatistics::TEvSaveStatisticsQueryResponse, + const TPathId&, ui64, const std::vector&, const std::vector&>; + + TSaveStatisticsRetryingQuery(const NActors::TActorId& replyActorId, + const TPathId& pathId, ui64 statType, std::vector&& columnTags, std::vector&& data) + : ReplyActorId(replyActorId) + , PathId(pathId) + , StatType(statType) + , ColumnTags(std::move(columnTags)) + , Data(std::move(data)) + {} + + void Bootstrap() { + Register(new TSaveRetryingQuery( + SelfId(), + TSaveRetryingQuery::IRetryPolicy::GetExponentialBackoffPolicy( + TSaveRetryingQuery::Retryable, TDuration::MilliSeconds(10), + TDuration::MilliSeconds(200), TDuration::Seconds(1), + std::numeric_limits::max(), TDuration::Seconds(1)), + PathId, StatType, ColumnTags, Data + )); + Become(&TSaveStatisticsRetryingQuery::StateFunc); + } + + STRICT_STFUNC(StateFunc, + hFunc(TEvStatistics::TEvSaveStatisticsQueryResponse, Handle); + ) + + void Handle(TEvStatistics::TEvSaveStatisticsQueryResponse::TPtr& ev) { + Send(ReplyActorId, ev->Release().Release()); + PassAway(); + } +}; + +NActors::IActor* CreateSaveStatisticsQuery(const NActors::TActorId& replyActorId, + const TPathId& pathId, ui64 statType, std::vector&& columnTags, std::vector&& data) { - return new TSaveStatisticsQuery(pathId, statType, std::move(columnTags), std::move(data)); + return new TSaveStatisticsRetryingQuery(replyActorId, pathId, statType, std::move(columnTags), std::move(data)); } @@ -231,6 +279,8 @@ class TLoadStatisticsQuery : public NKikimr::TQueryBase { void OnFinish(Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { Y_UNUSED(issues); auto response = std::make_unique(); + response->Status = status; + response->Issues = std::move(issues); response->Success = (status == Ydb::StatusIds::SUCCESS); response->Cookie = Cookie; if (response->Success) { @@ -240,10 +290,54 @@ class TLoadStatisticsQuery : public NKikimr::TQueryBase { } }; -NActors::IActor* CreateLoadStatisticsQuery(const TPathId& pathId, ui64 statType, - ui32 columnTag, ui64 cookie) +class TLoadStatisticsRetryingQuery : public TActorBootstrapped { +private: + const NActors::TActorId ReplyActorId; + const TPathId PathId; + const ui64 StatType; + const ui32 ColumnTag; + const ui64 Cookie; + +public: + using TLoadRetryingQuery = TQueryRetryActor< + TLoadStatisticsQuery, TEvStatistics::TEvLoadStatisticsQueryResponse, + const TPathId&, ui64, ui32, ui64>; + + TLoadStatisticsRetryingQuery(const NActors::TActorId& replyActorId, + const TPathId& pathId, ui64 statType, ui32 columnTag, ui64 cookie) + : ReplyActorId(replyActorId) + , PathId(pathId) + , StatType(statType) + , ColumnTag(columnTag) + , Cookie(cookie) + {} + + void Bootstrap() { + Register(new TLoadRetryingQuery( + SelfId(), + TLoadRetryingQuery::IRetryPolicy::GetExponentialBackoffPolicy( + TLoadRetryingQuery::Retryable, TDuration::MilliSeconds(10), + TDuration::MilliSeconds(200), TDuration::Seconds(1), + std::numeric_limits::max(), TDuration::Seconds(1)), + PathId, StatType, ColumnTag, Cookie + )); + Become(&TLoadStatisticsRetryingQuery::StateFunc); + } + + STRICT_STFUNC(StateFunc, + hFunc(TEvStatistics::TEvLoadStatisticsQueryResponse, Handle); + ) + + void Handle(TEvStatistics::TEvLoadStatisticsQueryResponse::TPtr& ev) { + Send(ReplyActorId, ev->Release().Release()); + PassAway(); + } +}; + +NActors::IActor* CreateLoadStatisticsQuery(const NActors::TActorId& replyActorId, + const TPathId& pathId, ui64 statType, ui32 columnTag, ui64 cookie) { - return new TLoadStatisticsQuery(pathId, statType, columnTag, cookie); + return new TLoadStatisticsRetryingQuery(replyActorId, pathId, statType, columnTag, cookie); } @@ -288,14 +382,53 @@ class TDeleteStatisticsQuery : public NKikimr::TQueryBase { void OnFinish(Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { Y_UNUSED(issues); auto response = std::make_unique(); + response->Status = status; + response->Issues = std::move(issues); response->Success = (status == Ydb::StatusIds::SUCCESS); Send(Owner, response.release()); } }; -NActors::IActor* CreateDeleteStatisticsQuery(const TPathId& pathId) +class TDeleteStatisticsRetryingQuery : public TActorBootstrapped { +private: + const NActors::TActorId ReplyActorId; + const TPathId PathId; + +public: + using TDeleteRetryingQuery = TQueryRetryActor< + TDeleteStatisticsQuery, TEvStatistics::TEvDeleteStatisticsQueryResponse, + const TPathId&>; + + TDeleteStatisticsRetryingQuery(const NActors::TActorId& replyActorId, const TPathId& pathId) + : ReplyActorId(replyActorId) + , PathId(pathId) + {} + + void Bootstrap() { + Register(new TDeleteRetryingQuery( + SelfId(), + TDeleteRetryingQuery::IRetryPolicy::GetExponentialBackoffPolicy( + TDeleteRetryingQuery::Retryable, TDuration::MilliSeconds(10), + TDuration::MilliSeconds(200), TDuration::Seconds(1), + std::numeric_limits::max(), TDuration::Seconds(1)), + PathId + )); + Become(&TDeleteStatisticsRetryingQuery::StateFunc); + } + + STRICT_STFUNC(StateFunc, + hFunc(TEvStatistics::TEvDeleteStatisticsQueryResponse, Handle); + ) + + void Handle(TEvStatistics::TEvDeleteStatisticsQueryResponse::TPtr& ev) { + Send(ReplyActorId, ev->Release().Release()); + PassAway(); + } +}; + +NActors::IActor* CreateDeleteStatisticsQuery(const NActors::TActorId& replyActorId, const TPathId& pathId) { - return new TDeleteStatisticsQuery(pathId); + return new TDeleteStatisticsRetryingQuery(replyActorId, pathId); } } // NKikimr::NStat diff --git a/ydb/core/statistics/database/database.h b/ydb/core/statistics/database/database.h index cb81909c3d54..4aef1c33cd29 100644 --- a/ydb/core/statistics/database/database.h +++ b/ydb/core/statistics/database/database.h @@ -7,12 +7,12 @@ namespace NKikimr::NStat { NActors::IActor* CreateStatisticsTableCreator(std::unique_ptr event); -NActors::IActor* CreateSaveStatisticsQuery(const TPathId& pathId, ui64 statType, - std::vector&& columnTags, std::vector&& data); +NActors::IActor* CreateSaveStatisticsQuery(const NActors::TActorId& replyActorId, + const TPathId& pathId, ui64 statType, std::vector&& columnTags, std::vector&& data); -NActors::IActor* CreateLoadStatisticsQuery(const TPathId& pathId, ui64 statType, - ui32 columnTag, ui64 cookie); +NActors::IActor* CreateLoadStatisticsQuery(const NActors::TActorId& replyActorId, + const TPathId& pathId, ui64 statType, ui32 columnTag, ui64 cookie); -NActors::IActor* CreateDeleteStatisticsQuery(const TPathId& pathId); +NActors::IActor* CreateDeleteStatisticsQuery(const NActors::TActorId& replyActorId, const TPathId& pathId); }; diff --git a/ydb/core/statistics/database/ut/ut_database.cpp b/ydb/core/statistics/database/ut/ut_database.cpp index 50b19d056a91..ff4c47a862ce 100644 --- a/ydb/core/statistics/database/ut/ut_database.cpp +++ b/ydb/core/statistics/database/ut/ut_database.cpp @@ -31,20 +31,20 @@ Y_UNIT_TEST_SUITE(StatisticsSaveLoad) { std::vector columnTags = {1, 2}; std::vector data = {"dataA", "dataB"}; - runtime.Register(CreateSaveStatisticsQuery( + runtime.Register(CreateSaveStatisticsQuery(sender, pathId, statType, std::move(columnTags), std::move(data)), 0, 0, TMailboxType::Simple, 0, sender); auto saveResponse = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(saveResponse->Get()->Success); - runtime.Register(CreateLoadStatisticsQuery(pathId, statType, 1, 1), + runtime.Register(CreateLoadStatisticsQuery(sender, pathId, statType, 1, 1), 0, 0, TMailboxType::Simple, 0, sender); auto loadResponseA = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(loadResponseA->Get()->Success); UNIT_ASSERT(loadResponseA->Get()->Data); UNIT_ASSERT_VALUES_EQUAL(*loadResponseA->Get()->Data, "dataA"); - runtime.Register(CreateLoadStatisticsQuery(pathId, statType, 2, 1), + runtime.Register(CreateLoadStatisticsQuery(sender, pathId, statType, 2, 1), 0, 0, TMailboxType::Simple, 0, sender); auto loadResponseB = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(loadResponseB->Get()->Success); @@ -73,18 +73,18 @@ Y_UNIT_TEST_SUITE(StatisticsSaveLoad) { std::vector columnTags = {1, 2}; std::vector data = {"dataA", "dataB"}; - runtime.Register(CreateSaveStatisticsQuery( + runtime.Register(CreateSaveStatisticsQuery(sender, pathId, statType, std::move(columnTags), std::move(data)), 0, 0, TMailboxType::Simple, 0, sender); auto saveResponse = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(saveResponse->Get()->Success); - runtime.Register(CreateDeleteStatisticsQuery(pathId), + runtime.Register(CreateDeleteStatisticsQuery(sender, pathId), 0, 0, TMailboxType::Simple, 0, sender); auto deleteResponse = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(deleteResponse->Get()->Success); - runtime.Register(CreateLoadStatisticsQuery(pathId, statType, 1, 1), + runtime.Register(CreateLoadStatisticsQuery(sender, pathId, statType, 1, 1), 0, 0, TMailboxType::Simple, 0, sender); auto loadResponseA = runtime.GrabEdgeEvent(sender); UNIT_ASSERT(!loadResponseA->Get()->Success); diff --git a/ydb/core/statistics/events.h b/ydb/core/statistics/events.h index ca23149140fa..97154c2d67c1 100644 --- a/ydb/core/statistics/events.h +++ b/ydb/core/statistics/events.h @@ -3,8 +3,11 @@ #include #include #include +#include #include #include +#include + namespace NKikimr { namespace NStat { @@ -187,6 +190,9 @@ struct TEvStatistics { TEvSaveStatisticsQueryResponse, EvSaveStatisticsQueryResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + TPathId PathId; bool Success = true; }; @@ -194,6 +200,8 @@ struct TEvStatistics { TEvLoadStatisticsQueryResponse, EvLoadStatisticsQueryResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; bool Success = true; ui64 Cookie = 0; std::optional Data; @@ -203,6 +211,8 @@ struct TEvStatistics { TEvDeleteStatisticsQueryResponse, EvDeleteStatisticsQueryResponse> { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; bool Success = true; }; diff --git a/ydb/core/statistics/service/service.cpp b/ydb/core/statistics/service/service.cpp index d828bc550b07..2b59af317b23 100644 --- a/ydb/core/statistics/service/service.cpp +++ b/ydb/core/statistics/service/service.cpp @@ -28,6 +28,7 @@ namespace NStat { static constexpr TDuration DefaultAggregateKeepAlivePeriod = TDuration::MilliSeconds(500); static constexpr TDuration DefaultAggregateKeepAliveTimeout = TDuration::Seconds(3); static constexpr TDuration DefaultAggregateKeepAliveAckTimeout = TDuration::Seconds(3); +static constexpr TDuration DefaultStatisticsRequestTimeout = TDuration::Seconds(5); static constexpr size_t DefaultMaxInFlightTabletRequests = 5; static constexpr size_t DefaultFanOutFactor = 5; @@ -37,6 +38,7 @@ TStatServiceSettings::TStatServiceSettings() : AggregateKeepAlivePeriod(DefaultAggregateKeepAlivePeriod) , AggregateKeepAliveTimeout(DefaultAggregateKeepAliveTimeout) , AggregateKeepAliveAckTimeout(DefaultAggregateKeepAliveAckTimeout) + , StatisticsRequestTimeout(DefaultStatisticsRequestTimeout) , MaxInFlightTabletRequests(DefaultMaxInFlightTabletRequests) , FanOutFactor(DefaultFanOutFactor) {} diff --git a/ydb/core/statistics/service/service.h b/ydb/core/statistics/service/service.h index abe356fe0271..4b29bc8bcbef 100644 --- a/ydb/core/statistics/service/service.h +++ b/ydb/core/statistics/service/service.h @@ -9,6 +9,7 @@ struct TStatServiceSettings { TDuration AggregateKeepAlivePeriod; TDuration AggregateKeepAliveTimeout; TDuration AggregateKeepAliveAckTimeout; + TDuration StatisticsRequestTimeout; size_t MaxInFlightTabletRequests; size_t FanOutFactor; @@ -29,6 +30,11 @@ struct TStatServiceSettings { return *this; } + TStatServiceSettings& SetStatisticsRequestTimeout(const TDuration& val) { + StatisticsRequestTimeout = val; + return *this; + } + TStatServiceSettings& SetMaxInFlightTabletRequests(size_t val) { MaxInFlightTabletRequests = val; return *this; diff --git a/ydb/core/statistics/service/service_impl.cpp b/ydb/core/statistics/service/service_impl.cpp index 3ddac1412c62..ac553927cbe8 100644 --- a/ydb/core/statistics/service/service_impl.cpp +++ b/ydb/core/statistics/service/service_impl.cpp @@ -25,7 +25,6 @@ namespace NKikimr { namespace NStat { - struct TAggregationStatistics { using TColumnsStatistics = ::google::protobuf::RepeatedPtrField<::NKikimrStat::TColumnStatistics>; @@ -128,6 +127,7 @@ class TStatService : public TActorBootstrapped { EvDispatchKeepAlive, EvKeepAliveTimeout, EvKeepAliveAckTimeout, + EvStatisticsRequestTimeout, EvEnd }; @@ -155,6 +155,13 @@ class TStatService : public TActorBootstrapped { ui64 Round; ui32 NodeId; }; + + struct TEvStatisticsRequestTimeout: public NActors::TEventLocal { + TEvStatisticsRequestTimeout(ui64 round, ui64 tabletId): Round(round), TabletId(tabletId) {} + + ui64 Round; + ui64 TabletId; + }; }; void Bootstrap() { @@ -195,6 +202,7 @@ class TStatService : public TActorBootstrapped { hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); hFunc(TEvPrivate::TEvDispatchKeepAlive, Handle); hFunc(TEvPrivate::TEvKeepAliveTimeout, Handle); + hFunc(TEvPrivate::TEvStatisticsRequestTimeout, Handle); hFunc(TEvStatistics::TEvStatisticsResponse, Handle); hFunc(TEvStatistics::TEvAggregateStatisticsResponse, Handle); @@ -670,10 +678,15 @@ class TStatService : public TActorBootstrapped { for (const auto& req : request.StatRequests) { auto& response = request.StatResponses.emplace_back(); response.Req = req; + if (!req.ColumnTag) { + response.Success = false; + ++reqIndex; + continue; + } ui64 loadCookie = NextLoadQueryCookie++; LoadQueriesInFlight[loadCookie] = std::make_pair(requestId, reqIndex); - Register(CreateLoadStatisticsQuery(req.PathId, request.StatType, - *req.ColumnTag, loadCookie)); + Register(CreateLoadStatisticsQuery(SelfId(), + req.PathId, request.StatType, *req.ColumnTag, loadCookie)); ++request.ReplyCounter; ++reqIndex; } @@ -902,7 +915,31 @@ class TStatService : public TActorBootstrapped { } } - void SendStatisticsRequest(const TActorId& clientId) { + void Handle(TEvPrivate::TEvStatisticsRequestTimeout::TPtr& ev) { + const auto round = ev->Get()->Round; + if (IsNotCurrentRound(round)) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Skip TEvStatisticsRequestTimeout"); + return; + } + + const auto tabletId = ev->Get()->TabletId; + auto tabletPipe = AggregationStatistics.LocalTablets.TabletsPipes.find(tabletId); + if (tabletPipe == AggregationStatistics.LocalTablets.TabletsPipes.end()) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "Tablet " << tabletId << " has already been processed"); + return; + } + + LOG_ERROR_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "No result was received from the tablet " << tabletId); + + auto clientId = tabletPipe->second; + OnTabletError(tabletId); + NTabletPipe::CloseClient(SelfId(), clientId); + } + + void SendStatisticsRequest(const TActorId& clientId, ui64 tabletId) { auto request = std::make_unique(); auto& record = request->Record; record.MutableTypes()->Add(NKikimrStat::TYPE_COUNT_MIN_SKETCH); @@ -916,7 +953,9 @@ class TStatService : public TActorBootstrapped { columnTags->Add(tag); } - NTabletPipe::SendData(SelfId(), clientId, request.release(), AggregationStatistics.Round); + const auto round = AggregationStatistics.Round; + NTabletPipe::SendData(SelfId(), clientId, request.release(), round); + Schedule(Settings.StatisticsRequestTimeout, new TEvPrivate::TEvStatisticsRequestTimeout(round, tabletId)); LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "TEvStatisticsRequest send" @@ -928,7 +967,7 @@ class TStatService : public TActorBootstrapped { LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, "Tablet " << tabletId << " is not local."); - constexpr auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; + const auto error = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; AggregationStatistics.FailedTablets.emplace_back(tabletId, 0, error); AggregationStatistics.LocalTablets.TabletsPipes.erase(tabletId); @@ -966,7 +1005,7 @@ class TStatService : public TActorBootstrapped { if (tabletPipe != tabletsPipes.end() && clientId == tabletPipe->second) { if (ev->Get()->Status == NKikimrProto::OK) { - SendStatisticsRequest(clientId); + SendStatisticsRequest(clientId, tabletId); } else { OnTabletError(tabletId); } diff --git a/ydb/core/statistics/service/ut/ut_service.cpp b/ydb/core/statistics/service/ut/ut_service.cpp index a885c1b06838..8425d12a4a9e 100644 --- a/ydb/core/statistics/service/ut/ut_service.cpp +++ b/ydb/core/statistics/service/ut/ut_service.cpp @@ -532,6 +532,67 @@ Y_UNIT_TEST_SUITE(StatisticsService) { UNIT_ASSERT_VALUES_EQUAL(expectedError, actualError); } } + + Y_UNIT_TEST(ShouldBeCcorrectProcessingTabletTimeout) { + size_t nodeCount = 1; + auto runtime = TTestActorRuntime(nodeCount, 1, false); + auto settings = GetDefaultSettings() + .SetStatisticsRequestTimeout(TDuration::MilliSeconds(10)); + auto indexToActorMap = InitializeRuntime(runtime, nodeCount, settings); + auto nodeIdToIndexMap = GetNodeIdToIndexMap(indexToActorMap); + std::vector localTabletsIds = {1, 2, 3, 4, 5, 6, 7}; + std::vector nodesTablets = {{.NodeId = indexToActorMap[0].NodeId(), .Ids{localTabletsIds}}}; + + std::unordered_map pipeToTablet; + std::vector observers; + observers.emplace_back(runtime.AddObserver([&](TEvTabletResolver::TEvForward::TPtr& ev) { + auto tabletId = ev->Get()->TabletID; + auto recipient = indexToActorMap[nodeIdToIndexMap[ev->Sender.NodeId()]]; + pipeToTablet[ev->Sender] = tabletId; + + runtime.Send(new IEventHandle(recipient, ev->Sender, + new TEvTabletPipe::TEvClientConnected(tabletId, NKikimrProto::OK, ev->Sender, ev->Sender, + true, false, 0), 0, ev->Cookie), nodeIdToIndexMap[ev->Sender.NodeId()], true); + ev.Reset(); + })); + observers.emplace_back(runtime.AddObserver([&](TAutoPtr& ev) { + switch (ev->GetTypeRewrite()) { + case TEvTabletPipe::EvSend: + auto msg = ev->Get(); + if (msg != nullptr) { + auto tabletId = pipeToTablet[ev->Recipient]; + if (tabletId % 2 != 0) { + auto senderNodeIndex = nodeIdToIndexMap[ev->Sender.NodeId()]; + runtime.Send(new IEventHandle(ev->Sender, ev->Sender, + CreateStatisticsResponse(TStatisticsResponse{ + .TabletId = tabletId, + .Status = NKikimrStat::TEvStatisticsResponse::STATUS_SUCCESS + }).release(), 0, ev->Cookie), senderNodeIndex, true); + } + } + break; + } + })); + + auto sender = runtime.AllocateEdgeActor(); + runtime.Send(indexToActorMap[0], sender, CreateStatisticsRequest(TAggregateStatisticsRequest{ + .Round = 1, + .PathId{3, 3}, + .Nodes{ nodesTablets }, + .ColumnTags{1} + }).release()); + + auto res = runtime.GrabEdgeEvent(sender); + const auto& record = res->Get()->Record; + size_t expectedFailedTabletsCount = 3; + UNIT_ASSERT_VALUES_EQUAL(expectedFailedTabletsCount, record.GetFailedTablets().size()); + + ui32 expectedError = NKikimrStat::TEvAggregateStatisticsResponse::TYPE_NON_LOCAL_TABLET; + for (const auto& fail : record.GetFailedTablets()) { + ui32 actualError = fail.GetError(); + UNIT_ASSERT_VALUES_EQUAL(expectedError, actualError); + } + } } } // NSysView diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index efdc2eb37b92..04ac02d7ae8e 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -408,7 +408,9 @@ void Analyze(TTestActorRuntime& runtime, ui64 saTabletId, const std::vector(sender); - UNIT_ASSERT_VALUES_EQUAL(evResponse->Get()->Record.GetOperationId(), operationId); + const auto& record = evResponse->Get()->Record; + UNIT_ASSERT_VALUES_EQUAL(record.GetOperationId(), operationId); + UNIT_ASSERT_VALUES_EQUAL(record.GetStatus(), NKikimrStat::TEvAnalyzeResponse::STATUS_SUCCESS); } void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyzedTable& table) { @@ -433,6 +435,17 @@ void AnalyzeStatus(TTestActorRuntime& runtime, TActorId sender, ui64 saTabletId, UNIT_ASSERT_VALUES_EQUAL(analyzeStatusResponse->Get()->Record.GetStatus(), expectedStatus); } +void WaitForSavedStatistics(TTestActorRuntime& runtime, const TPathId& pathId) { + bool eventSeen = false; + auto observer = runtime.AddObserver([&](auto& ev){ + if (ev->Get()->PathId == pathId) + eventSeen = true; + }); + + runtime.WaitFor("TEvSaveStatisticsQueryResponse", [&]{ return eventSeen; }); +} + + } // NStat } // NKikimr diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index 1caf652aa43d..426175572f73 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -114,6 +114,7 @@ void Analyze(TTestActorRuntime& runtime, ui64 saTabletId, const std::vector Date: Thu, 29 Aug 2024 16:09:21 +0000 Subject: [PATCH 09/17] merge fixes --- ydb/core/statistics/aggregator/tx_ack_timeout.cpp | 2 +- ydb/core/statistics/ut_common/ut_common.cpp | 6 +++--- ydb/core/statistics/ut_common/ut_common.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ydb/core/statistics/aggregator/tx_ack_timeout.cpp b/ydb/core/statistics/aggregator/tx_ack_timeout.cpp index 6afa1bd86afe..d83ce5b10d57 100644 --- a/ydb/core/statistics/aggregator/tx_ack_timeout.cpp +++ b/ydb/core/statistics/aggregator/tx_ack_timeout.cpp @@ -9,7 +9,7 @@ struct TStatisticsAggregator::TTxAckTimeout : public TTxBase { TTxType GetTxType() const override { return TXTYPE_ACK_TIMEOUT; } - bool Execute(TTransactionContext& txc, const TActorContext&) override { + bool Execute(TTransactionContext& /*txc*/, const TActorContext&) override { SA_LOG_D("[" << Self->TabletID() << "] TTxAckTimeout::Execute"); return true; } diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index 04ac02d7ae8e..b53ea9407d73 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -52,7 +52,7 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool Settings->SetDomainName("Root"); Settings->SetNodeCount(staticNodes); Settings->SetDynamicNodeCount(dynamicNodes); - Settings->SetUseRealThreads(false); + Settings->SetUseRealThreads(useRealThreads); NKikimrConfig::TFeatureFlags featureFlags; featureFlags.SetEnableStatistics(true); @@ -309,12 +309,12 @@ void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableN UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } -std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, const TPathId& pathId, ui64 columnTag) { +std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPathId pathId, ui64 columnTag) { auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); NStat::TRequest req; req.PathId = pathId; - req.ColumnTag = 1; + req.ColumnTag = columnTag; auto evGet = std::make_unique(); evGet->StatType = NStat::EStatType::COUNT_MIN_SKETCH; diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index 426175572f73..d31dee1a4e1a 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -24,7 +24,7 @@ NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings( class TTestEnv { public: - TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 1, ui32 storagePools = 1); + TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 1, ui32 storagePools = 1, bool useRealThreads = false); ~TTestEnv(); Tests::TServer& GetServer() const { From 664ee1366f557528b8919fe30f2aeaba25a75a7b Mon Sep 17 00:00:00 2001 From: pilik Date: Mon, 5 Aug 2024 21:00:32 +0300 Subject: [PATCH 10/17] SQL command ANALYZE (#6996) --- .../executer_actor/kqp_scheme_executer.cpp | 24 ++ ydb/core/kqp/gateway/actors/analyze_actor.cpp | 227 ++++++++++++++++++ ydb/core/kqp/gateway/actors/analyze_actor.h | 66 +++++ ydb/core/kqp/gateway/actors/ya.make | 2 + ydb/core/kqp/gateway/kqp_ic_gateway.cpp | 17 ++ ydb/core/kqp/host/kqp_gateway_proxy.cpp | 33 +++ ydb/core/kqp/provider/yql_kikimr_datasink.cpp | 35 +++ ydb/core/kqp/provider/yql_kikimr_exec.cpp | 27 +++ .../kqp/provider/yql_kikimr_expr_nodes.json | 11 + ydb/core/kqp/provider/yql_kikimr_gateway.h | 7 + ydb/core/kqp/provider/yql_kikimr_provider.cpp | 1 + ydb/core/kqp/provider/yql_kikimr_provider.h | 1 + .../kqp/provider/yql_kikimr_provider_impl.h | 2 + ydb/core/kqp/provider/yql_kikimr_type_ann.cpp | 10 + ydb/core/kqp/ut/query/kqp_analyze_ut.cpp | 108 +++++++++ ydb/core/kqp/ut/query/ya.make | 2 + ydb/core/protos/kqp_physical.proto | 6 + ydb/core/statistics/ut_common/ut_common.cpp | 3 + ydb/library/yql/sql/v1/SQLv1.g.in | 5 + ydb/library/yql/sql/v1/format/sql_format.cpp | 9 +- .../yql/sql/v1/format/sql_format_ut.cpp | 14 ++ ydb/library/yql/sql/v1/node.h | 6 + ydb/library/yql/sql/v1/query.cpp | 55 +++++ ydb/library/yql/sql/v1/source.h | 1 + ydb/library/yql/sql/v1/sql.cpp | 1 + ydb/library/yql/sql/v1/sql_query.cpp | 33 +++ ydb/library/yql/sql/v1/sql_ut.cpp | 4 +- 27 files changed, 707 insertions(+), 3 deletions(-) create mode 100644 ydb/core/kqp/gateway/actors/analyze_actor.cpp create mode 100644 ydb/core/kqp/gateway/actors/analyze_actor.h create mode 100644 ydb/core/kqp/ut/query/kqp_analyze_ut.cpp diff --git a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp index 69acc8a2e6c2..f61cb1116634 100644 --- a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp @@ -2,6 +2,7 @@ #include "kqp_executer_impl.h" #include +#include #include #include #include @@ -307,6 +308,29 @@ class TKqpSchemeExecuter : public TActorBootstrapped { break; } + case NKqpProto::TKqpSchemeOperation::kAnalyzeTable: { + const auto& analyzeOperation = schemeOp.GetAnalyzeTable(); + + auto analyzePromise = NewPromise(); + + TVector columns{analyzeOperation.columns().begin(), analyzeOperation.columns().end()}; + IActor* analyzeActor = new TAnalyzeActor(analyzeOperation.GetTablePath(), columns, analyzePromise); + + auto actorSystem = TlsActivationContext->AsActorContext().ExecutorThread.ActorSystem; + RegisterWithSameMailbox(analyzeActor); + + auto selfId = SelfId(); + analyzePromise.GetFuture().Subscribe([actorSystem, selfId](const TFuture& future) { + auto ev = MakeHolder(); + ev->Result = future.GetValue(); + + actorSystem->Send(selfId, ev.Release()); + }); + + Become(&TKqpSchemeExecuter::ExecuteState); + return; + } + default: InternalError(TStringBuilder() << "Unexpected scheme operation: " << (ui32) schemeOp.GetOperationCase()); diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.cpp b/ydb/core/kqp/gateway/actors/analyze_actor.cpp new file mode 100644 index 000000000000..15b115697739 --- /dev/null +++ b/ydb/core/kqp/gateway/actors/analyze_actor.cpp @@ -0,0 +1,227 @@ +#include "analyze_actor.h" + +#include +#include +#include +#include + + +namespace NKikimr::NKqp { + +enum { + FirstRoundCookie = 0, + SecondRoundCookie = 1, +}; + +using TNavigate = NSchemeCache::TSchemeCacheNavigate; + +void TAnalyzeActor::Bootstrap() { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto navigate = std::make_unique(); + auto& entry = navigate->ResultSet.emplace_back(); + entry.Path = SplitPath(TablePath); + entry.Operation = TNavigate::EOp::OpTable; + entry.RequestType = TNavigate::TEntry::ERequestType::ByPath; + navigate->Cookie = FirstRoundCookie; + + Send(NKikimr::MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + + Become(&TAnalyzeActor::StateWork); +} + +void TAnalyzeActor::SendAnalyzeStatus() { + Y_ABORT_UNLESS(StatisticsAggregatorId.has_value()); + + auto getStatus = std::make_unique(); + auto& record = getStatus->Record; + PathIdFromPathId(PathId, record.MutablePathId()); + + Send( + MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(getStatus.release(), StatisticsAggregatorId.value(), true) + ); +} + +void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + Y_UNUSED(ctx); + + SendAnalyzeStatus(); +} + +void TAnalyzeActor::Handle(TEvAnalyzePrivate::TEvAnalyzeStatusCheck::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev); + Y_UNUSED(ctx); + + SendAnalyzeStatus(); +} + +void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + switch (record.GetStatus()) { + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_UNSPECIFIED: { + Promise.SetValue( + NYql::NCommon::ResultFromError( + YqlIssue( + {}, NYql::TIssuesIds::UNEXPECTED, + TStringBuilder() << "Statistics Aggregator unspecified error" + ) + ) + ); + this->Die(ctx); + return; + } + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION: { + NYql::IKikimrGateway::TGenericResult result; + result.SetSuccess(); + Promise.SetValue(std::move(result)); + + this->Die(ctx); + return; + } + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED: { + Schedule(TDuration::Seconds(10), new TEvAnalyzePrivate::TEvAnalyzeStatusCheck()); + return; + } + case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS: { + Schedule(TDuration::Seconds(5), new TEvAnalyzePrivate::TEvAnalyzeStatusCheck()); + return; + } + } +} + +void TAnalyzeActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { + std::unique_ptr navigate(ev->Get()->Request.Release()); + Y_ABORT_UNLESS(navigate->ResultSet.size() == 1); + auto& entry = navigate->ResultSet.front(); + + if (entry.Status != TNavigate::EStatus::Ok) { + NYql::EYqlIssueCode error; + switch (entry.Status) { + case TNavigate::EStatus::PathErrorUnknown: + case TNavigate::EStatus::RootUnknown: + case TNavigate::EStatus::PathNotTable: + case TNavigate::EStatus::TableCreationNotComplete: + error = NYql::TIssuesIds::KIKIMR_SCHEME_ERROR; + case TNavigate::EStatus::LookupError: + case TNavigate::EStatus::RedirectLookupError: + error = NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE; + default: + error = NYql::TIssuesIds::DEFAULT_ERROR; + } + Promise.SetValue( + NYql::NCommon::ResultFromIssues( + error, + TStringBuilder() << "Can't get statistics aggregator ID. " << entry.Status, + {} + ) + ); + this->Die(ctx); + return; + } + + if (navigate->Cookie == SecondRoundCookie) { + if (entry.DomainInfo->Params.HasStatisticsAggregator()) { + SendStatisticsAggregatorAnalyze(entry, ctx); + } else { + Promise.SetValue( + NYql::NCommon::ResultFromIssues( + NYql::TIssuesIds::DEFAULT_ERROR, + TStringBuilder() << "Can't get statistics aggregator ID.", {} + ) + ); + } + + this->Die(ctx); + return; + } + + PathId = entry.TableId.PathId; + + auto& domainInfo = entry.DomainInfo; + + auto navigateDomainKey = [this] (TPathId domainKey) { + using TNavigate = NSchemeCache::TSchemeCacheNavigate; + auto navigate = std::make_unique(); + auto& entry = navigate->ResultSet.emplace_back(); + entry.TableId = TTableId(domainKey.OwnerId, domainKey.LocalPathId); + entry.Operation = TNavigate::EOp::OpPath; + entry.RequestType = TNavigate::TEntry::ERequestType::ByTableId; + entry.RedirectRequired = false; + navigate->Cookie = SecondRoundCookie; + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release())); + }; + + if (!domainInfo->IsServerless()) { + if (domainInfo->Params.HasStatisticsAggregator()) { + SendStatisticsAggregatorAnalyze(entry, ctx); + return; + } + + navigateDomainKey(domainInfo->DomainKey); + } else { + navigateDomainKey(domainInfo->ResourcesDomainKey); + } +} + +void TAnalyzeActor::SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TActorContext& ctx) { + Y_ABORT_UNLESS(entry.DomainInfo->Params.HasStatisticsAggregator()); + + StatisticsAggregatorId = entry.DomainInfo->Params.GetStatisticsAggregator(); + + auto analyzeRequest = std::make_unique(); + auto& record = analyzeRequest->Record; + auto table = record.AddTables(); + + PathIdFromPathId(PathId, table->MutablePathId()); + + + THashMap tagByColumnName; + for (const auto& [_, tableInfo]: entry.Columns) { + tagByColumnName[TString(tableInfo.Name)] = tableInfo.Id; + } + + for (const auto& columnName: Columns) { + if (!tagByColumnName.contains(columnName)){ + Promise.SetValue( + NYql::NCommon::ResultFromError( + YqlIssue( + {}, NYql::TIssuesIds::UNEXPECTED, + TStringBuilder() << "No such column: " << columnName << " in the " << TablePath + ) + ) + ); + this->Die(ctx); + return; + } + + *table->MutableColumnTags()->Add() = tagByColumnName[columnName]; + } + + Send( + MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(analyzeRequest.release(), entry.DomainInfo->Params.GetStatisticsAggregator(), true), + IEventHandle::FlagTrackDelivery + ); +} + +void TAnalyzeActor::HandleUnexpectedEvent(ui32 typeRewrite) { + ALOG_CRIT( + NKikimrServices::KQP_GATEWAY, + "TAnalyzeActor, unexpected event, request type: " << typeRewrite; + ); + + Promise.SetValue( + NYql::NCommon::ResultFromError( + YqlIssue( + {}, NYql::TIssuesIds::UNEXPECTED, + TStringBuilder() << "Unexpected event: " << typeRewrite + ) + ) + ); + + this->PassAway(); +} + +}// end of NKikimr::NKqp diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.h b/ydb/core/kqp/gateway/actors/analyze_actor.h new file mode 100644 index 000000000000..af0b83cb1ce6 --- /dev/null +++ b/ydb/core/kqp/gateway/actors/analyze_actor.h @@ -0,0 +1,66 @@ +#include +#include +#include +#include + +#include + + +namespace NKikimr::NKqp { + + +struct TEvAnalyzePrivate { + enum EEv { + EvAnalyzeStatusCheck = EventSpaceBegin(TEvents::ES_PRIVATE), + EvEnd + }; + + struct TEvAnalyzeStatusCheck : public TEventLocal {}; +}; + +class TAnalyzeActor : public NActors::TActorBootstrapped { +public: + TAnalyzeActor(TString tablePath, TVector columns, NThreading::TPromise promise) + : TablePath(tablePath) + , Columns(columns) + , Promise(promise) + {} + + void Bootstrap(); + + STFUNC(StateWork) { + switch(ev->GetTypeRewrite()) { + HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + HFunc(NStat::TEvStatistics::TEvAnalyzeResponse, Handle); + HFunc(NStat::TEvStatistics::TEvAnalyzeStatusResponse, Handle); + HFunc(TEvAnalyzePrivate::TEvAnalyzeStatusCheck, Handle); + default: + HandleUnexpectedEvent(ev->GetTypeRewrite()); + } + } + +private: + void Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx); + + void Handle(NStat::TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev, const TActorContext& ctx); + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); + + void Handle(TEvAnalyzePrivate::TEvAnalyzeStatusCheck::TPtr& ev, const TActorContext& ctx); + + void HandleUnexpectedEvent(ui32 typeRewrite); + + void SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeCacheNavigate::TEntry&, const TActorContext&); + + void SendAnalyzeStatus(); + +private: + TString TablePath; + TVector Columns; + NThreading::TPromise Promise; + // For Statistics Aggregator + std::optional StatisticsAggregatorId; + TPathId PathId; +}; + +} // end of NKikimr::NKqp diff --git a/ydb/core/kqp/gateway/actors/ya.make b/ydb/core/kqp/gateway/actors/ya.make index 69553c63d3f0..2b426471d0d4 100644 --- a/ydb/core/kqp/gateway/actors/ya.make +++ b/ydb/core/kqp/gateway/actors/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( scheme.cpp + analyze_actor.cpp ) PEERDIR( @@ -11,6 +12,7 @@ PEERDIR( ydb/library/yql/providers/common/gateway ydb/core/tx/schemeshard ydb/library/actors/core + ydb/library/services ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index e5d6bbfcf461..4990244c7a94 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -1,5 +1,6 @@ #include "kqp_gateway.h" #include "actors/kqp_ic_gateway_actors.h" +#include "actors/analyze_actor.h" #include "actors/scheme.h" #include "kqp_metadata_loader.h" #include "local_rpc/helper.h" @@ -1386,6 +1387,22 @@ class TKikimrIcGateway : public IKqpGateway { } } + TFuture Analyze(const TString& cluster, const NYql::TAnalyzeSettings& settings) override { + try { + if (!CheckCluster(cluster)) { + return InvalidCluster(cluster); + } + + auto analyzePromise = NewPromise(); + IActor* analyzeActor = new TAnalyzeActor(settings.TablePath, settings.Columns, analyzePromise); + RegisterActor(analyzeActor); + + return analyzePromise.GetFuture(); + } catch (yexception& e) { + return MakeFuture(ResultFromException(e)); + } + } + template class IObjectModifier { public: diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index e5a84a6b7a08..9287933c8580 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -2056,6 +2056,39 @@ class TKqpGatewayProxy : public IKikimrGateway { } } + TFuture Analyze(const TString& cluster, const NYql::TAnalyzeSettings& settings) override { + CHECK_PREPARED_DDL(Analyze); + + try { + if (cluster != SessionCtx->GetCluster()) { + return MakeFuture(ResultFromError("Invalid cluster: " + cluster)); + } + + NKqpProto::TKqpAnalyzeOperation analyzeTx; + analyzeTx.SetTablePath(settings.TablePath); + for (const auto& column: settings.Columns) { + *analyzeTx.AddColumns() = column; + } + + if (IsPrepare()) { + auto& phyQuery = *SessionCtx->Query().PreparingQuery->MutablePhysicalQuery(); + auto& phyTx = *phyQuery.AddTransactions(); + phyTx.SetType(NKqpProto::TKqpPhyTx::TYPE_SCHEME); + + phyTx.MutableSchemeOperation()->MutableAnalyzeTable()->Swap(&analyzeTx); + + TGenericResult result; + result.SetSuccess(); + return MakeFuture(result); + } else { + return Gateway->Analyze(cluster, settings); + } + } + catch (yexception& e) { + return MakeFuture(ResultFromException(e)); + } + } + TVector GetCollectedSchemeData() override { return Gateway->GetCollectedSchemeData(); } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index a4d88aff12d2..2ce3e49e146a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -79,6 +79,16 @@ class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer { return TStatus::Ok; } + TStatus HandleAnalyze(TKiAnalyzeTable node, TExprContext& ctx) override { + Y_UNUSED(ctx); + + auto cluster = node.DataSink().Cluster(); + auto table = node.Table(); + + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + return TStatus::Ok; + } + TStatus HandleCreateTopic(TKiCreateTopic node, TExprContext& ctx) override { Y_UNUSED(ctx); Y_UNUSED(node); @@ -275,6 +285,9 @@ class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer { } SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; + } else if (mode == "analyze") { + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + return TStatus::Ok; } else { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Unsupported Kikimr table write mode: " << settings.Mode.Cast().Value())); @@ -549,6 +562,10 @@ class TKikimrDataSink : public TDataProviderBase return true; } + if (node.IsCallable(TKiAnalyzeTable::CallableName())) { + return true; + } + if (auto maybeRight = TMaybeNode(&node).Tuple().Maybe()) { if (maybeRight.Input().Maybe()) { return true; @@ -997,6 +1014,20 @@ class TKikimrDataSink : public TDataProviderBase .Done() .Ptr(); } + } else if (mode == "analyze") { + auto columns = Build(ctx, node->Pos()); + + for (const auto& column: settings.Columns.Cast().Ptr()->Children()) { + columns.Add(column); + } + + return Build(ctx, node->Pos()) + .World(node->Child(0)) + .DataSink(node->Child(1)) + .Table().Build(key.GetTablePath()) + .Columns(columns.Done()) + .Done() + .Ptr(); } else { return Build(ctx, node->Pos()) .World(node->Child(0)) @@ -1571,6 +1602,10 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt return HandleAlterSequence(node.Cast(), ctx); } + if (auto node = callable.Maybe()) { + return HandleAnalyze(node.Cast(), ctx); + } + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSink) Unsupported function: " << callable.CallableName())); return TStatus::Error; diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 8c1d07ed998a..717a4e23568f 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -280,6 +280,18 @@ namespace { }; } + TAnalyzeSettings ParseAnalyzeSettings(const TKiAnalyzeTable& analyze) { + TVector columns; + for (const auto& column: analyze.Columns()) { + columns.push_back(TString(column.Ptr()->Content())); + } + + return TAnalyzeSettings{ + .TablePath = TString(analyze.Table()), + .Columns = std::move(columns) + }; + } + TAlterColumnTableSettings ParseAlterColumnTableSettings(TKiAlterTable alter) { return TAlterColumnTableSettings{ .Table = TString(alter.Table()) @@ -2281,6 +2293,21 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer(input)) { + auto cluster = TString(maybeAnalyze.Cast().DataSink().Cluster()); + + TAnalyzeSettings analyzeSettings = ParseAnalyzeSettings(maybeAnalyze.Cast()); + + auto future = Gateway->Analyze(cluster, analyzeSettings); + + return WrapFuture(future, + [](const IKikimrGateway::TGenericResult& res, const TExprNode::TPtr& input, TExprContext& ctx) { + Y_UNUSED(res); + auto resultNode = ctx.NewWorld(input->Pos()); + return resultNode; + }, "Executing ANALYZE"); + } + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "(Kikimr DataSink) Failed to execute node: " << input->Content())); return SyncError(); diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json index 4d8f0946cacb..79450c4b92eb 100644 --- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json +++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json @@ -493,6 +493,17 @@ {"Index": 2, "Name": "Replication", "Type": "TCoAtom"}, {"Index": 3, "Name": "Cascade", "Type": "TCoAtom"} ] + }, + { + "Name": "TKiAnalyzeTable", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KiAnalyze!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"}, + {"Index": 2, "Name": "Table", "Type": "TCoAtom"}, + {"Index": 3, "Name": "Columns", "Type": "TCoAtomList"} + ] } ] } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index a470e9b960b7..41de17474747 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -776,6 +776,11 @@ struct TDropReplicationSettings { bool Cascade = false; }; +struct TAnalyzeSettings { + TString TablePath; + TVector Columns; +}; + struct TKikimrListPathItem { TKikimrListPathItem(TString name, bool isDirectory) { Name = name; @@ -1002,6 +1007,8 @@ class IKikimrGateway : public TThrRefBase { virtual NThreading::TFuture DropExternalTable(const TString& cluster, const TDropExternalTableSettings& settings, bool missingOk) = 0; + virtual NThreading::TFuture Analyze(const TString& cluster, const TAnalyzeSettings& settings) = 0; + virtual TVector GetCollectedSchemeData() = 0; virtual NThreading::TFuture ExecuteLiteral(const TString& program, const NKikimrMiniKQL::TType& resultType, NKikimr::NKqp::TTxAllocatorState::TPtr txAlloc) = 0; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index ac310064077c..bfeba1dd97cb 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -73,6 +73,7 @@ struct TKikimrData { DataSinkNames.insert(TKiCreateSequence::CallableName()); DataSinkNames.insert(TKiDropSequence::CallableName()); DataSinkNames.insert(TKiAlterSequence::CallableName()); + DataSinkNames.insert(TKiAnalyzeTable::CallableName()); CommitModes.insert(CommitModeFlush); CommitModes.insert(CommitModeRollback); diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index 95c31375eccf..98422b9de94a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -240,6 +240,7 @@ enum class TYdbOperation : ui32 { CreateReplication = 1 << 24, AlterReplication = 1 << 25, DropReplication = 1 << 26, + Analyze = 1 << 27, }; Y_DECLARE_FLAGS(TYdbOperations, TYdbOperation); diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index f267f3f5a86e..a7d95b99f129 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -74,6 +74,8 @@ class TKiSinkVisitorTransformer : public TSyncTransformerBase { virtual TStatus HandleModifyPermissions(NNodes::TKiModifyPermissions node, TExprContext& ctx) = 0; virtual TStatus HandleReturningList(NNodes::TKiReturningList node, TExprContext& ctx) = 0; + + virtual TStatus HandleAnalyze(NNodes::TKiAnalyzeTable node, TExprContext& ctx) = 0; }; class TKikimrKey { diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index a5746e0be2c0..dc2cb9179a80 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -2020,6 +2020,16 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over return false; } + virtual TStatus HandleAnalyze(NNodes::TKiAnalyzeTable node, TExprContext& ctx) override { + auto table = SessionCtx->Tables().EnsureTableExists(TString(node.DataSink().Cluster()), TString(node.Table().Value()), node.Pos(), ctx); + if (!table) { + return TStatus::Error; + } + + node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn()); + return TStatus::Ok; + } + private: TIntrusivePtr Gateway; TIntrusivePtr SessionCtx; diff --git a/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp b/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp new file mode 100644 index 000000000000..b53cef9ada50 --- /dev/null +++ b/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include +#include + +#include + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NTable; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpAnalyze) { + +void CreateTable(NStat::TTestEnv& env, const TString& databaseName, const TString& tableName) { + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto fullTableName = Sprintf("Root/%s/%s", databaseName.c_str(), tableName.c_str()); + auto result = session.ExecuteSchemeQuery(Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key) + ) + )", fullTableName.c_str())).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + TValueBuilder rows; + rows.BeginList(); + for (size_t i = 0; i < 1000; ++i) { + auto key = TValueBuilder().Uint64(i).Build(); + auto value = TValueBuilder().OptionalString("Hello, world!").Build(); + + rows.AddListItem(); + rows.BeginStruct(); + rows.AddMember("Key", key); + rows.AddMember("Value", value); + rows.EndStruct(); + } + rows.EndList(); + + result = client.BulkUpsert(fullTableName, rows.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); +} + +using namespace NStat; + +Y_UNIT_TEST(AnalyzeDatashardTable) { + TTestEnv env(1, 1, 1, true); + CreateDatabase(env, "Database"); + + TTableClient client(env.GetDriver()); + auto session = client.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteSchemeQuery( + Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key) + ) + )", "Root/Database/Table") + ).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + TValueBuilder rows; + rows.BeginList(); + for (size_t i = 0; i < 1500; ++i) { + auto key = TValueBuilder().Uint64(i).Build(); + auto value = TValueBuilder().OptionalString("Hello,world!").Build(); + + rows.AddListItem(); + rows.BeginStruct(); + rows.AddMember("Key", key); + rows.AddMember("Value", value); + rows.EndStruct(); + } + rows.EndList(); + + result = client.BulkUpsert("Root/Database/Table", rows.Build()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + result = session.ExecuteSchemeQuery( + Sprintf(R"(ANALYZE `Root/%s/%s`)", "Database", "Table") + ).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + auto& runtime = *env.GetServer().GetRuntime(); + ui64 saTabletId; + auto pathId = ResolvePathId(runtime, "/Root/Database/Table", nullptr, &saTabletId); + + auto countMin = ExtractCountMin(runtime, pathId, 2); + TString value = "Hello,world!"; + auto stat = countMin->Probe(value.Data(), value.Size()); + UNIT_ASSERT_C(stat >= 1500, ToString(stat)); +} + + +} // suite + + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/query/ya.make b/ydb/core/kqp/ut/query/ya.make index 2d9745cba212..91c714510791 100644 --- a/ydb/core/kqp/ut/query/ya.make +++ b/ydb/core/kqp/ut/query/ya.make @@ -17,6 +17,7 @@ ELSE() ENDIF() SRCS( + kqp_analyze_ut.cpp kqp_explain_ut.cpp kqp_limits_ut.cpp kqp_params_ut.cpp @@ -26,6 +27,7 @@ SRCS( ) PEERDIR( + ydb/core/statistics/ut_common ydb/public/sdk/cpp/client/ydb_proto ydb/core/kqp ydb/core/kqp/ut/common diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto index 0ec497cca01f..d4c8e625912e 100644 --- a/ydb/core/protos/kqp_physical.proto +++ b/ydb/core/protos/kqp_physical.proto @@ -385,6 +385,11 @@ message TKqpPhyResult { optional uint32 QueryResultIndex = 5; } +message TKqpAnalyzeOperation { + string TablePath = 1; + repeated string Columns = 2; +} + message TKqpPhyMetadataOperation { message TColumnValue { string Column = 1; @@ -449,6 +454,7 @@ message TKqpSchemeOperation { NKikimrSchemeOp.TModifyScheme CreateResourcePool = 38; NKikimrSchemeOp.TModifyScheme AlterResourcePool = 39; NKikimrSchemeOp.TModifyScheme DropResourcePool = 40; + TKqpAnalyzeOperation AnalyzeTable = 41; } } diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index b53ea9407d73..8ff5063832e8 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -83,6 +83,9 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool CSController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); Server->GetRuntime()->SetLogPriority(NKikimrServices::STATISTICS, NActors::NLog::PRI_DEBUG); + Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG); + // Server->GetRuntime()->SetLogPriority(NKikimrServices::K, NActors::NLog::PRI_DEBUG); + } TTestEnv::~TTestEnv() { diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 9957d7144a59..9983e403ea80 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -66,6 +66,7 @@ sql_stmt_core: | create_resource_pool_stmt | alter_resource_pool_stmt | drop_resource_pool_stmt + | analyze_stmt ; expr: @@ -981,6 +982,10 @@ commit_stmt: COMMIT; rollback_stmt: ROLLBACK; +analyze_table: simple_table_ref (LPAREN column_list RPAREN)?; +analyze_table_list: analyze_table (COMMA analyze_table)* COMMA?; +analyze_stmt: ANALYZE analyze_table_list; + // Special rules that allow to use certain keywords as identifiers. identifier: ID_PLAIN | ID_QUOTED; id: identifier | keyword; diff --git a/ydb/library/yql/sql/v1/format/sql_format.cpp b/ydb/library/yql/sql/v1/format/sql_format.cpp index 085b8fe818df..e5e569dcef7d 100644 --- a/ydb/library/yql/sql/v1/format/sql_format.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format.cpp @@ -937,6 +937,12 @@ friend struct TStaticData; VisitAllFields(TRule_drop_table_stmt::GetDescriptor(), msg); } + void VisitAnalyze(const TRule_analyze_stmt& msg) { + PosFromToken(msg.GetToken1()); + NewLine(); + VisitAllFields(TRule_analyze_stmt::GetDescriptor(), msg); + } + void VisitUse(const TRule_use_stmt& msg) { PosFromToken(msg.GetToken1()); NewLine(); @@ -2743,7 +2749,8 @@ TStaticData::TStaticData() {TRule_drop_view_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitDropView)}, {TRule_create_resource_pool_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCreateResourcePool)}, {TRule_alter_resource_pool_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAlterResourcePool)}, - {TRule_drop_resource_pool_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitDropResourcePool)} + {TRule_drop_resource_pool_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitDropResourcePool)}, + {TRule_analyze_stmt::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitAnalyze)} }) , ObfuscatingVisitDispatch({ {TToken::GetDescriptor(), MakeObfuscatingFunctor(&TObfuscatingVisitor::VisitToken)}, diff --git a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp index cca94f066896..12236c09e017 100644 --- a/ydb/library/yql/sql/v1/format/sql_format_ut.cpp +++ b/ydb/library/yql/sql/v1/format/sql_format_ut.cpp @@ -1599,4 +1599,18 @@ FROM Input MATCH_RECOGNIZE (PATTERN (A) DEFINE A AS A); TSetup setup; setup.Run(cases); } + + + + Y_UNIT_TEST(Analyze) { + TCases cases = { + {"analyze table (col1, col2, col3)", + "ANALYZE table (col1, col2, col3);\n"}, + {"analyze table", + "ANALYZE table;\n"} + }; + + TSetup setup; + setup.Run(cases); + } } diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index 66a92cd38fee..9f8e696ac25e 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -1203,6 +1203,12 @@ namespace NSQLTranslationV1 { bool Temporary = false; }; + struct TTableRef; + struct TAnalyzeParams { + std::shared_ptr Table; + TVector Columns; + }; + struct TAlterTableParameters { TVector AddColumns; TVector DropColumns; diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index e13803eefe27..93c29415db10 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -3141,4 +3141,59 @@ class TWorldFor final : public TAstListNode { TNodePtr BuildWorldForNode(TPosition pos, TNodePtr list, TNodePtr bodyNode, TNodePtr elseNode, bool isEvaluate, bool isParallel) { return new TWorldFor(pos, list, bodyNode, elseNode, isEvaluate, isParallel); } + +class TAnalyzeNode final: public TAstListNode { +public: + TAnalyzeNode(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TAnalyzeParams& params, TScopedStatePtr scoped) + : TAstListNode(pos) + , Service(service) + , Cluster(cluster) + , Params(params) + , Scoped(scoped) + { + FakeSource = BuildFakeSource(pos); + scoped->UseCluster(Service, Cluster); + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_UNUSED(src); + auto keys = Params.Table->Keys->GetTableKeys()->BuildKeys(ctx, ITableKeys::EBuildKeysMode::DROP); + if (!keys || !keys->Init(ctx, FakeSource.Get())) { + return false; + } + + auto opts = Y(); + + auto columns = Y(); + for (const auto& column: Params.Columns) { + columns->Add(Q(column)); + } + opts->Add(Q(Y(Q("columns"), Q(columns)))); + + opts->Add(Q(Y(Q("mode"), Q("analyze")))); + Add("block", Q(Y( + Y("let", "sink", Y("DataSink", BuildQuotedAtom(Pos, Service), Scoped->WrapCluster(Cluster, ctx))), + Y("let", "world", Y(TString(WriteName), "world", "sink", keys, Y("Void"), Q(opts))), + Y("return", ctx.PragmaAutoCommit ? Y(TString(CommitName), "world", "sink") : AstNode("world")) + ))); + + return TAstListNode::DoInit(ctx, FakeSource.Get()); + } + + TPtr DoClone() const final { + return {}; + } +private: + TString Service; + TDeferredAtom Cluster; + TAnalyzeParams Params; + + TScopedStatePtr Scoped; + TSourcePtr FakeSource; +}; + +TNodePtr BuildAnalyze(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TAnalyzeParams& params, TScopedStatePtr scoped) { + return new TAnalyzeNode(pos, service, cluster, params, scoped); +} + } // namespace NSQLTranslationV1 diff --git a/ydb/library/yql/sql/v1/source.h b/ydb/library/yql/sql/v1/source.h index a58402f1dc19..3c41453ebb48 100644 --- a/ydb/library/yql/sql/v1/source.h +++ b/ydb/library/yql/sql/v1/source.h @@ -310,6 +310,7 @@ namespace NSQLTranslationV1 { TNodePtr BuildDropTable(TPosition pos, const TTableRef& table, bool missingOk, ETableType tableType, TScopedStatePtr scoped); TNodePtr BuildWriteTable(TPosition pos, const TString& label, const TTableRef& table, EWriteColumnMode mode, TNodePtr options, TScopedStatePtr scoped); + TNodePtr BuildAnalyze(TPosition pos, const TString& service, const TDeferredAtom& cluster, const TAnalyzeParams& params, TScopedStatePtr scoped); TSourcePtr TryMakeSourceFromExpression(TPosition pos, TContext& ctx, const TString& currService, const TDeferredAtom& currCluster, TNodePtr node, const TString& view = {}); void MakeTableFromExpression(TPosition pos, TContext& ctx, TNodePtr node, TDeferredAtom& table, const TString& prefix = {}); diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index e50c776269e5..f2bdd926ec79 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -167,6 +167,7 @@ bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { case TRule_sql_stmt_core::kAltSqlStmtCore45: // create resource pool case TRule_sql_stmt_core::kAltSqlStmtCore46: // alter resource pool case TRule_sql_stmt_core::kAltSqlStmtCore47: // drop resource pool + case TRule_sql_stmt_core::kAltSqlStmtCore51: // analyze return false; } } diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index 18a916e71fe7..08ce8a20c80a 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -1344,6 +1344,39 @@ bool TSqlQuery::Statement(TVector& blocks, const TRule_sql_stmt_core& AddStatementToBlocks(blocks, BuildDropObjectOperation(Ctx.Pos(), objectId, "RESOURCE_POOL", false, {}, context)); break; } + case TRule_sql_stmt_core::kAltSqlStmtCore51: { + // analyze_stmt: ANALYZE table_ref + Ctx.BodyPart(); + const auto& rule = core.GetAlt_sql_stmt_core51().GetRule_analyze_stmt1(); + + if (!rule.GetRule_analyze_table_list2().GetBlock2().empty()) { + Error() << "ANALYZE with multitables hasn't been implemented yet"; + return false; + } + auto analyzeTable = rule.GetRule_analyze_table_list2().GetRule_analyze_table1(); + + TVector columns; + if (analyzeTable.HasBlock2()) { + auto columnsNode = + analyzeTable.GetBlock2().GetRule_column_list2(); + + if (columnsNode.HasRule_column_name1()) { + columns.push_back(Id(columnsNode.GetRule_column_name1().GetRule_an_id2(), *this)); + for (const auto& columnNode: columnsNode.GetBlock2()) { + columns.push_back(Id(columnNode.GetRule_column_name2().GetRule_an_id2(), *this)); + } + } + } + + TTableRef tr; + if (!SimpleTableRefImpl(rule.GetRule_analyze_table_list2().GetRule_analyze_table1().GetRule_simple_table_ref1(), tr)) { + return false; + } + + auto params = TAnalyzeParams{.Table = std::make_shared(tr), .Columns = std::move(columns)}; + AddStatementToBlocks(blocks, BuildAnalyze(Ctx.Pos(), tr.Service, tr.Cluster, params, Ctx.Scoped)); + break; + } case TRule_sql_stmt_core::ALT_NOT_SET: Ctx.IncrementMonCounter("sql_errors", "UnknownStatement" + internalStatementName); AltNotImplemented("sql_stmt_core", core); diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 90dae8d2ea58..2a65ccfcbba7 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -2493,8 +2493,8 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { } Y_UNIT_TEST(AlterTableAddIndexWithIsNotSupported) { - ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL WITH (a=b, c=d, e=f) ON (col)", - "
:1:40: Error: local: alternative is not implemented yet: 725:7: local_index\n"); + ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx GLOBAL ON (col) WITH (a=b)", + "
:1:40: Error: with: alternative is not implemented yet: 743:20: global_index\n"); } Y_UNIT_TEST(AlterTableAlterIndexSetPartitioningIsCorrect) { From 3edf5d1697f3c2de8d6364fdbf0c954f40915725 Mon Sep 17 00:00:00 2001 From: pilik Date: Fri, 23 Aug 2024 09:55:43 +0300 Subject: [PATCH 11/17] [KQP] ANALYZE retries has been added. (#8115) --- ydb/core/kqp/gateway/actors/analyze_actor.cpp | 115 +++++++++--------- ydb/core/kqp/gateway/actors/analyze_actor.h | 28 +++-- 2 files changed, 79 insertions(+), 64 deletions(-) diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.cpp b/ydb/core/kqp/gateway/actors/analyze_actor.cpp index 15b115697739..f887a84d93c5 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.cpp +++ b/ydb/core/kqp/gateway/actors/analyze_actor.cpp @@ -1,7 +1,6 @@ #include "analyze_actor.h" #include -#include #include #include @@ -29,65 +28,23 @@ void TAnalyzeActor::Bootstrap() { Become(&TAnalyzeActor::StateWork); } -void TAnalyzeActor::SendAnalyzeStatus() { - Y_ABORT_UNLESS(StatisticsAggregatorId.has_value()); - - auto getStatus = std::make_unique(); - auto& record = getStatus->Record; - PathIdFromPathId(PathId, record.MutablePathId()); - - Send( - MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward(getStatus.release(), StatisticsAggregatorId.value(), true) - ); -} - void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ev); Y_UNUSED(ctx); - SendAnalyzeStatus(); -} + const auto& record = ev->Get()->Record; + const TString operationId = record.GetOperationId(); -void TAnalyzeActor::Handle(TEvAnalyzePrivate::TEvAnalyzeStatusCheck::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - Y_UNUSED(ctx); - - SendAnalyzeStatus(); -} - -void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - switch (record.GetStatus()) { - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_UNSPECIFIED: { - Promise.SetValue( - NYql::NCommon::ResultFromError( - YqlIssue( - {}, NYql::TIssuesIds::UNEXPECTED, - TStringBuilder() << "Statistics Aggregator unspecified error" - ) - ) - ); - this->Die(ctx); - return; - } - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION: { - NYql::IKikimrGateway::TGenericResult result; - result.SetSuccess(); - Promise.SetValue(std::move(result)); - - this->Die(ctx); - return; - } - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED: { - Schedule(TDuration::Seconds(10), new TEvAnalyzePrivate::TEvAnalyzeStatusCheck()); - return; - } - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS: { - Schedule(TDuration::Seconds(5), new TEvAnalyzePrivate::TEvAnalyzeStatusCheck()); - return; - } + if (operationId != OperationId) { + ALOG_CRIT(NKikimrServices::KQP_GATEWAY, + "TAnalyzeActor, TEvAnalyzeResponse has operationId=" << operationId + << " , but expected " << OperationId); } + + NYql::IKikimrGateway::TGenericResult result; + result.SetSuccess(); + Promise.SetValue(std::move(result)); + this->Die(ctx); } void TAnalyzeActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { @@ -165,13 +122,57 @@ void TAnalyzeActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& } } +TDuration TAnalyzeActor::CalcBackoffTime() { + ui32 backoffSlots = 1 << RetryCount; + TDuration maxDuration = RetryInterval * backoffSlots; + + double uncertaintyRatio = std::max(std::min(UncertainRatio, 1.0), 0.0); + double uncertaintyMultiplier = RandomNumber() * uncertaintyRatio - uncertaintyRatio + 1.0; + + double durationMs = round(maxDuration.MilliSeconds() * uncertaintyMultiplier); + durationMs = std::max(std::min(durationMs, MaxBackoffDurationMs), 0.0); + return TDuration::MilliSeconds(durationMs); +} + +void TAnalyzeActor::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev, ctx); + + if (RetryCount >= MaxRetryCount) { + Promise.SetValue( + NYql::NCommon::ResultFromError( + YqlIssue( + {}, NYql::TIssuesIds::UNEXPECTED, + TStringBuilder() << "Can't establish connection with the Statistics Aggregator!" + ) + ) + ); + this->Die(ctx); + return; + } + + ++RetryCount; + Schedule(CalcBackoffTime(), new TEvAnalyzePrivate::TEvAnalyzeRetry()); +} + +void TAnalyzeActor::Handle(TEvAnalyzePrivate::TEvAnalyzeRetry::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev, ctx); + + auto analyzeRequest = std::make_unique(); + analyzeRequest->Record = Request.Record; + Send( + MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(analyzeRequest.release(), StatisticsAggregatorId.value(), true), + IEventHandle::FlagTrackDelivery + ); +} + void TAnalyzeActor::SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TActorContext& ctx) { Y_ABORT_UNLESS(entry.DomainInfo->Params.HasStatisticsAggregator()); StatisticsAggregatorId = entry.DomainInfo->Params.GetStatisticsAggregator(); - auto analyzeRequest = std::make_unique(); - auto& record = analyzeRequest->Record; + auto& record = Request.Record; + record.SetOperationId(OperationId); auto table = record.AddTables(); PathIdFromPathId(PathId, table->MutablePathId()); @@ -199,6 +200,8 @@ void TAnalyzeActor::SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeC *table->MutableColumnTags()->Add() = tagByColumnName[columnName]; } + auto analyzeRequest = std::make_unique(); + analyzeRequest->Record = Request.Record; Send( MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(analyzeRequest.release(), entry.DomainInfo->Params.GetStatisticsAggregator(), true), diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.h b/ydb/core/kqp/gateway/actors/analyze_actor.h index af0b83cb1ce6..b99f2f29866e 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.h +++ b/ydb/core/kqp/gateway/actors/analyze_actor.h @@ -3,6 +3,7 @@ #include #include +#include #include @@ -11,11 +12,11 @@ namespace NKikimr::NKqp { struct TEvAnalyzePrivate { enum EEv { - EvAnalyzeStatusCheck = EventSpaceBegin(TEvents::ES_PRIVATE), + EvAnalyzeRetry = EventSpaceBegin(TEvents::ES_PRIVATE), EvEnd }; - struct TEvAnalyzeStatusCheck : public TEventLocal {}; + struct TEvAnalyzeRetry : public TEventLocal {}; }; class TAnalyzeActor : public NActors::TActorBootstrapped { @@ -32,8 +33,8 @@ class TAnalyzeActor : public NActors::TActorBootstrapped { switch(ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); HFunc(NStat::TEvStatistics::TEvAnalyzeResponse, Handle); - HFunc(NStat::TEvStatistics::TEvAnalyzeStatusResponse, Handle); - HFunc(TEvAnalyzePrivate::TEvAnalyzeStatusCheck, Handle); + HFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + HFunc(TEvAnalyzePrivate::TEvAnalyzeRetry, Handle); default: HandleUnexpectedEvent(ev->GetTypeRewrite()); } @@ -42,17 +43,18 @@ class TAnalyzeActor : public NActors::TActorBootstrapped { private: void Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx); - void Handle(NStat::TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev, const TActorContext& ctx); - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvAnalyzePrivate::TEvAnalyzeStatusCheck::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev, const TActorContext& ctx); + + void Handle(TEvAnalyzePrivate::TEvAnalyzeRetry::TPtr& ev, const TActorContext& ctx); void HandleUnexpectedEvent(ui32 typeRewrite); +private: void SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeCacheNavigate::TEntry&, const TActorContext&); - void SendAnalyzeStatus(); + TDuration CalcBackoffTime(); private: TString TablePath; @@ -61,6 +63,16 @@ class TAnalyzeActor : public NActors::TActorBootstrapped { // For Statistics Aggregator std::optional StatisticsAggregatorId; TPathId PathId; + TString OperationId; + + // for retries + NStat::TEvStatistics::TEvAnalyze Request; + TDuration RetryInterval = TDuration::MilliSeconds(5); + size_t RetryCount = 0; + + constexpr static size_t MaxRetryCount = 10; + constexpr static double UncertainRatio = 0.5; + constexpr static double MaxBackoffDurationMs = TDuration::Seconds(15).MilliSeconds(); }; } // end of NKikimr::NKqp From f750c361ddfa324989117a759666733202f6f74b Mon Sep 17 00:00:00 2001 From: pilik Date: Wed, 28 Aug 2024 18:07:32 +0300 Subject: [PATCH 12/17] [KQP] Analyze OLAP supported, OLTP unsupported. (#8395) --- ydb/core/kqp/provider/yql_kikimr_datasink.cpp | 7 +- ydb/core/kqp/ut/query/kqp_analyze_ut.cpp | 78 ++++++++++--------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 2ce3e49e146a..176161b5952c 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -881,11 +881,16 @@ class TKikimrDataSink : public TDataProviderBase return false; } - if (tableDesc.Metadata->Kind == EKikimrTableKind::Olap && mode != "replace" && mode != "drop" && mode != "drop_if_exists" && mode != "insert_abort" && mode != "update" && mode != "upsert" && mode != "delete" && mode != "update_on" && mode != "delete_on") { + if (tableDesc.Metadata->Kind == EKikimrTableKind::Olap && mode != "replace" && mode != "drop" && mode != "drop_if_exists" && mode != "insert_abort" && mode != "update" && mode != "upsert" && mode != "delete" && mode != "update_on" && mode != "delete_on" && mode != "analyze") { ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Write mode '" << static_cast(mode) << "' is not supported for olap tables.")); return true; } + if (tableDesc.Metadata->Kind == EKikimrTableKind::Datashard && mode == "analyze") { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << static_cast(mode) << " is not supported for oltp tables.")); + return true; + } + return false; } diff --git a/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp b/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp index b53cef9ada50..e4708c2676e1 100644 --- a/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_analyze_ut.cpp @@ -16,58 +16,52 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpAnalyze) { -void CreateTable(NStat::TTestEnv& env, const TString& databaseName, const TString& tableName) { - TTableClient client(env.GetDriver()); - auto session = client.CreateSession().GetValueSync().GetSession(); - - auto fullTableName = Sprintf("Root/%s/%s", databaseName.c_str(), tableName.c_str()); - auto result = session.ExecuteSchemeQuery(Sprintf(R"( - CREATE TABLE `%s` ( - Key Uint64 NOT NULL, - Value String, - PRIMARY KEY (Key) - ) - )", fullTableName.c_str())).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - TValueBuilder rows; - rows.BeginList(); - for (size_t i = 0; i < 1000; ++i) { - auto key = TValueBuilder().Uint64(i).Build(); - auto value = TValueBuilder().OptionalString("Hello, world!").Build(); - - rows.AddListItem(); - rows.BeginStruct(); - rows.AddMember("Key", key); - rows.AddMember("Value", value); - rows.EndStruct(); - } - rows.EndList(); - - result = client.BulkUpsert(fullTableName, rows.Build()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); -} - using namespace NStat; -Y_UNIT_TEST(AnalyzeDatashardTable) { +Y_UNIT_TEST_TWIN(AnalyzeTable, ColumnStore) { TTestEnv env(1, 1, 1, true); CreateDatabase(env, "Database"); TTableClient client(env.GetDriver()); auto session = client.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteSchemeQuery( - Sprintf(R"( + TString createTable = Sprintf(R"( CREATE TABLE `%s` ( Key Uint64 NOT NULL, Value String, PRIMARY KEY (Key) ) - )", "Root/Database/Table") - ).GetValueSync(); + )", "Root/Database/Table"); + if (ColumnStore) { + createTable += + R"( + PARTITION BY HASH(Key) + WITH ( + STORE = COLUMN, + AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 16 + ) + )"; + } + + auto result = session.ExecuteSchemeQuery(createTable).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + if (ColumnStore) { + result = session.ExecuteSchemeQuery( + Sprintf(R"( + ALTER OBJECT `%s` (TYPE TABLE) + SET ( + ACTION=UPSERT_INDEX, + NAME=cms_value, + TYPE=COUNT_MIN_SKETCH, + FEATURES=`{"column_names" : ['Value']}` + ); + )", "Root/Database/Table" + ) + ).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + TValueBuilder rows; rows.BeginList(); for (size_t i = 0; i < 1500; ++i) { @@ -88,7 +82,15 @@ Y_UNIT_TEST(AnalyzeDatashardTable) { result = session.ExecuteSchemeQuery( Sprintf(R"(ANALYZE `Root/%s/%s`)", "Database", "Table") ).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + if (ColumnStore) { + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } else { + UNIT_ASSERT(!result.IsSuccess()); + auto issues = result.GetIssues().ToString(); + UNIT_ASSERT_C(issues.find("analyze is not supported for oltp tables.") != TString::npos, issues); + return; + } auto& runtime = *env.GetServer().GetRuntime(); ui64 saTabletId; From 1061b8d987e65df72e8f34ccb1d8cc25ced85e9b Mon Sep 17 00:00:00 2001 From: stanislav_shchetinin Date: Thu, 29 Aug 2024 23:55:21 +0300 Subject: [PATCH 13/17] $$$[KQP] Status processing added$$$ (#8502) --- ydb/core/kqp/gateway/actors/analyze_actor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.cpp b/ydb/core/kqp/gateway/actors/analyze_actor.cpp index f887a84d93c5..5636f670fd5e 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.cpp +++ b/ydb/core/kqp/gateway/actors/analyze_actor.cpp @@ -34,6 +34,12 @@ void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, c const auto& record = ev->Get()->Record; const TString operationId = record.GetOperationId(); + const auto status = record.GetStatus(); + + if (status != NKikimrStat::TEvAnalyzeResponse::STATUS_SUCCESS) { + ALOG_CRIT(NKikimrServices::KQP_GATEWAY, + "TAnalyzeActor, TEvAnalyzeResponse has status=" << status); + } if (operationId != OperationId) { ALOG_CRIT(NKikimrServices::KQP_GATEWAY, From 8cbafd01011b83e949991ac90c3b9e3bfef0a909 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Fri, 30 Aug 2024 09:34:03 +0000 Subject: [PATCH 14/17] merge fix --- ydb/core/kqp/gateway/actors/analyze_actor.cpp | 14 +++++++++++++- ydb/core/kqp/gateway/actors/analyze_actor.h | 6 +----- ydb/core/statistics/ut_common/ut_common.cpp | 5 +---- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.cpp b/ydb/core/kqp/gateway/actors/analyze_actor.cpp index 5636f670fd5e..326d39a44002 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.cpp +++ b/ydb/core/kqp/gateway/actors/analyze_actor.cpp @@ -1,6 +1,7 @@ #include "analyze_actor.h" #include +#include #include #include @@ -14,6 +15,18 @@ enum { using TNavigate = NSchemeCache::TSchemeCacheNavigate; +TString MakeOperationId() { + TULIDGenerator ulidGen; + return ulidGen.Next(TActivationContext::Now()).ToBinary(); +} + +TAnalyzeActor::TAnalyzeActor(TString tablePath, TVector columns, NThreading::TPromise promise) + : TablePath(tablePath) + , Columns(columns) + , Promise(promise) + , OperationId(MakeOperationId()) +{} + void TAnalyzeActor::Bootstrap() { using TNavigate = NSchemeCache::TSchemeCacheNavigate; auto navigate = std::make_unique(); @@ -29,7 +42,6 @@ void TAnalyzeActor::Bootstrap() { } void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); Y_UNUSED(ctx); const auto& record = ev->Get()->Record; diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.h b/ydb/core/kqp/gateway/actors/analyze_actor.h index b99f2f29866e..f59fba90c2b1 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.h +++ b/ydb/core/kqp/gateway/actors/analyze_actor.h @@ -21,11 +21,7 @@ struct TEvAnalyzePrivate { class TAnalyzeActor : public NActors::TActorBootstrapped { public: - TAnalyzeActor(TString tablePath, TVector columns, NThreading::TPromise promise) - : TablePath(tablePath) - , Columns(columns) - , Promise(promise) - {} + TAnalyzeActor(TString tablePath, TVector columns, NThreading::TPromise promise); void Bootstrap(); diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index 8ff5063832e8..72d1b726d238 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -83,9 +83,6 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool CSController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); Server->GetRuntime()->SetLogPriority(NKikimrServices::STATISTICS, NActors::NLog::PRI_DEBUG); - Server->GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_DEBUG); - // Server->GetRuntime()->SetLogPriority(NKikimrServices::K, NActors::NLog::PRI_DEBUG); - } TTestEnv::~TTestEnv() { @@ -312,7 +309,7 @@ void DropTable(TTestEnv& env, const TString& databaseName, const TString& tableN UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } -std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, TPathId pathId, ui64 columnTag) { +std::shared_ptr ExtractCountMin(TTestActorRuntime& runtime, const TPathId& pathId, ui64 columnTag) { auto statServiceId = NStat::MakeStatServiceID(runtime.GetNodeId(1)); NStat::TRequest req; From a424f4c22b566f2d38be61278912335fcd880823 Mon Sep 17 00:00:00 2001 From: andrew stalin Date: Fri, 30 Aug 2024 17:19:33 +0700 Subject: [PATCH 15/17] StatisticsService: updated HTML page (#8507) --- ydb/core/statistics/service/service_impl.cpp | 82 ++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/ydb/core/statistics/service/service_impl.cpp b/ydb/core/statistics/service/service_impl.cpp index ac553927cbe8..e8116833033b 100644 --- a/ydb/core/statistics/service/service_impl.cpp +++ b/ydb/core/statistics/service/service_impl.cpp @@ -1246,16 +1246,76 @@ class TStatService : public TActorBootstrapped { TBase::PassAway(); } - void Handle(NMon::TEvHttpInfo::TPtr& ev) { - auto& request = ev->Get()->Request; + void PrintStatServiceState(TStringStream& str) { + HTML(str) { + PRE() { + str << "---- StatisticsService ----" << Endl << Endl; + str << "StatisticsAggregatorId: " << StatisticsAggregatorId << Endl; + str << "SAPipeClientId: " << SAPipeClientId << Endl; + + str << "InFlight: " << InFlight.size(); + { + ui32 simple{ 0 }; + ui32 countMin{ 0 }; + for (auto it = InFlight.begin(); it != InFlight.end(); ++it) { + if (it->second.StatType == EStatType::SIMPLE) { + ++simple; + } else if (it->second.StatType == EStatType::COUNT_MIN_SKETCH) { + ++countMin; + } + } + str << "[SIMPLE: " << simple << ", COUNT_MIN_SKETCH: " << countMin << "]" << Endl; + } + str << "NextRequestId: " << NextRequestId << Endl; - if (!EnableColumnStatistics) { - Send(ev->Sender, new NMon::TEvHttpInfoRes("Column statistics is disabled")); - return; + str << "LoadQueriesInFlight: " << LoadQueriesInFlight.size() << Endl; + str << "NextLoadQueryCookie: " << NextLoadQueryCookie << Endl; + + str << "NeedSchemeShards: " << NeedSchemeShards.size() << Endl; + str << "Statistics: " << Statistics.size() << Endl; + + str << "ResolveSAStage: "; + if (ResolveSAStage == RSA_INITIAL) { + str << "RSA_INITIAL"; + } else if (ResolveSAStage == RSA_IN_FLIGHT) { + str << "RSA_IN_FLIGHT"; + } + else { + str << "RSA_FINISHED"; + } + str << Endl; + + str << "AggregateKeepAlivePeriod: " << Settings.AggregateKeepAlivePeriod << Endl; + str << "AggregateKeepAliveTimeout: " << Settings.AggregateKeepAliveTimeout << Endl; + str << "AggregateKeepAliveAckTimeout: " << Settings.AggregateKeepAliveAckTimeout << Endl; + str << "StatisticsRequestTimeout: " << Settings.StatisticsRequestTimeout << Endl; + str << "MaxInFlightTabletRequests: " << Settings.MaxInFlightTabletRequests << Endl; + str << "FanOutFactor: " << Settings.FanOutFactor << Endl; + + str << "---- AggregationStatistics ----" << Endl; + str << "Round: " << AggregationStatistics.Round << Endl; + str << "Cookie: " << AggregationStatistics.Cookie << Endl; + str << "PathId: " << AggregationStatistics.PathId.ToString() << Endl; + str << "LastAckHeartbeat: " << AggregationStatistics.LastAckHeartbeat << Endl; + str << "ParentNode: " << AggregationStatistics.ParentNode << Endl; + str << "PprocessedNodes: " << AggregationStatistics.PprocessedNodes << Endl; + str << "TotalStatisticsResponse: " << AggregationStatistics.TotalStatisticsResponse << Endl; + str << "Nodes: " << AggregationStatistics.Nodes.size() << Endl; + str << "CountMinSketches: " << AggregationStatistics.CountMinSketches.size() << Endl; + } } + } + + void Handle(NMon::TEvHttpInfo::TPtr& ev) { + auto& request = ev->Get()->Request; auto method = request.GetMethod(); if (method == HTTP_METHOD_POST) { + if (!EnableColumnStatistics) { + Send(ev->Sender, new NMon::TEvHttpInfoRes("Column statistics is disabled")); + return; + } + auto& params = request.GetPostParams(); auto itAction = params.find("action"); if (itAction == params.end()) { @@ -1276,6 +1336,18 @@ class TStatService : public TActorBootstrapped { } else if (method == HTTP_METHOD_GET) { auto& params = request.GetParams(); + if (params.empty()) { + TStringStream str; + PrintStatServiceState(str); + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); + return; + } + + if (!EnableColumnStatistics) { + Send(ev->Sender, new NMon::TEvHttpInfoRes("Column statistics is disabled")); + return; + } + auto itAction = params.find("action"); if (itAction == params.end()) { Send(ev->Sender, new NMon::TEvHttpInfoRes("'action' parameter is required")); From b152db7121c6d9414efed651005612d666e92f83 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Fri, 30 Aug 2024 13:07:58 +0000 Subject: [PATCH 16/17] merge fixes from Ildar --- ydb/core/statistics/ut_common/ut_common.cpp | 4 ++-- ydb/core/tx/columnshard/splitter/chunks.h | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index 72d1b726d238..1219807ddf0e 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -78,8 +78,8 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, bool DriverConfig = NYdb::TDriverConfig().SetEndpoint(Endpoint); Driver = MakeHolder(DriverConfig); - CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - CSController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); + CSController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + CSController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); CSController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); Server->GetRuntime()->SetLogPriority(NKikimrServices::STATISTICS, NActors::NLog::PRI_DEBUG); diff --git a/ydb/core/tx/columnshard/splitter/chunks.h b/ydb/core/tx/columnshard/splitter/chunks.h index bb0bbb287ce7..8ee403ccb24f 100644 --- a/ydb/core/tx/columnshard/splitter/chunks.h +++ b/ydb/core/tx/columnshard/splitter/chunks.h @@ -156,6 +156,14 @@ class TChunkedBatchReader { std::vector::const_iterator end() const { return Columns.end(); } + + std::vector::iterator begin() { + return Columns.begin(); + } + + std::vector::iterator end() { + return Columns.end(); + } }; } From 4a848196b5dc404fa0be06f6a97e451da27b2326 Mon Sep 17 00:00:00 2001 From: azevaykin Date: Fri, 30 Aug 2024 13:36:49 +0000 Subject: [PATCH 17/17] sql grammar fix --- ydb/library/yql/sql/v1/sql.cpp | 2 +- ydb/library/yql/sql/v1/sql_query.cpp | 4 ++-- ydb/library/yql/sql/v1/sql_ut.cpp | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index f2bdd926ec79..57d9ce996e44 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -167,7 +167,7 @@ bool NeedUseForAllStatements(const TRule_sql_stmt_core::AltCase& subquery) { case TRule_sql_stmt_core::kAltSqlStmtCore45: // create resource pool case TRule_sql_stmt_core::kAltSqlStmtCore46: // alter resource pool case TRule_sql_stmt_core::kAltSqlStmtCore47: // drop resource pool - case TRule_sql_stmt_core::kAltSqlStmtCore51: // analyze + case TRule_sql_stmt_core::kAltSqlStmtCore48: // analyze return false; } } diff --git a/ydb/library/yql/sql/v1/sql_query.cpp b/ydb/library/yql/sql/v1/sql_query.cpp index 08ce8a20c80a..f49e44e7f807 100644 --- a/ydb/library/yql/sql/v1/sql_query.cpp +++ b/ydb/library/yql/sql/v1/sql_query.cpp @@ -1344,10 +1344,10 @@ bool TSqlQuery::Statement(TVector& blocks, const TRule_sql_stmt_core& AddStatementToBlocks(blocks, BuildDropObjectOperation(Ctx.Pos(), objectId, "RESOURCE_POOL", false, {}, context)); break; } - case TRule_sql_stmt_core::kAltSqlStmtCore51: { + case TRule_sql_stmt_core::kAltSqlStmtCore48: { // analyze_stmt: ANALYZE table_ref Ctx.BodyPart(); - const auto& rule = core.GetAlt_sql_stmt_core51().GetRule_analyze_stmt1(); + const auto& rule = core.GetAlt_sql_stmt_core48().GetRule_analyze_stmt1(); if (!rule.GetRule_analyze_table_list2().GetBlock2().empty()) { Error() << "ANALYZE with multitables hasn't been implemented yet"; diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 2a65ccfcbba7..8434e1aab863 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -2497,6 +2497,11 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { "
:1:40: Error: with: alternative is not implemented yet: 743:20: global_index\n"); } + Y_UNIT_TEST(AlterTableAddIndexLocalIsNotSupported) { + ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL ON (col)", + "
:1:40: Error: local: alternative is not implemented yet: 743:35: local_index\n"); + } + Y_UNIT_TEST(AlterTableAlterIndexSetPartitioningIsCorrect) { const auto result = SqlToYql("USE plato; ALTER TABLE table ALTER INDEX index SET AUTO_PARTITIONING_MIN_PARTITIONS_COUNT 10"); UNIT_ASSERT_C(result.IsOk(), result.Issues.ToString());