diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 8ae4cd55ae93..30fb6cf05263 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -24,6 +24,15 @@ ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingConsistency64 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingModuloN ydb/core/kqp/ut/olap KqpOlapBlobsSharing.UpsertWhileSplitTest ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL +ydb/core/kqp/ut/olap KqpOlapSysView.StatsSysViewBytesDictStatActualization +ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_SumL_GroupL_OrderL +ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesActualization +ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesInBS +ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesInLocalMetadata +ydb/core/tx/columnshard/ut_rw Normalizers.CleanEmptyPortionsNormalizer +ydb/core/kqp/ut/pg KqpPg.CreateIndex +ydb/core/kqp/ut/query KqpLimits.QueryReplySize +ydb/core/kqp/ut/query KqpQuery.QueryTimeout ydb/core/kqp/ut/query KqpLimits.ComputeActorMemoryAllocationFailureQueryService ydb/core/kqp/ut/query KqpLimits.QueryExecTimeoutCancel ydb/core/kqp/ut/query KqpStats.SysViewClientLost diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 67fd92b0f7c7..d703e291d4cb 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -183,6 +183,7 @@ struct TKikimrEvents : TEvents { ES_GROUPED_ALLOCATIONS_MANAGER = 4260, ES_INCREMENTAL_RESTORE_SCAN = 4261, ES_FEATURE_FLAGS = 4262, + ES_PRIORITY_QUEUE = 4263, }; }; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index e34030b4ba13..4e4311aa7195 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -185,6 +185,8 @@ #include #include #include +#include +#include #include #include #include @@ -2199,6 +2201,28 @@ void TCompDiskLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* } } +TCompPrioritiesInitializer::TCompPrioritiesInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TCompPrioritiesInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + NPrioritiesQueue::TConfig serviceConfig; + if (Config.HasCompPrioritiesConfig()) { + Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetCompPrioritiesConfig())); + } + + if (serviceConfig.IsEnabled()) { + TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); + TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_COMP_PRIORITIES"); + + auto service = NPrioritiesQueue::TCompServiceOperator::CreateService(serviceConfig, conveyorGroup); + + setup->LocalServices.push_back(std::make_pair( + NPrioritiesQueue::TCompServiceOperator::MakeServiceId(NodeId), + TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); + } +} + TCompConveyorInitializer::TCompConveyorInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 44a20f28a224..d33901a3dd0b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -410,6 +410,12 @@ class TGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +class TCompPrioritiesInitializer: public IKikimrServicesInitializer { +public: + TCompPrioritiesInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TCompConveyorInitializer: public IKikimrServicesInitializer { public: TCompConveyorInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index f7458b872525..e60f4d631b49 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1613,6 +1613,10 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TScanConveyorInitializer(runConfig)); } + if (serviceMask.EnableCompPriorities) { + sil->AddServiceInitializer(new TCompPrioritiesInitializer(runConfig)); + } + if (serviceMask.EnableCompConveyor) { sil->AddServiceInitializer(new TCompConveyorInitializer(runConfig)); } diff --git a/ydb/core/driver_lib/run/service_mask.h b/ydb/core/driver_lib/run/service_mask.h index 044557229c6b..9bb31d2df8b8 100644 --- a/ydb/core/driver_lib/run/service_mask.h +++ b/ydb/core/driver_lib/run/service_mask.h @@ -80,6 +80,7 @@ union TBasicKikimrServicesMask { bool EnableCompDiskLimiter:1; bool EnableGroupedMemoryLimiter:1; bool EnableAwsService:1; + bool EnableCompPriorities : 1; }; struct { diff --git a/ydb/core/formats/arrow/accessor/plain/accessor.h b/ydb/core/formats/arrow/accessor/plain/accessor.h index a00826161c40..73b8510080ae 100644 --- a/ydb/core/formats/arrow/accessor/plain/accessor.h +++ b/ydb/core/formats/arrow/accessor/plain/accessor.h @@ -32,6 +32,10 @@ class TTrivialArray: public IChunkedArray { } public: + const std::shared_ptr& GetArray() const { + return Array; + } + TTrivialArray(const std::shared_ptr& data) : TBase(data->length(), EType::Array, data->type()) , Array(data) { diff --git a/ydb/core/formats/arrow/accessor/plain/constructor.cpp b/ydb/core/formats/arrow/accessor/plain/constructor.cpp index 2a2b4657596b..c81b65023f35 100644 --- a/ydb/core/formats/arrow/accessor/plain/constructor.cpp +++ b/ydb/core/formats/arrow/accessor/plain/constructor.cpp @@ -35,10 +35,15 @@ std::shared_ptr TConstructor::DoGetExpectedSchema(const std::shar std::shared_ptr TConstructor::DoConstruct( const std::shared_ptr& columnData, const TChunkConstructionData& externalInfo) const { - auto chunked = columnData->GetChunkedArray(); auto schema = std::make_shared(arrow::FieldVector({ std::make_shared("val", externalInfo.GetColumnType()) })); - auto table = arrow::Table::Make(schema, { chunked }, columnData->GetRecordsCount()); - return NArrow::ToBatch(table, true); + if (columnData->GetType() == IChunkedArray::EType::Array) { + const auto* arr = static_cast(columnData.get()); + return arrow::RecordBatch::Make(schema, columnData->GetRecordsCount(), { arr->GetArray() }); + } else { + auto chunked = columnData->GetChunkedArray(); + auto table = arrow::Table::Make(schema, { chunked }, columnData->GetRecordsCount()); + return NArrow::ToBatch(table, chunked->num_chunks() > 1); + } } } // namespace NKikimr::NArrow::NAccessor::NPlain diff --git a/ydb/core/formats/arrow/program.cpp b/ydb/core/formats/arrow/program.cpp index 50071d8490e3..cf95164f3dec 100644 --- a/ydb/core/formats/arrow/program.cpp +++ b/ydb/core/formats/arrow/program.cpp @@ -555,7 +555,7 @@ arrow::Status TDatumBatch::AddColumn(const std::string& name, arrow::Datum&& col auto field = arrow::field(name, column.type()); if (!field || !field->type()->Equals(column.type())) { - return arrow::Status::Invalid("Cannot create field."); + return arrow::Status::Invalid("Cannot create field " + name + ". type:" + field->type()->ToString() + " vs " + column.type()->ToString()); } if (!column.is_scalar() && column.length() != Rows) { return arrow::Status::Invalid("Wrong column length."); diff --git a/ydb/core/formats/arrow/reader/position.cpp b/ydb/core/formats/arrow/reader/position.cpp index b728405769d7..e4f4d799a5a1 100644 --- a/ydb/core/formats/arrow/reader/position.cpp +++ b/ydb/core/formats/arrow/reader/position.cpp @@ -133,6 +133,15 @@ TSortableScanData::TSortableScanData( BuildPosition(position); } +TSortableScanData::TSortableScanData( + const ui64 position, const std::shared_ptr& batch) { + for (auto&& c : batch->columns()) { + Columns.emplace_back(std::make_shared(c)); + } + Fields = batch->schema()->fields(); + BuildPosition(position); +} + TSortableScanData::TSortableScanData(const ui64 position, const std::shared_ptr& batch, const std::vector& columns) { for (auto&& i : columns) { auto c = batch->GetColumnByName(i); diff --git a/ydb/core/formats/arrow/reader/position.h b/ydb/core/formats/arrow/reader/position.h index 6dcd5ce144cf..f7f6c41dc208 100644 --- a/ydb/core/formats/arrow/reader/position.h +++ b/ydb/core/formats/arrow/reader/position.h @@ -75,6 +75,7 @@ class TSortableScanData { return StartPosition <= position && position < FinishPosition; } public: + TSortableScanData(const ui64 position, const std::shared_ptr& batch); TSortableScanData(const ui64 position, const std::shared_ptr& batch, const std::vector& columns); TSortableScanData(const ui64 position, const std::shared_ptr& batch, const std::vector& columns); TSortableScanData(const ui64 position, const std::shared_ptr& batch, const std::vector& columns); @@ -357,6 +358,19 @@ class TSortableBatchPosition { Y_ABORT_UNLESS(Sorting->GetColumns().size()); } + template + TSortableBatchPosition(const std::shared_ptr& batch, const ui32 position, const bool reverseSort) + : Position(position) + , ReverseSort(reverseSort) { + Y_ABORT_UNLESS(batch); + Y_ABORT_UNLESS(batch->num_rows()); + RecordsCount = batch->num_rows(); + AFL_VERIFY(Position < RecordsCount)("position", Position)("count", RecordsCount); + Sorting = std::make_shared(Position, batch); + Y_DEBUG_ABORT_UNLESS(batch->ValidateFull().ok()); + Y_ABORT_UNLESS(Sorting->GetColumns().size()); + } + std::partial_ordering GetReverseForCompareResult(const std::partial_ordering directResult) const { if (directResult == std::partial_ordering::less) { return std::partial_ordering::greater; @@ -496,19 +510,17 @@ class TIntervalPositions { void AddPosition(TIntervalPosition&& intervalPosition) { if (Positions.size()) { - AFL_VERIFY(Positions.back() < intervalPosition)("back", Positions.back().DebugJson())("pos", intervalPosition.DebugJson()); + AFL_VERIFY_DEBUG(Positions.back() < intervalPosition)("back", Positions.back().DebugJson())("pos", intervalPosition.DebugJson()); } Positions.emplace_back(std::move(intervalPosition)); } void AddPosition(TSortableBatchPosition&& position, const bool includePositionToLeftInterval) { - TIntervalPosition intervalPosition(std::move(position), includePositionToLeftInterval); - AddPosition(std::move(intervalPosition)); + AddPosition(TIntervalPosition(std::move(position), includePositionToLeftInterval)); } void AddPosition(const TSortableBatchPosition& position, const bool includePositionToLeftInterval) { - TIntervalPosition intervalPosition(position, includePositionToLeftInterval); - AddPosition(std::move(intervalPosition)); + AddPosition(TIntervalPosition(position, includePositionToLeftInterval)); } }; @@ -580,7 +592,11 @@ class TRWSortableBatchPosition: public TSortableBatchPosition, public TMoveOnly result.emplace_back(nullptr); return result; } + if (!it.IsValid()) { + return { batch }; + } TRWSortableBatchPosition pos(batch, 0, columnNames, {}, false); + it.SkipToUpper(pos); bool batchFinished = false; i64 recordsCountSplitted = 0; for (; it.IsValid() && !batchFinished; it.Next()) { @@ -636,6 +652,10 @@ class TRWSortableBatchPosition: public TSortableBatchPosition, public TMoveOnly const auto& CurrentPosition() const { return Current->first; } + + void SkipToUpper(const TSortableBatchPosition& /*toPos*/) { + return; + } }; template @@ -666,6 +686,10 @@ class TRWSortableBatchPosition: public TSortableBatchPosition, public TMoveOnly const auto& CurrentPosition() const { return *Current; } + + void SkipToUpper(const TSortableBatchPosition& /*toPos*/) { + return; + } }; template @@ -676,8 +700,8 @@ class TRWSortableBatchPosition: public TSortableBatchPosition, public TMoveOnly class TIntervalPointsIterator { private: - typename TIntervalPositions::const_iterator Current; - typename TIntervalPositions::const_iterator End; + TIntervalPositions::const_iterator Current; + TIntervalPositions::const_iterator End; public: TIntervalPointsIterator(const TIntervalPositions& container) @@ -696,6 +720,20 @@ class TRWSortableBatchPosition: public TSortableBatchPosition, public TMoveOnly const auto& CurrentPosition() const { return Current->GetPosition(); } + + struct TComparator { + bool operator()(const TIntervalPosition& pos, const TSortableBatchPosition& value) const { + return pos.GetPosition() < value; + } + bool operator()(const TSortableBatchPosition& value, const TIntervalPosition& pos) const { + return value < pos.GetPosition(); + } + + }; + + void SkipToUpper(const TSortableBatchPosition& toPos) { + Current = std::upper_bound(Current, End, toPos, TComparator()); + } }; static std::vector> SplitByBordersInIntervalPositions( diff --git a/ydb/core/formats/arrow/save_load/loader.cpp b/ydb/core/formats/arrow/save_load/loader.cpp index 33685100bd15..24b01d7ff759 100644 --- a/ydb/core/formats/arrow/save_load/loader.cpp +++ b/ydb/core/formats/arrow/save_load/loader.cpp @@ -55,6 +55,17 @@ TChunkConstructionData TColumnLoader::BuildAccessorContext(const ui32 recordsCou return TChunkConstructionData(recordsCount, DefaultValue, ResultField->type()); } +TConclusion> TColumnLoader::ApplyConclusion(const TString& dataStr, const ui32 recordsCount) const { + auto result = Apply(dataStr); + if (result.ok()) { + return BuildAccessor(*result, BuildAccessorContext(recordsCount)); + } else { + AFL_ERROR(NKikimrServices::ARROW_HELPER)("event", "cannot_parse_blob")("data_size", dataStr.size())( + "expected_records_count", recordsCount)("problem", result.status().ToString()); + return TConclusionStatus::Fail(result.status().ToString()); + } +} + std::shared_ptr TColumnLoader::ApplyVerified(const TString& dataStr, const ui32 recordsCount) const { auto data = TStatusValidator::GetValid(Apply(dataStr)); return BuildAccessor(data, BuildAccessorContext(recordsCount)); diff --git a/ydb/core/formats/arrow/save_load/loader.h b/ydb/core/formats/arrow/save_load/loader.h index eb5a2740ef9b..64ecb6c21168 100644 --- a/ydb/core/formats/arrow/save_load/loader.h +++ b/ydb/core/formats/arrow/save_load/loader.h @@ -41,6 +41,7 @@ class TColumnLoader { TChunkConstructionData BuildAccessorContext(const ui32 recordsCount) const; std::shared_ptr ApplyVerified(const TString& data, const ui32 expectedRecordsCount) const; + TConclusion> ApplyConclusion(const TString& data, const ui32 expectedRecordsCount) const; std::shared_ptr ApplyRawVerified(const TString& data) const; }; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index d5ae1c444051..4a34c3292d66 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -620,6 +620,11 @@ message TConveyorConfig { optional double WorkersCountDouble = 5; } +message TPrioritiesQueueConfig { + optional bool Enabled = 1 [default = true]; + optional uint32 Limit = 2 [default = 32]; +} + message TLimiterConfig { optional bool Enabled = 1 [default = true]; optional uint64 Limit = 2; @@ -2039,9 +2044,10 @@ message TAppConfig { optional TLimiterConfig CompDiskLimiterConfig = 79; optional TMetadataCacheConfig MetadataCacheConfig = 80; optional TMemoryControllerConfig MemoryControllerConfig = 81; - optional TGroupedMemoryLimiterConfig GroupedMemoryLimiterConfig = 82; + optional TGroupedMemoryLimiterConfig GroupedMemoryLimiterConfig = 82; optional NKikimrReplication.TReplicationDefaults ReplicationConfig = 83; optional TShutdownConfig ShutdownConfig = 84; + optional TPrioritiesQueueConfig CompPrioritiesConfig = 85; repeated TNamedConfig NamedConfigs = 100; optional string ClusterYamlConfig = 101; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 447b936fb918..b161c98de09b 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -140,7 +140,9 @@ message TConfigItem { BackgroundCleaningConfigItem = 77; MetadataCacheConfigItem = 80; MemoryControllerConfigItem = 81; + GroupedMemoryLimiterConfig = 82; ReplicationConfigItem = 83; + CompPrioritiesConfig = 85; NamedConfigsItem = 100; ClusterYamlConfigItem = 101; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 01aba0b2d661..90e748b2660e 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -453,9 +453,14 @@ message TCompactionPlannerConstructorContainer { optional uint32 FreshnessCheckDurationSeconds = 2 [default = 300]; } + message TLCOptimizer { + + } + oneof Implementation { TLOptimizer LBuckets = 20; TSOptimizer SBuckets = 21; + TLCOptimizer LCBuckets = 22; } } diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index ce14d46c4698..645a60063616 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -534,6 +534,8 @@ struct Schema : NIceDb::Schema { struct TierName: Column<11, NScheme::NTypeIds::Utf8> {}; struct Stats: Column<12, NScheme::NTypeIds::Utf8> {}; struct Optimized: Column<13, NScheme::NTypeIds::Uint8> {}; + struct CompactionLevel: Column<14, NScheme::NTypeIds::Uint64> {}; + struct Details: Column<15, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -549,7 +551,9 @@ struct Schema : NIceDb::Schema { Activity, TierName, Stats, - Optimized + Optimized, + CompactionLevel, + Details >; }; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index c733d0e3050d..972b3a3da373 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -119,6 +119,7 @@ #include #include #include +#include #include #include @@ -832,6 +833,11 @@ namespace Tests { const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); Runtime->RegisterService(NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } + { + auto* actor = NPrioritiesQueue::TCompServiceOperator::CreateService(NPrioritiesQueue::TConfig(), new ::NMonitoring::TDynamicCounters()); + const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); + Runtime->RegisterService(NPrioritiesQueue::TCompServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); + } { auto* actor = NConveyor::TScanServiceOperator::CreateService(NConveyor::TConfig(), new ::NMonitoring::TDynamicCounters()); const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index 5478b7f6c8cc..bc723afd9c24 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -103,6 +103,7 @@ PEERDIR( ydb/services/ext_index/service ydb/services/ymq ydb/core/tx/conveyor/service + ydb/core/tx/priorities/service ydb/core/tx/limiter/grouped_memory/usage ydb/services/fq ydb/services/kesus diff --git a/ydb/core/tx/columnshard/background_controller.cpp b/ydb/core/tx/columnshard/background_controller.cpp index 7449e7d31ff4..1a26f8ed32f7 100644 --- a/ydb/core/tx/columnshard/background_controller.cpp +++ b/ydb/core/tx/columnshard/background_controller.cpp @@ -4,14 +4,18 @@ namespace NKikimr::NColumnShard { bool TBackgroundController::StartCompaction(const NOlap::TPlanCompactionInfo& info) { - Y_ABORT_UNLESS(ActiveCompactionInfo.emplace(info.GetPathId(), info).second); + auto it = ActiveCompactionInfo.find(info.GetPathId()); + if (it == ActiveCompactionInfo.end()) { + it = ActiveCompactionInfo.emplace(info.GetPathId(), info.GetPathId()).first; + } + it->second.Start(); return true; } void TBackgroundController::CheckDeadlines() { for (auto&& i : ActiveCompactionInfo) { if (TMonotonic::Now() - i.second.GetStartTime() > NOlap::TCompactionLimits::CompactionTimeout) { - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_compaction"); + AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_compaction")("path_id", i.first); Y_DEBUG_ABORT_UNLESS(false); } } diff --git a/ydb/core/tx/columnshard/background_controller.h b/ydb/core/tx/columnshard/background_controller.h index b57a29d5b072..817258b2c2fe 100644 --- a/ydb/core/tx/columnshard/background_controller.h +++ b/ydb/core/tx/columnshard/background_controller.h @@ -15,6 +15,7 @@ class TBackgroundController { using TCurrentCompaction = THashMap; TCurrentCompaction ActiveCompactionInfo; + std::optional WaitingCompactionPriority; std::shared_ptr Counters; bool ActiveCleanupPortions = false; @@ -25,21 +26,35 @@ class TBackgroundController { TBackgroundController(std::shared_ptr counters) : Counters(std::move(counters)) { } - THashSet GetConflictTTLPortions() const; THashSet GetConflictCompactionPortions() const; + void UpdateWaitingPriority(const ui64 priority) { + if (!WaitingCompactionPriority || *WaitingCompactionPriority < priority) { + WaitingCompactionPriority = priority; + } + } + + void ResetWaitingPriority() { + WaitingCompactionPriority.reset(); + } + + std::optional GetWaitingPriorityOptional() { + return WaitingCompactionPriority; + } + void CheckDeadlines(); void CheckDeadlinesIndexation(); bool StartCompaction(const NOlap::TPlanCompactionInfo& info); void FinishCompaction(const NOlap::TPlanCompactionInfo& info) { - Y_ABORT_UNLESS(ActiveCompactionInfo.erase(info.GetPathId())); + auto it = ActiveCompactionInfo.find(info.GetPathId()); + AFL_VERIFY(it != ActiveCompactionInfo.end()); + if (it->second.Finish()) { + ActiveCompactionInfo.erase(it); + } Counters->OnCompactionFinish(info.GetPathId()); } - const TCurrentCompaction& GetActiveCompaction() const { - return ActiveCompactionInfo; - } ui32 GetCompactionsCount() const { return ActiveCompactionInfo.size(); } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/read.h b/ydb/core/tx/columnshard/blobs_action/abstract/read.h index cd2a272feebb..e533ee9cf85e 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/read.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/read.h @@ -72,7 +72,7 @@ class TActionReadBlobs { TString Extract(const TBlobRange& bRange) { auto it = Blobs.find(bRange); - AFL_VERIFY(it != Blobs.end()); + AFL_VERIFY(it != Blobs.end())("range", bRange.ToString()); TString result = it->second; Blobs.erase(it); return result; diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp index 9e6c7738b8b1..20f26e5b1986 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp @@ -10,7 +10,7 @@ namespace NKikimr::NColumnShard { bool TTxBlobsWritingFinished::DoExecute(TTransactionContext& txc, const TActorContext&) { TMemoryProfileGuard mpg("TTxBlobsWritingFinished::Execute"); txc.DB.NoMoreReadsForTx(); - CommitSnapshot = NOlap::TSnapshot::MaxForPlanStep(Self->GetOutdatedStep()); + CommitSnapshot = Self->GetCurrentSnapshotForInternalModification(); NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("tx_state", "execute"); ACFL_DEBUG("event", "start_execute"); @@ -84,10 +84,12 @@ void TTxBlobsWritingFinished::DoComplete(const TActorContext& ctx) { i.DoSendReply(ctx); } auto& index = Self->MutableIndexAs(); + std::set pathIds; for (auto&& pack : Packs) { const auto& writeMeta = pack.GetWriteMeta(); AFL_VERIFY(!writeMeta.HasLongTxId()); auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); + pathIds.emplace(op->GetPathId()); auto& granule = index.MutableGranuleVerified(op->GetPathId()); for (auto&& portion : pack.GetPortions()) { if (op->GetBehaviour() == EOperationBehaviour::WriteWithLock || op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { @@ -97,18 +99,18 @@ void TTxBlobsWritingFinished::DoComplete(const TActorContext& ctx) { Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite); } } - if (op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - AFL_VERIFY(CommitSnapshot); - granule.CommitImmediateOnComplete(portion.GetPortionInfo(), index); - } else { - granule.InsertPortionOnComplete(portion.GetPortionInfo()); - } + granule.InsertPortionOnComplete(portion.GetPortionInfo()); + } + if (op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { + AFL_VERIFY(CommitSnapshot); + Self->OperationsManager->AddTemporaryTxLink(op->GetLockId()); + Self->OperationsManager->CommitTransactionOnComplete(*Self, op->GetLockId(), *CommitSnapshot); } Self->Counters.GetCSCounters().OnWriteTxComplete(now - writeMeta.GetWriteStartInstant()); Self->Counters.GetCSCounters().OnSuccessWriteResponse(); } + Self->SetupCompaction(pathIds); Self->Counters.GetTabletCounters()->IncCounter(COUNTER_IMMEDIATE_TX_COMPLETED); - Self->SetupCompaction(); } TTxBlobsWritingFinished::TTxBlobsWritingFinished(TColumnShard* self, const NKikimrProto::EReplyStatus writeStatus, diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 4bd2b6faf9c3..8bbcfce0e07d 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -34,7 +34,7 @@ bool TTxWrite::CommitOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali } bool TTxWrite::DoExecute(TTransactionContext& txc, const TActorContext&) { - CommitSnapshot = NOlap::TSnapshot::MaxForPlanStep(Self->GetOutdatedStep()); + CommitSnapshot = Self->GetCurrentSnapshotForInternalModification(); TMemoryProfileGuard mpg("TTxWrite::Execute"); NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("tx_state", "execute"); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp index 57a1eee50146..4c97c6d3f9bb 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp @@ -48,9 +48,10 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) } void TTxWriteIndex::Complete(const TActorContext& ctx) { - TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())); CompleteReady = true; auto changes = Ev->Get()->IndexChanges; + TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())( + "task_id", changes->GetTaskIdentifier())); TMemoryProfileGuard mpg("TTxWriteIndex::Complete::" + changes->TypeString()); ACFL_DEBUG("event", "TTxWriteIndex::Complete")("change_type", changes->TypeString())("details", changes->DebugString()); @@ -83,10 +84,9 @@ TTxWriteIndex::TTxWriteIndex(TColumnShard* self, TEvPrivate::TEvWriteIndex::TPtr { AFL_VERIFY(Ev && Ev->Get()->IndexChanges); - NOlap::TSnapshot snapshot(Self->LastPlannedStep, Self->LastPlannedTxId); auto changes = Ev->Get()->IndexChanges; if (Ev->Get()->GetPutStatus() == NKikimrProto::OK) { - AFL_VERIFY(Self->TablesManager.MutablePrimaryIndex().ApplyChangesOnTxCreate(changes, snapshot)); + AFL_VERIFY(Self->TablesManager.MutablePrimaryIndex().ApplyChangesOnTxCreate(changes, Self->GetCurrentSnapshotForInternalModification())); } } diff --git a/ydb/core/tx/columnshard/blobs_reader/task.cpp b/ydb/core/tx/columnshard/blobs_reader/task.cpp index 8306ccbc6309..bdf7b83f061e 100644 --- a/ydb/core/tx/columnshard/blobs_reader/task.cpp +++ b/ydb/core/tx/columnshard/blobs_reader/task.cpp @@ -89,7 +89,7 @@ ITask::ITask(const TReadActionsCollection& actions, const TString& taskCustomer, , TaskCustomer(taskCustomer) { Agents = actions; - AFL_VERIFY(!Agents.IsEmpty()); +// AFL_VERIFY(!Agents.IsEmpty()); for (auto&& i : Agents) { AFL_VERIFY(i.second->GetExpectedBlobsCount()); } diff --git a/ydb/core/tx/columnshard/blobs_reader/task.h b/ydb/core/tx/columnshard/blobs_reader/task.h index 1f04c963fb9a..509fb0d9ee22 100644 --- a/ydb/core/tx/columnshard/blobs_reader/task.h +++ b/ydb/core/tx/columnshard/blobs_reader/task.h @@ -65,7 +65,7 @@ class TCompositeReadBlobs { } TString Extract(const TString& storageId, const TBlobRange& range) { auto it = BlobsByStorage.find(storageId); - AFL_VERIFY(it != BlobsByStorage.end()); + AFL_VERIFY(it != BlobsByStorage.end())("range", range.ToString())("storage_id", storageId); auto result = it->second.Extract(range); if (it->second.IsEmpty()) { BlobsByStorage.erase(it); diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index f3a6b9e99db9..56794f1c0520 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -11,6 +11,7 @@ #include #include +#include #include namespace NKikimr { @@ -29,6 +30,9 @@ void TColumnShard::CleanupActors(const TActorContext& ctx) { } ctx.Send(ResourceSubscribeActor, new TEvents::TEvPoisonPill); ctx.Send(BufferizationWriteActorId, new TEvents::TEvPoisonPill); + if (PrioritizationClientId) { + NPrioritiesQueue::TCompServiceOperator::UnregisterClient(PrioritizationClientId); + } for (auto&& i : ActorsToStop) { ctx.Send(i, new TEvents::TEvPoisonPill); } @@ -101,6 +105,7 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { Settings.RegisterControls(icb); ResourceSubscribeActor = ctx.Register(new NOlap::NResourceBroker::NSubscribe::TActor(TabletID(), SelfId())); BufferizationWriteActorId = ctx.Register(new NColumnShard::NWriting::TActor(TabletID(), SelfId())); + PrioritizationClientId = NPrioritiesQueue::TCompServiceOperator::RegisterClient(); Execute(CreateTxInitSchema(), ctx); } diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index b4841f72f7b6..1f43312fdbef 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -88,6 +88,7 @@ TColumnShard::EOverloadStatus TColumnShard::CheckOverloaded(const ui64 pathId) c } void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& ev, const TActorContext& ctx) { + TMemoryProfileGuard mpg("TEvWritePortionResult"); NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvWritePortionResult"); AFL_VERIFY(ev->Get()->GetWriteStatus() == NKikimrProto::OK); @@ -447,6 +448,7 @@ class TAbortWriteTransaction: public NTabletFlatExecutor::TTransactionBase #include #include +#include +#include +#include #include namespace NKikimr::NColumnShard { @@ -511,7 +515,7 @@ void TColumnShard::EnqueueBackgroundActivities(const bool periodic) { SharingSessionsManager->Start(*this); SetupIndexation(); - SetupCompaction(); + SetupCompaction({}); SetupCleanupPortions(); SetupCleanupTables(); SetupTtl(); @@ -705,33 +709,70 @@ void TColumnShard::SetupIndexation() { } } -void TColumnShard::SetupCompaction() { - if (!AppDataVerified().ColumnShardConfig.GetCompactionEnabled() || !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::Compaction)) { +namespace { +class TCompactionAllocated: public NPrioritiesQueue::IRequest { +private: + const NActors::TActorId TabletActorId; + virtual void DoOnAllocated(const std::shared_ptr& guard) override { + NActors::TActorContext::AsActorContext().Send(TabletActorId, new TEvPrivate::TEvStartCompaction(guard)); + } + +public: + TCompactionAllocated(const NActors::TActorId& tabletActorId) + : TabletActorId(tabletActorId) + { + + } +}; +} // namespace + +void TColumnShard::SetupCompaction(const std::set& pathIds) { + if (!AppDataVerified().ColumnShardConfig.GetCompactionEnabled() || + !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::Compaction)) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_compaction")("reason", "disabled"); return; } - Counters.GetCSCounters().OnSetupCompaction(); BackgroundController.CheckDeadlines(); - while (BackgroundController.GetCompactionsCount() < TSettings::MAX_ACTIVE_COMPACTIONS) { - auto indexChanges = TablesManager.MutablePrimaryIndex().StartCompaction(DataLocksManager); - if (!indexChanges) { - LOG_S_DEBUG("Compaction not started: cannot prepare compaction at tablet " << TabletID()); - break; + if (BackgroundController.GetCompactionsCount()) { + return; + } + const ui64 priority = TablesManager.MutablePrimaryIndex().GetCompactionPriority(DataLocksManager, pathIds, BackgroundController.GetWaitingPriorityOptional()); + if (priority) { + BackgroundController.UpdateWaitingPriority(priority); + if (pathIds.size()) { + NPrioritiesQueue::TCompServiceOperator::AskMax(PrioritizationClientId, priority, std::make_shared(SelfId())); + } else { + NPrioritiesQueue::TCompServiceOperator::Ask(PrioritizationClientId, priority, std::make_shared(SelfId())); } + } +} - indexChanges->Start(*this); - - auto actualIndexInfo = std::make_shared(TablesManager.GetPrimaryIndex()->GetVersionedIndex()); - auto ev = std::make_unique(actualIndexInfo, indexChanges, Settings.CacheDataAfterCompaction); - const TString externalTaskId = indexChanges->GetTaskIdentifier(); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "compaction")("external_task_id", externalTaskId); +void TColumnShard::StartCompaction(const std::shared_ptr& guard) { + Counters.GetCSCounters().OnSetupCompaction(); + BackgroundController.ResetWaitingPriority(); - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - ResourceSubscribeActor, std::make_shared( - std::make_shared(std::move(ev), SelfId(), TabletID(), Counters.GetCompactionCounters(), GetLastCompletedTx()), 0, indexChanges->CalcMemoryForUsage(), externalTaskId, CompactTaskSubscription)); + auto indexChanges = TablesManager.MutablePrimaryIndex().StartCompaction(DataLocksManager); + if (!indexChanges) { + LOG_S_DEBUG("Compaction not started: cannot prepare compaction at tablet " << TabletID()); + return; } + auto compaction = dynamic_pointer_cast(indexChanges); + compaction->SetQueueGuard(guard); + indexChanges->Start(*this); + + auto actualIndexInfo = std::make_shared(TablesManager.GetPrimaryIndex()->GetVersionedIndex()); + auto ev = std::make_unique(actualIndexInfo, indexChanges, Settings.CacheDataAfterCompaction); + const TString externalTaskId = indexChanges->GetTaskIdentifier(); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "compaction")("external_task_id", externalTaskId); + + NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( + ResourceSubscribeActor, std::make_shared( + std::make_shared( + std::move(ev), SelfId(), TabletID(), Counters.GetCompactionCounters(), GetLastCompletedTx()), + 0, indexChanges->CalcMemoryForUsage(), externalTaskId, CompactTaskSubscription)); + LOG_S_DEBUG("ActiveCompactions: " << BackgroundController.GetCompactionsCount() << " at tablet " << TabletID()); } @@ -847,6 +888,10 @@ void TColumnShard::SetupGC() { } } +void TColumnShard::Handle(TEvPrivate::TEvStartCompaction::TPtr& ev, const TActorContext& /*ctx*/) { + StartCompaction(ev->Get()->GetGuard()); +} + void TColumnShard::Handle(TEvPrivate::TEvGarbageCollectionFinished::TPtr& ev, const TActorContext& ctx) { Execute(new TTxGarbageCollectionFinished(this, ev->Get()->Action), ctx); } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 16ceed2681cc..74cd2fd8796b 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -217,6 +217,7 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void Handle(TEvMediatorTimecast::TEvRegisterTabletResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvMediatorTimecast::TEvNotifyPlanStep::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvStartCompaction::TPtr& ev, const TActorContext& ctx); void Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvScanStats::TPtr& ev, const TActorContext& ctx); @@ -376,6 +377,7 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa HFunc(TEvTxProcessing::TEvPlanStep, Handle); HFunc(TEvColumnShard::TEvWrite, Handle); HFunc(TEvPrivate::TEvWriteBlobsResult, Handle); + HFunc(TEvPrivate::TEvStartCompaction, Handle); HFunc(NPrivateEvents::NWrite::TEvWritePortionResult, Handle); HFunc(TEvMediatorTimecast::TEvRegisterTabletResult, Handle); @@ -430,6 +432,8 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa std::shared_ptr BackgroundSessionsManager; std::shared_ptr DataLocksManager; + ui64 PrioritizationClientId = 0; + using TSchemaPreset = TSchemaPreset; using TTableInfo = TTableInfo; @@ -535,7 +539,9 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void StartIndexTask(std::vector&& dataToIndex, const i64 bytesToIndex); void SetupIndexation(); - void SetupCompaction(); + void SetupCompaction(const std::set& pathIds); + void StartCompaction(const std::shared_ptr& guard); + bool SetupTtl(const THashMap& pathTtls = {}); void SetupCleanupPortions(); void SetupCleanupTables(); @@ -566,6 +572,10 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa return NOlap::TSnapshot(LastPlannedStep, LastPlannedTxId); } + NOlap::TSnapshot GetCurrentSnapshotForInternalModification() const { + return NOlap::TSnapshot::MaxForPlanStep(GetOutdatedStep()); + } + const std::shared_ptr& GetSharingSessionsManager() const { return SharingSessionsManager; } diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 2f7c887a4367..a90647ed1e2c 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace NKikimr::NOlap::NReader { class IApplyAction; @@ -55,12 +56,23 @@ struct TEvPrivate { EvTaskProcessedResult, EvPingSnapshotsUsage, EvWritePortionResult, + EvStartCompaction, EvEnd }; static_assert(EvEnd < EventSpaceEnd(TEvents::ES_PRIVATE), "expect EvEnd < EventSpaceEnd(TEvents::ES_PRIVATE)"); + class TEvStartCompaction: public NActors::TEventLocal { + private: + YDB_READONLY_DEF(std::shared_ptr, Guard); + + public: + TEvStartCompaction(const std::shared_ptr& g) + : Guard(g) { + } + }; + class TEvTaskProcessedResult: public NActors::TEventLocal { private: TConclusion> Result; diff --git a/ydb/core/tx/columnshard/common/limits.h b/ydb/core/tx/columnshard/common/limits.h index b30432dfb2fd..bef0c657b0de 100644 --- a/ydb/core/tx/columnshard/common/limits.h +++ b/ydb/core/tx/columnshard/common/limits.h @@ -4,7 +4,7 @@ namespace NKikimr::NOlap { class TGlobalLimits { public: - static constexpr inline ui64 TxWriteLimitBytes = 256 * 1024 * 1024; + static constexpr inline ui64 TxWriteLimitBytes = 312 * 1024 * 1024; static constexpr inline ui64 TTLCompactionMemoryLimit = 1ULL << 30; static constexpr inline ui64 InsertCompactionMemoryLimit = 1ULL << 30; static constexpr inline ui64 GeneralCompactionMemoryLimit = 3ULL << 30; diff --git a/ydb/core/tx/columnshard/common/snapshot.cpp b/ydb/core/tx/columnshard/common/snapshot.cpp index eb6e62ccac0c..e0e873488985 100644 --- a/ydb/core/tx/columnshard/common/snapshot.cpp +++ b/ydb/core/tx/columnshard/common/snapshot.cpp @@ -43,4 +43,8 @@ NKikimr::NOlap::TSnapshot TSnapshot::MaxForPlanInstant(const TInstant planInstan return TSnapshot(planInstant.MilliSeconds(), ::Max()); } +NJson::TJsonValue TSnapshot::SerializeToJson() const { + return DebugJson(); +} + }; diff --git a/ydb/core/tx/columnshard/common/snapshot.h b/ydb/core/tx/columnshard/common/snapshot.h index 4bc99d268420..7f04203eea8a 100644 --- a/ydb/core/tx/columnshard/common/snapshot.h +++ b/ydb/core/tx/columnshard/common/snapshot.h @@ -26,6 +26,8 @@ class TSnapshot { , TxId(txId) { } + NJson::TJsonValue SerializeToJson() const; + constexpr TInstant GetPlanInstant() const noexcept { return TInstant::MilliSeconds(PlanStep); } diff --git a/ydb/core/tx/columnshard/common/volume.cpp b/ydb/core/tx/columnshard/common/volume.cpp new file mode 100644 index 000000000000..aecdaf1a5b42 --- /dev/null +++ b/ydb/core/tx/columnshard/common/volume.cpp @@ -0,0 +1,19 @@ +#include "volume.h" +#include + +namespace NKikimr::NOlap { + +TBlobsVolume TBlobsVolume::operator-(const TBlobsVolume& item) const { + AFL_VERIFY(item.BlobBytes <= BlobBytes); + AFL_VERIFY(item.RawBytes <= RawBytes); + return TBlobsVolume(BlobBytes - item.BlobBytes, RawBytes - item.RawBytes); +} + +void TBlobsVolume::operator-=(const TBlobsVolume& item) { + AFL_VERIFY(item.BlobBytes <= BlobBytes); + AFL_VERIFY(item.RawBytes <= RawBytes); + BlobBytes -= item.BlobBytes; + RawBytes -= item.RawBytes; +} + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/common/volume.h b/ydb/core/tx/columnshard/common/volume.h new file mode 100644 index 000000000000..dae887a480cb --- /dev/null +++ b/ydb/core/tx/columnshard/common/volume.h @@ -0,0 +1,40 @@ +#pragma once +#include + +#include + +namespace NKikimr::NOlap { +class TBlobsVolume { +private: + YDB_READONLY(ui64, BlobBytes, 0); + YDB_READONLY(ui64, RawBytes, 0); + +public: + TBlobsVolume(const ui64 blob, const ui64 raw) + : BlobBytes(blob) + , RawBytes(raw) { + } + + TBlobsVolume operator+(const TBlobsVolume& item) const { + return TBlobsVolume(BlobBytes + item.BlobBytes, RawBytes + item.RawBytes); + } + + void Clear() { + BlobBytes = 0; + RawBytes = 0; + } + + bool CheckWithMax(const TBlobsVolume& maxLimit) const { + return BlobBytes < maxLimit.BlobBytes && RawBytes < maxLimit.RawBytes; + } + + void operator+=(const TBlobsVolume& item) { + BlobBytes += item.BlobBytes; + RawBytes += item.RawBytes; + } + + TBlobsVolume operator-(const TBlobsVolume& item) const; + + void operator-=(const TBlobsVolume& item); +}; +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/common/ya.make b/ydb/core/tx/columnshard/common/ya.make index c7d8a27bf3ee..049f150dc259 100644 --- a/ydb/core/tx/columnshard/common/ya.make +++ b/ydb/core/tx/columnshard/common/ya.make @@ -8,6 +8,7 @@ SRCS( portion.cpp tablet_id.cpp blob.cpp + volume.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/counters/common/private.cpp b/ydb/core/tx/columnshard/counters/common/private.cpp index 560ba1ec11e5..db93c893e13d 100644 --- a/ydb/core/tx/columnshard/counters/common/private.cpp +++ b/ydb/core/tx/columnshard/counters/common/private.cpp @@ -12,7 +12,7 @@ class TRegularSignalBuilderActor: public NActors::TActorBootstrappedResendStatus(); - Schedule(TDuration::Seconds(5), new NActors::TEvents::TEvWakeup); + Schedule(TDuration::Seconds(13), new NActors::TEvents::TEvWakeup); } public: TRegularSignalBuilderActor(std::shared_ptr agent) @@ -23,7 +23,7 @@ class TRegularSignalBuilderActor: public NActors::TActorBootstrappedResendStatus(); - Schedule(TDuration::Seconds(5), new NActors::TEvents::TEvWakeup); + Schedule(TDuration::Seconds(13), new NActors::TEvents::TEvWakeup); Become(&TRegularSignalBuilderActor::StateMain); } diff --git a/ydb/core/tx/columnshard/counters/portions.cpp b/ydb/core/tx/columnshard/counters/portions.cpp new file mode 100644 index 000000000000..e7476b4ebb23 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/portions.cpp @@ -0,0 +1,50 @@ +#include "portions.h" +#include + +namespace NKikimr::NColumnShard { + +void TPortionCategoryCounters::AddPortion(const std::shared_ptr& p) { + RecordsCount->Add(p->NumRows()); + Count->Add(1); + BlobBytes->Add(p->GetTotalBlobBytes()); + RawBytes->Add(p->GetTotalRawBytes()); +} + +void TPortionCategoryCounters::RemovePortion(const std::shared_ptr& p) { + RecordsCount->Remove(p->NumRows()); + Count->Remove(1); + BlobBytes->Remove(p->GetTotalBlobBytes()); + RawBytes->Remove(p->GetTotalRawBytes()); +} + +} // namespace NKikimr::NColumnShard + +namespace NKikimr::NOlap { + +void TSimplePortionsGroupInfo::AddPortion(const std::shared_ptr& p) { + AFL_VERIFY(p); + AddPortion(*p); +} +void TSimplePortionsGroupInfo::AddPortion(const TPortionInfo& p) { + BlobBytes += p.GetTotalBlobBytes(); + RawBytes += p.GetTotalRawBytes(); + Count += 1; + RecordsCount += p.NumRows(); +} + +void TSimplePortionsGroupInfo::RemovePortion(const std::shared_ptr& p) { + AFL_VERIFY(p); + RemovePortion(*p); +} +void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) { + BlobBytes -= p.GetTotalBlobBytes(); + RawBytes -= p.GetTotalRawBytes(); + Count -= 1; + RecordsCount -= p.NumRows(); + AFL_VERIFY(RawBytes >= 0); + AFL_VERIFY(BlobBytes >= 0); + AFL_VERIFY(Count >= 0); + AFL_VERIFY(RecordsCount >= 0); +} + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/counters/portions.h b/ydb/core/tx/columnshard/counters/portions.h new file mode 100644 index 000000000000..9c9c8a4875a4 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/portions.h @@ -0,0 +1,128 @@ +#pragma once +#include "common/agent.h" +#include "common/client.h" +#include "common/owner.h" + +#include +#include + +namespace NKikimr::NOlap { +class TPortionInfo; + +class TSimplePortionsGroupInfo { +private: + YDB_READONLY(i64, BlobBytes, 0); + YDB_READONLY(i64, RawBytes, 0); + YDB_READONLY(i64, Count, 0); + YDB_READONLY(i64, RecordsCount, 0); + +public: + NJson::TJsonValue SerializeToJson() const { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("blob_bytes", BlobBytes); + result.InsertValue("raw_bytes", RawBytes); + result.InsertValue("count", Count); + result.InsertValue("records_count", RecordsCount); + return result; + } + + ui64 PredictPackedBlobBytes(const std::optional kff) const { + if (kff) { + return RawBytes * *kff; + } else { + return BlobBytes; + } + } + + TString DebugString() const { + return TStringBuilder() << "{blob_bytes=" << BlobBytes << ";raw_bytes=" << RawBytes << ";count=" << Count << ";records=" << RecordsCount + << "}"; + } + + TSimplePortionsGroupInfo operator+(const TSimplePortionsGroupInfo& item) const { + TSimplePortionsGroupInfo result; + result.BlobBytes = BlobBytes + item.BlobBytes; + result.RawBytes = RawBytes + item.RawBytes; + result.Count = Count + item.Count; + result.RecordsCount = RecordsCount + item.RecordsCount; + return result; + } + + void AddPortion(const std::shared_ptr& p); + void RemovePortion(const std::shared_ptr& p); + + void AddPortion(const TPortionInfo& p); + void RemovePortion(const TPortionInfo& p); +}; + +class TPortionGroupCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + NMonitoring::TDynamicCounters::TCounterPtr Count; + NMonitoring::TDynamicCounters::TCounterPtr RawBytes; + NMonitoring::TDynamicCounters::TCounterPtr BlobBytes; + +public: + TPortionGroupCounters(const TString& kind, const NColumnShard::TCommonCountersOwner& baseOwner) + : TBase(baseOwner, "kind", kind) { + Count = TBase::GetDeriviative("Portions/Count"); + RawBytes = TBase::GetDeriviative("Portions/Raw/Bytes"); + BlobBytes = TBase::GetDeriviative("Portions/Blob/Bytes"); + } + + void OnData(const i64 portionsCount, const i64 portionBlobBytes, const i64 portionRawBytes) { + Count->Add(portionsCount); + RawBytes->Add(portionRawBytes); + BlobBytes->Add(portionBlobBytes); + } + + void OnData(const TSimplePortionsGroupInfo& group) { + Count->Add(group.GetCount()); + RawBytes->Add(group.GetRawBytes()); + BlobBytes->Add(group.GetBlobBytes()); + } +}; + +} // namespace NKikimr::NOlap + +namespace NKikimr::NColumnShard { + +class TPortionCategoryCounterAgents: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + +public: + const std::shared_ptr RecordsCount; + const std::shared_ptr Count; + const std::shared_ptr BlobBytes; + const std::shared_ptr RawBytes; + TPortionCategoryCounterAgents(TCommonCountersOwner& base, const TString& categoryName) + : TBase(base, "category", categoryName) + , RecordsCount(TBase::GetValueAutoAggregations("ByGranule/Portions/RecordsCount")) + , Count(TBase::GetValueAutoAggregations("ByGranule/Portions/Count")) + , BlobBytes(TBase::GetValueAutoAggregations("ByGranule/Portions/Blob/Bytes")) + , RawBytes(TBase::GetValueAutoAggregations("ByGranule/Portions/Raw/Bytes")) { + } +}; + +class TPortionCategoryCounters { +private: + std::shared_ptr RecordsCount; + std::shared_ptr Count; + std::shared_ptr BlobBytes; + std::shared_ptr RawBytes; + +public: + TPortionCategoryCounters(TPortionCategoryCounterAgents& agents) { + RecordsCount = agents.RecordsCount->GetClient(); + Count = agents.Count->GetClient(); + BlobBytes = agents.BlobBytes->GetClient(); + RawBytes = agents.RawBytes->GetClient(); + } + + void AddPortion(const std::shared_ptr& p); + + void RemovePortion(const std::shared_ptr& p); +}; + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/counters/ya.make b/ydb/core/tx/columnshard/counters/ya.make index 8707d6080e30..d11886716d09 100644 --- a/ydb/core/tx/columnshard/counters/ya.make +++ b/ydb/core/tx/columnshard/counters/ya.make @@ -13,6 +13,7 @@ SRCS( req_tracer.cpp scan.cpp splitter.cpp + portions.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/data_locks/locks/list.h b/ydb/core/tx/columnshard/data_locks/locks/list.h index 512386e985b1..b5d46c0a123c 100644 --- a/ydb/core/tx/columnshard/data_locks/locks/list.h +++ b/ydb/core/tx/columnshard/data_locks/locks/list.h @@ -1,7 +1,7 @@ #pragma once #include "abstract.h" #include -#include +#include namespace NKikimr::NOlap::NDataLocks { @@ -63,6 +63,14 @@ class TListPortionsLock: public ILock { Granules.emplace(address.GetPathId()); } } + + TListPortionsLock(const TString& lockName, const THashSet& portions, const bool readOnly = false) + : TBase(lockName, readOnly) { + for (auto&& address : portions) { + Portions.emplace(address); + Granules.emplace(address.GetPathId()); + } + } }; class TListTablesLock: public ILock { diff --git a/ydb/core/tx/columnshard/data_locks/locks/snapshot.h b/ydb/core/tx/columnshard/data_locks/locks/snapshot.h index 78edc72599f0..1e346f5f04b6 100644 --- a/ydb/core/tx/columnshard/data_locks/locks/snapshot.h +++ b/ydb/core/tx/columnshard/data_locks/locks/snapshot.h @@ -1,7 +1,7 @@ #pragma once #include "abstract.h" #include -#include +#include namespace NKikimr::NOlap::NDataLocks { diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index 721270ea63f3..db3f969460c6 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -199,6 +199,7 @@ class TColumnEngineChanges { EStage Stage = EStage::Created; std::shared_ptr LockGuard; TString AbortedReason; + const TString TaskIdentifier = TGUID::CreateTimebased().AsGuidString(); protected: virtual void DoDebugString(TStringOutput& out) const = 0; @@ -219,7 +220,6 @@ class TColumnEngineChanges { virtual NColumnShard::ECumulativeCounters GetCounterIndex(const bool isSuccess) const = 0; - const TString TaskIdentifier = TGUID::Create().AsGuidString(); virtual ui64 DoCalcMemoryForUsage() const = 0; virtual std::shared_ptr DoBuildDataLock() const = 0; std::shared_ptr BuildDataLock() const { @@ -288,7 +288,7 @@ class TColumnEngineChanges { std::vector> GetReadingActions() const { auto result = BlobsAction.GetReadingActions(); - Y_ABORT_UNLESS(result.size()); +// Y_ABORT_UNLESS(result.size()); return result; } virtual TString TypeString() const = 0; diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.cpp index e3e2a73955d6..664ef41d4415 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.cpp +++ b/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.cpp @@ -1,5 +1,12 @@ #include "compaction_info.h" +#include + namespace NKikimr::NOlap { +bool TPlanCompactionInfo::Finish() { + AFL_VERIFY(Count); + return --Count == 0; } + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h b/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h index 0e1ee6b72325..1020f2580ad8 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h @@ -11,8 +11,17 @@ class TGranuleMeta; class TPlanCompactionInfo { private: ui64 PathId = 0; - const TMonotonic StartTime = TMonotonic::Now(); + TMonotonic StartTime = TMonotonic::Now(); + ui32 Count = 0; + public: + void Start() { + StartTime = TMonotonic::Now(); + ++Count; + } + + bool Finish(); + TMonotonic GetStartTime() const { return StartTime; } diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index d0bd7e541a1a..7172cb3bb660 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -1,6 +1,6 @@ #include "compaction.h" #include -#include +#include #include #include @@ -30,7 +30,7 @@ void TCompactColumnEngineChanges::DoCompile(TFinalizationContext& context) { void TCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { TBase::DoStart(self); - Y_ABORT_UNLESS(SwitchedPortions.size()); +// Y_ABORT_UNLESS(SwitchedPortions.size()); THashMap> blobRanges; auto& index = self.GetIndexAs().GetVersionedIndex(); for (const auto& p : SwitchedPortions) { @@ -79,7 +79,7 @@ TCompactColumnEngineChanges::TCompactColumnEngineChanges(std::shared_ptrGetPathId() == GranuleMeta->GetPathId()); } - Y_ABORT_UNLESS(SwitchedPortions.size()); +// Y_ABORT_UNLESS(SwitchedPortions.size()); } TCompactColumnEngineChanges::~TCompactColumnEngineChanges() { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.h b/ydb/core/tx/columnshard/engines/changes/compaction.h index fc449e341459..9b45d8338b76 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction.h @@ -25,7 +25,8 @@ class TCompactColumnEngineChanges: public TChangesWithAppend { NeedGranuleStatusProvide = false; } virtual std::shared_ptr DoBuildDataLockImpl() const override { - return std::make_shared(TypeString() + "::" + GetTaskIdentifier(), SwitchedPortions); + const THashSet pathIds = { GranuleMeta->GetPathId() }; + return std::make_shared(TypeString() + "::" + GetTaskIdentifier(), pathIds); } public: diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp index 90b241d3cff7..2cc7c33a86d3 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp @@ -140,7 +140,7 @@ std::vector TMerger::Execute(const std::shared } batchSlices.emplace_back(portionColumns, schemaDetails, Context.Counters.SplitterCounters); } - NArrow::NSplitter::TSimilarPacker slicer(NSplitter::TSplitSettings().GetExpectedPortionSize()); + NArrow::NSplitter::TSimilarPacker slicer(PortionExpectedSize); auto packs = slicer.Split(batchSlices); ui32 recordIdx = 0; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.h b/ydb/core/tx/columnshard/engines/changes/compaction/merger.h index 9c84799fe8ad..6b7de3217638 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merger.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.h @@ -11,6 +11,7 @@ namespace NKikimr::NOlap::NCompaction { class TMerger { private: YDB_ACCESSOR(bool, OptimizationWritingPackMode, false); + YDB_ACCESSOR(ui64, PortionExpectedSize, 1.5 * (1 << 20)); std::vector> Batches; std::vector> Filters; const TConstructionContext& Context; diff --git a/ydb/core/tx/columnshard/engines/changes/counters/general.h b/ydb/core/tx/columnshard/engines/changes/counters/general.h index 11c038122e93..98deee42a6af 100644 --- a/ydb/core/tx/columnshard/engines/changes/counters/general.h +++ b/ydb/core/tx/columnshard/engines/changes/counters/general.h @@ -1,5 +1,9 @@ #pragma once #include +#include + +#include + #include #include @@ -12,42 +16,77 @@ class TGeneralCompactionCounters: public NColumnShard::TCommonCountersOwner { NMonitoring::TDynamicCounters::TCounterPtr FullBlobsAppendBytes; NMonitoring::TDynamicCounters::TCounterPtr SplittedBlobsAppendCount; NMonitoring::TDynamicCounters::TCounterPtr SplittedBlobsAppendBytes; - NMonitoring::TDynamicCounters::TCounterPtr RepackPortionsCount; - NMonitoring::TDynamicCounters::TCounterPtr RepackPortionsBytes; - NMonitoring::TDynamicCounters::TCounterPtr RepackInsertedPortionsBytes; - NMonitoring::TDynamicCounters::TCounterPtr RepackCompactedPortionsBytes; - NMonitoring::TDynamicCounters::TCounterPtr RepackOtherPortionsBytes; - NMonitoring::THistogramPtr HistogramRepackPortionsBytes; + + TPortionGroupCounters RepackPortions; + TPortionGroupCounters RepackInsertedPortions; + TPortionGroupCounters RepackCompactedPortions; + THashMap RepackPortionsFromLevel; + THashMap RepackPortionsToLevel; + THashMap MovePortionsFromLevel; + THashMap MovePortionsToLevel; + NMonitoring::THistogramPtr HistogramRepackPortionsRawBytes; + NMonitoring::THistogramPtr HistogramRepackPortionsBlobBytes; NMonitoring::THistogramPtr HistogramRepackPortionsCount; + public: TGeneralCompactionCounters() : TBase("GeneralCompaction") - { + , RepackPortions("ALL", CreateSubGroup("action", "repack")) + , RepackInsertedPortions("INSERTED", CreateSubGroup("action", "repack")) + , RepackCompactedPortions("COMPACTED", CreateSubGroup("action", "repack")) { + for (ui32 i = 0; i < 10; ++i) { + RepackPortionsFromLevel.emplace( + i, TPortionGroupCounters("level=" + ::ToString(i), CreateSubGroup("action", "repack").CreateSubGroup("direction", "from"))); + RepackPortionsToLevel.emplace( + i, TPortionGroupCounters("level=" + ::ToString(i), CreateSubGroup("action", "repack").CreateSubGroup("direction", "to"))); + MovePortionsFromLevel.emplace( + i, TPortionGroupCounters("level=" + ::ToString(i), CreateSubGroup("action", "move").CreateSubGroup("direction", "from"))); + MovePortionsToLevel.emplace( + i, TPortionGroupCounters("level=" + ::ToString(i), CreateSubGroup("action", "move").CreateSubGroup("direction", "to"))); + } FullBlobsAppendCount = TBase::GetDeriviative("FullBlobsAppend/Count"); FullBlobsAppendBytes = TBase::GetDeriviative("FullBlobsAppend/Bytes"); SplittedBlobsAppendCount = TBase::GetDeriviative("SplittedBlobsAppend/Count"); SplittedBlobsAppendBytes = TBase::GetDeriviative("SplittedBlobsAppend/Bytes"); - RepackPortionsCount = TBase::GetDeriviative("RepackPortions/Count"); - RepackPortionsBytes = TBase::GetDeriviative("RepackPortions/Bytes"); - HistogramRepackPortionsBytes = TBase::GetHistogram("RepackPortions/Bytes", NMonitoring::ExponentialHistogram(18, 2, 256 * 1024)); - HistogramRepackPortionsCount = TBase::GetHistogram("RepackPortions/Count", NMonitoring::ExponentialHistogram(15, 2, 4)); + HistogramRepackPortionsRawBytes = TBase::GetHistogram("RepackPortions/Raw/Bytes", NMonitoring::ExponentialHistogram(18, 2, 256 * 1024)); + HistogramRepackPortionsBlobBytes = + TBase::GetHistogram("RepackPortions/Blob/Bytes", NMonitoring::ExponentialHistogram(18, 2, 256 * 1024)); + HistogramRepackPortionsCount = TBase::GetHistogram("RepackPortions/Count", NMonitoring::LinearHistogram(15, 10, 16)); + } + + static void OnRepackPortions(const TSimplePortionsGroupInfo& portions) { + Singleton()->RepackPortions.OnData(portions); + Singleton()->HistogramRepackPortionsCount->Collect(portions.GetCount()); + Singleton()->HistogramRepackPortionsBlobBytes->Collect(portions.GetBlobBytes()); + Singleton()->HistogramRepackPortionsRawBytes->Collect(portions.GetRawBytes()); + } + + static void OnRepackPortionsByLevel(const THashMap& portions, const ui32 targetLevelIdx) { + for (auto&& i : portions) { + auto& counters = (i.first == targetLevelIdx) ? Singleton()->RepackPortionsToLevel + : Singleton()->RepackPortionsFromLevel; + auto it = counters.find(i.first); + AFL_VERIFY(it != counters.end()); + it->second.OnData(i.second); + } + } - RepackInsertedPortionsBytes = TBase::GetDeriviative("RepackInsertedPortions/Bytes"); - RepackCompactedPortionsBytes = TBase::GetDeriviative("RepackCompactedPortions/Bytes"); - RepackOtherPortionsBytes = TBase::GetDeriviative("RepackOtherPortions/Bytes"); + static void OnMovePortionsByLevel(const THashMap& portions, const ui32 targetLevelIdx) { + for (auto&& i : portions) { + auto& counters = (i.first == targetLevelIdx) ? Singleton()->MovePortionsToLevel + : Singleton()->MovePortionsFromLevel; + auto it = counters.find(i.first); + AFL_VERIFY(it != counters.end()); + it->second.OnData(i.second); + } } - static void OnRepackPortions(const i64 portionsCount, const i64 portionBytes) { - Singleton()->RepackPortionsCount->Add(portionsCount); - Singleton()->RepackPortionsBytes->Add(portionBytes); - Singleton()->HistogramRepackPortionsCount->Collect(portionsCount); - Singleton()->HistogramRepackPortionsBytes->Collect(portionBytes); + static void OnRepackInsertedPortions(const TSimplePortionsGroupInfo& portions) { + Singleton()->RepackInsertedPortions.OnData(portions); } - static void OnPortionsKind(const i64 insertedBytes, const i64 compactedBytes, const i64 otherBytes) { - Singleton()->RepackInsertedPortionsBytes->Add(insertedBytes); - Singleton()->RepackCompactedPortionsBytes->Add(compactedBytes); - Singleton()->RepackOtherPortionsBytes->Add(otherBytes); + static void OnRepackCompactedPortions(const TSimplePortionsGroupInfo& portions) { + Singleton()->RepackCompactedPortions.OnData(portions); } static void OnSplittedBlobAppend(const i64 bytes) { @@ -61,4 +100,4 @@ class TGeneralCompactionCounters: public NColumnShard::TCommonCountersOwner { } }; -} +} // namespace NKikimr::NOlap::NChanges diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 380f6127b457..5fa023bc6b0c 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -1,11 +1,13 @@ #include "general_compaction.h" -#include "counters/general.h" #include "compaction/merger.h" +#include "counters/general.h" #include #include +#include +#include namespace NKikimr::NOlap::NCompaction { @@ -85,10 +87,13 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks( TConstructionContext& context, std::vector&& portions) noexcept { auto resultSchema = context.SchemaVersions.GetLastSchema(); auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); - + if (portions.empty()) { + return; + } std::shared_ptr stats = std::make_shared(); std::shared_ptr resultFiltered; NCompaction::TMerger merger(context, SaverContext); + merger.SetPortionExpectedSize(PortionExpectedSize); { std::set pkColumnIds; { @@ -138,7 +143,7 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks( for (auto&& i : portions) { auto blobsSchema = i.GetPortionInfo().GetSchema(context.SchemaVersions); - auto batch = i.RestoreBatch(*blobsSchema, *resultFiltered, seqDataColumnIds); + auto batch = i.RestoreBatch(*blobsSchema, *resultFiltered, seqDataColumnIds).DetachResult(); std::shared_ptr filter = BuildPortionFilter(shardingActual, batch, i.GetPortionInfo(), usedPortionIds, resultFiltered); merger.AddBatch(batch, filter); @@ -157,24 +162,25 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks( } TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept { - i64 portionsSize = 0; - i64 portionsCount = 0; - i64 insertedPortionsSize = 0; - i64 compactedPortionsSize = 0; - i64 otherPortionsSize = 0; + TSimplePortionsGroupInfo insertedPortions; + TSimplePortionsGroupInfo compactedPortions; + THashMap portionGroups; for (auto&& i : SwitchedPortions) { + portionGroups[i.GetMeta().GetCompactionLevel()].AddPortion(i); if (i.GetMeta().GetProduced() == TPortionMeta::EProduced::INSERTED) { - insertedPortionsSize += i.GetTotalBlobBytes(); + insertedPortions.AddPortion(i); } else if (i.GetMeta().GetProduced() == TPortionMeta::EProduced::SPLIT_COMPACTED) { - compactedPortionsSize += i.GetTotalBlobBytes(); + compactedPortions.AddPortion(i); } else { - otherPortionsSize += i.GetTotalBlobBytes(); + AFL_VERIFY(false); } - portionsSize += i.GetTotalBlobBytes(); - ++portionsCount; } - NChanges::TGeneralCompactionCounters::OnPortionsKind(insertedPortionsSize, compactedPortionsSize, otherPortionsSize); - NChanges::TGeneralCompactionCounters::OnRepackPortions(portionsCount, portionsSize); + NChanges::TGeneralCompactionCounters::OnRepackPortions(insertedPortions + compactedPortions); + NChanges::TGeneralCompactionCounters::OnRepackInsertedPortions(insertedPortions); + NChanges::TGeneralCompactionCounters::OnRepackCompactedPortions(compactedPortions); + if (TargetCompactionLevel) { + NChanges::TGeneralCompactionCounters::OnRepackPortionsByLevel(portionGroups, *TargetCompactionLevel); + } { std::vector portions = @@ -211,6 +217,7 @@ void TGeneralCompactColumnEngineChanges::DoWriteIndexOnComplete(NColumnShard::TC } void TGeneralCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { + AFL_VERIFY(PrioritiesAllocationGuard); TBase::DoStart(self); auto& g = *GranuleMeta; self.Counters.GetCSCounters().OnSplitCompactionInfo( @@ -221,8 +228,7 @@ NColumnShard::ECumulativeCounters TGeneralCompactColumnEngineChanges::GetCounter return isSuccess ? NColumnShard::COUNTER_COMPACTION_SUCCESS : NColumnShard::COUNTER_COMPACTION_FAIL; } -void TGeneralCompactColumnEngineChanges::AddCheckPoint( - const NArrow::NMerger::TSortableBatchPosition& position, const bool include) { +void TGeneralCompactColumnEngineChanges::AddCheckPoint(const NArrow::NMerger::TSortableBatchPosition& position, const bool include) { CheckPoints.InsertPosition(position, include); } diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.h b/ydb/core/tx/columnshard/engines/changes/general_compaction.h index ab6f1e18684e..a1ca732899c2 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.h @@ -1,13 +1,17 @@ #pragma once #include "compaction.h" + #include #include +#include namespace NKikimr::NOlap::NCompaction { class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { private: + YDB_ACCESSOR(ui64, PortionExpectedSize, 1.5 * (1 << 20)); using TBase = TCompactColumnEngineChanges; + std::shared_ptr PrioritiesAllocationGuard; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; NArrow::NMerger::TIntervalPositions CheckPoints; void BuildAppendedPortionsByChunks(TConstructionContext& context, std::vector&& portions) noexcept; @@ -15,6 +19,7 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { std::shared_ptr BuildPortionFilter(const std::optional& shardingActual, const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage, const ISnapshotSchema::TPtr& resultSchema) const; + protected: virtual TConclusionStatus DoConstructBlobs(TConstructionContext& context) noexcept override; @@ -35,12 +40,17 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { } return result; } + public: + void SetQueueGuard(const std::shared_ptr& g) { + PrioritiesAllocationGuard = g; + } using TBase::TBase; class TMemoryPredictorSimplePolicy: public IMemoryPredictor { private: ui64 SumMemory = 0; + public: virtual ui64 AddPortion(const TPortionInfo& portionInfo) override { for (auto&& i : portionInfo.GetRecords()) { @@ -57,6 +67,7 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { ui64 SumMemoryFix = 0; ui32 PortionsCount = 0; THashMap MaxMemoryByColumnChunk; + public: virtual ui64 AddPortion(const TPortionInfo& portionInfo) override; }; @@ -64,10 +75,13 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { static std::shared_ptr BuildMemoryPredictor(); void AddCheckPoint(const NArrow::NMerger::TSortableBatchPosition& position, const bool include); + void SetCheckPoints(NArrow::NMerger::TIntervalPositions&& positions) { + CheckPoints = std::move(positions); + } virtual TString TypeString() const override { return StaticTypeName(); } }; -} +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.h b/ydb/core/tx/columnshard/engines/changes/indexation.h index 4c7f8602a6f5..fe4bd5ce9491 100644 --- a/ydb/core/tx/columnshard/engines/changes/indexation.h +++ b/ydb/core/tx/columnshard/engines/changes/indexation.h @@ -42,6 +42,7 @@ class TInsertColumnEngineChanges: public TChangesWithAppend { TInsertColumnEngineChanges(std::vector&& dataToIndex, const TSaverContext& saverContext) : TBase(saverContext, NBlobOperations::EConsumer::INDEXATION) , DataToIndex(std::move(dataToIndex)) { + SetTargetCompactionLevel(0); } const std::vector& GetDataToIndex() const { diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index b4de9dda9889..cfe48e7f59a0 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -1,5 +1,7 @@ #include "with_appended.h" +#include "counters/general.h" + #include #include #include @@ -19,7 +21,8 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, const auto predRemoveDroppedTable = [self](const TWritePortionInfoWithBlobsResult& item) { auto& portionInfo = item.GetPortionResult(); if (!!self && !self->TablesManager.HasTable(portionInfo.GetPathId(), false)) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_inserted_data")("reason", "table_removed")("path_id", portionInfo.GetPathId()); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_inserted_data")("reason", "table_removed")( + "path_id", portionInfo.GetPathId()); return true; } else { return false; @@ -31,12 +34,22 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); } + if (PortionsToMove.size()) { + for (auto&& [_, i] : PortionsToMove) { + const auto pred = [&](TPortionInfo& portionCopy) { + portionCopy.MutableMeta().ResetCompactionLevel(TargetCompactionLevel.value_or(0)); + }; + context.EngineLogs.GetGranuleVerified(i->GetPathId()).ModifyPortionOnExecute(*context.DB, i, pred); + } + } } void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) { if (self) { + TStringBuilder sb; for (auto& portionBuilder : AppendedPortions) { auto& portionInfo = portionBuilder.GetPortionResult(); + sb << portionInfo.GetPortionId() << ","; switch (portionInfo.GetMeta().Produced) { case NOlap::TPortionMeta::EProduced::UNSPECIFIED: Y_ABORT_UNLESS(false); // unexpected @@ -57,6 +70,7 @@ void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self break; } } + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("portions", sb)("task_id", GetTaskIdentifier()); self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_PORTIONS_DEACTIVATED, PortionsToRemove.size()); THashSet blobsDeactivated; @@ -72,11 +86,25 @@ void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_BYTES_DEACTIVATED, blobId.BlobSize()); } } + if (PortionsToMove.size()) { + THashMap portionGroups; + for (auto&& [_, i] : PortionsToMove) { + portionGroups[i->GetMeta().GetCompactionLevel()].AddPortion(i); + } + NChanges::TGeneralCompactionCounters::OnMovePortionsByLevel(portionGroups, TargetCompactionLevel.value_or(0)); + for (auto&& [_, i] : PortionsToMove) { + const auto pred = [&](const std::shared_ptr& portion) { + portion->MutableMeta().ResetCompactionLevel(TargetCompactionLevel.value_or(0)); + }; + context.EngineLogs.MutableGranuleVerified(i->GetPathId()).ModifyPortionOnComplete(i, pred); + } + } { auto g = context.EngineLogs.GranulesStorage->GetStats()->StartPackModification(); for (auto& [_, portionInfo] : PortionsToRemove) { context.EngineLogs.AddCleanupPortion(portionInfo); - const TPortionInfo& oldInfo = context.EngineLogs.GetGranuleVerified(portionInfo.GetPathId()).GetPortionVerified(portionInfo.GetPortion()); + const TPortionInfo& oldInfo = + context.EngineLogs.GetGranuleVerified(portionInfo.GetPathId()).GetPortionVerified(portionInfo.GetPortion()); context.EngineLogs.UpsertPortion(portionInfo, &oldInfo); } for (auto& portionBuilder : AppendedPortions) { @@ -88,6 +116,7 @@ void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self void TChangesWithAppend::DoCompile(TFinalizationContext& context) { for (auto&& i : AppendedPortions) { i.GetPortionConstructor().SetPortionId(context.NextPortionId()); + i.GetPortionConstructor().MutableMeta().SetCompactionLevel(TargetCompactionLevel.value_or(0)); } for (auto& [_, portionInfo] : PortionsToRemove) { portionInfo.SetRemoveSnapshot(context.GetSnapshot()); @@ -95,8 +124,11 @@ void TChangesWithAppend::DoCompile(TFinalizationContext& context) { } void TChangesWithAppend::DoOnAfterCompile() { - for (auto&& i : AppendedPortions) { - i.FinalizePortionConstructor(); + if (AppendedPortions.size()) { + for (auto&& i : AppendedPortions) { + i.GetPortionConstructor().MutableMeta().SetCompactionLevel(TargetCompactionLevel.value_or(0)); + i.FinalizePortionConstructor(); + } } } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.h b/ydb/core/tx/columnshard/engines/changes/with_appended.h index e35dfbbe4acc..e2beda084c3c 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.h +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.h @@ -10,7 +10,10 @@ class TChangesWithAppend: public TColumnEngineChanges { private: using TBase = TColumnEngineChanges; THashMap PortionsToRemove; + THashMap> PortionsToMove; + protected: + std::optional TargetCompactionLevel; TSaverContext SaverContext; virtual void DoCompile(TFinalizationContext& context) override; virtual void DoOnAfterCompile() override; @@ -19,18 +22,25 @@ class TChangesWithAppend: public TColumnEngineChanges { virtual void DoStart(NColumnShard::TColumnShard& self) override; virtual void DoDebugString(TStringOutput& out) const override { - out << "remove=" << PortionsToRemove.size() << ";append=" << AppendedPortions.size() << ";"; + out << "remove=" << PortionsToRemove.size() << ";append=" << AppendedPortions.size() << ";move=" << PortionsToMove.size(); } virtual std::shared_ptr DoBuildDataLockImpl() const = 0; virtual std::shared_ptr DoBuildDataLock() const override final { auto actLock = DoBuildDataLockImpl(); + THashSet portions; + for (auto&& i : PortionsToRemove) { + AFL_VERIFY(portions.emplace(i.first).second); + } + for (auto&& i : PortionsToMove) { + AFL_VERIFY(portions.emplace(i.first).second); + } if (actLock) { - auto selfLock = std::make_shared(TypeString() + "::" + GetTaskIdentifier() + "::REMOVE", PortionsToRemove); + auto selfLock = std::make_shared(TypeString() + "::" + GetTaskIdentifier() + "::REMOVE/MOVE", portions); return std::make_shared(TypeString() + "::" + GetTaskIdentifier(), std::vector>({actLock, selfLock})); } else { - auto selfLock = std::make_shared(TypeString() + "::" + GetTaskIdentifier(), PortionsToRemove); + auto selfLock = std::make_shared(TypeString() + "::" + GetTaskIdentifier(), portions); return selfLock; } } @@ -42,6 +52,13 @@ class TChangesWithAppend: public TColumnEngineChanges { } + void AddMovePortions(const std::vector>& portions) { + for (auto&& i : portions) { + AFL_VERIFY(i); + AFL_VERIFY(PortionsToMove.emplace(i->GetAddress(), i).second)("portion_id", i->GetPortionId()); + } + } + const THashMap& GetPortionsToRemove() const { return PortionsToRemove; } @@ -54,6 +71,10 @@ class TChangesWithAppend: public TColumnEngineChanges { return PortionsToRemove.size(); } + void SetTargetCompactionLevel(const ui64 level) { + TargetCompactionLevel = level; + } + void AddPortionToRemove(const TPortionInfo& info) { AFL_VERIFY(!info.HasRemoveSnapshot()); AFL_VERIFY(PortionsToRemove.emplace(info.GetAddress(), info).second); @@ -72,4 +93,4 @@ class TChangesWithAppend: public TColumnEngineChanges { } }; -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index cef8b3442f2d..88b9b2f9f324 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -283,7 +283,10 @@ class IColumnEngine { ui64 pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter, const bool withUncommitted) const = 0; virtual std::shared_ptr StartInsert(std::vector&& dataToIndex) noexcept = 0; virtual std::shared_ptr StartCompaction(const std::shared_ptr& dataLocksManager) noexcept = 0; - virtual std::shared_ptr StartCleanupPortions(const TSnapshot& snapshot, const THashSet& pathsToDrop, const std::shared_ptr& dataLocksManager) noexcept = 0; + virtual ui64 GetCompactionPriority( + const std::shared_ptr& dataLocksManager, const std::set& pathIds, const std::optional waitingPriority) noexcept = 0; + virtual std::shared_ptr StartCleanupPortions(const TSnapshot& snapshot, + const THashSet& pathsToDrop, const std::shared_ptr& dataLocksManager) noexcept = 0; virtual std::shared_ptr StartCleanupTables(const THashSet& pathsToDrop) noexcept = 0; virtual std::vector> StartTtl(const THashMap& pathEviction, const std::shared_ptr& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0; virtual bool ApplyChangesOnTxCreate(std::shared_ptr changes, const TSnapshot& snapshot) noexcept = 0; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 07d066b00288..eff47a91c008 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -299,6 +299,16 @@ std::shared_ptr TColumnEngineForLogs::StartInsert(st return changes; } +ui64 TColumnEngineForLogs::GetCompactionPriority(const std::shared_ptr& dataLocksManager, const std::set& pathIds, + const std::optional waitingPriority) noexcept { + auto priority = GranulesStorage->GetCompactionPriority(dataLocksManager, pathIds, waitingPriority); + if (!priority) { + return 0; + } else { + return priority->GetGeneralPriority(); + } +} + std::shared_ptr TColumnEngineForLogs::StartCompaction(const std::shared_ptr& dataLocksManager) noexcept { AFL_VERIFY(dataLocksManager); auto granule = GranulesStorage->GetGranuleForCompaction(dataLocksManager); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 29d88384078c..fc66cb4da1bc 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -5,8 +5,8 @@ #include "changes/actualization/controller/controller.h" #include "scheme/tier_info.h" -#include "storage/granule.h" -#include "storage/storage.h" +#include "storage/granule/granule.h" +#include "storage/granule/storage.h" #include #include @@ -113,6 +113,8 @@ class TColumnEngineForLogs: public IColumnEngine { } std::shared_ptr StartInsert(std::vector&& dataToIndex) noexcept override; + ui64 GetCompactionPriority(const std::shared_ptr& dataLocksManager, const std::set& pathIds, + const std::optional waitingPriority) noexcept override; std::shared_ptr StartCompaction(const std::shared_ptr& dataLocksManager) noexcept override; std::shared_ptr StartCleanupPortions(const TSnapshot& snapshot, const THashSet& pathsToDrop, const std::shared_ptr& dataLocksManager) noexcept override; diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.cpp b/ydb/core/tx/columnshard/engines/portions/constructor.cpp index 7f76fc10749c..5125d60f292c 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor.cpp @@ -10,6 +10,8 @@ namespace NKikimr::NOlap { TPortionInfo TPortionInfoConstructor::Build(const bool needChunksNormalization) { + AFL_VERIFY(!Constructed); + Constructed = true; TPortionInfo result(MetaConstructor.Build()); AFL_VERIFY(PathId); result.PathId = PathId; @@ -80,9 +82,10 @@ TPortionInfo TPortionInfoConstructor::Build(const bool needChunksNormalization) AFL_VERIFY(itBlobIdx == BlobIdxs.end()); } - result.Indexes = Indexes; - result.Records = Records; - result.BlobIds = BlobIds; + result.Indexes = std::move(Indexes); + result.Records = std::move(Records); + result.BlobIds = std::move(BlobIds); + result.Precalculate(); return result; } diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.h b/ydb/core/tx/columnshard/engines/portions/constructor.h index 4fc28abf9abe..e86db08d493a 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor.h @@ -15,6 +15,7 @@ class TGranuleShardingInfo; class TPortionInfoConstructor { private: + bool Constructed = false; YDB_ACCESSOR(ui64, PathId, 0); std::optional PortionId; @@ -73,7 +74,7 @@ class TPortionInfoConstructor { void AddMetadata(const ISnapshotSchema& snapshotSchema, const std::shared_ptr& batch); - void AddMetadata(const ISnapshotSchema& snapshotSchema, const ui32 deletionsCount, const NArrow::TFirstLastSpecialKeys& firstLastRecords, const NArrow::TMinMaxSpecialKeys& minMaxSpecial) { + void AddMetadata(const ISnapshotSchema& snapshotSchema, const ui32 deletionsCount, const NArrow::TFirstLastSpecialKeys& firstLastRecords, const std::optional& minMaxSpecial) { MetaConstructor.FillMetaInfo(firstLastRecords, deletionsCount, minMaxSpecial, snapshotSchema.GetIndexInfo()); } diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp index fa21c6feeeb6..948a199c960f 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp @@ -4,22 +4,25 @@ namespace NKikimr::NOlap { -void TPortionMetaConstructor::FillMetaInfo(const NArrow::TFirstLastSpecialKeys& primaryKeys, const ui32 deletionsCount, const NArrow::TMinMaxSpecialKeys& snapshotKeys, const TIndexInfo& indexInfo) { +void TPortionMetaConstructor::FillMetaInfo(const NArrow::TFirstLastSpecialKeys& primaryKeys, const ui32 deletionsCount, const std::optional& snapshotKeys, const TIndexInfo& indexInfo) { AFL_VERIFY(!FirstAndLastPK); FirstAndLastPK = *primaryKeys.BuildAccordingToSchemaVerified(indexInfo.GetReplaceKey()); AFL_VERIFY(!RecordSnapshotMin); AFL_VERIFY(!RecordSnapshotMax); DeletionsCount = deletionsCount; - { - auto cPlanStep = snapshotKeys.GetBatch()->GetColumnByName(TIndexInfo::SPEC_COL_PLAN_STEP); - auto cTxId = snapshotKeys.GetBatch()->GetColumnByName(TIndexInfo::SPEC_COL_TX_ID); + if (snapshotKeys) { + auto cPlanStep = snapshotKeys->GetBatch()->GetColumnByName(TIndexInfo::SPEC_COL_PLAN_STEP); + auto cTxId = snapshotKeys->GetBatch()->GetColumnByName(TIndexInfo::SPEC_COL_TX_ID); Y_ABORT_UNLESS(cPlanStep && cTxId); Y_ABORT_UNLESS(cPlanStep->type_id() == arrow::UInt64Type::type_id); Y_ABORT_UNLESS(cTxId->type_id() == arrow::UInt64Type::type_id); const arrow::UInt64Array& cPlanStepArray = static_cast(*cPlanStep); const arrow::UInt64Array& cTxIdArray = static_cast(*cTxId); RecordSnapshotMin = TSnapshot(cPlanStepArray.GetView(0), cTxIdArray.GetView(0)); - RecordSnapshotMax = TSnapshot(cPlanStepArray.GetView(snapshotKeys.GetBatch()->num_rows() - 1), cTxIdArray.GetView(snapshotKeys.GetBatch()->num_rows() - 1)); + RecordSnapshotMax = TSnapshot(cPlanStepArray.GetView(snapshotKeys->GetBatch()->num_rows() - 1), cTxIdArray.GetView(snapshotKeys->GetBatch()->num_rows() - 1)); + } else { + RecordSnapshotMin = TSnapshot::Zero(); + RecordSnapshotMax = TSnapshot::Zero(); } } @@ -27,6 +30,7 @@ TPortionMetaConstructor::TPortionMetaConstructor(const TPortionMeta& meta) { FirstAndLastPK = meta.ReplaceKeyEdges; RecordSnapshotMin = meta.RecordSnapshotMin; RecordSnapshotMax = meta.RecordSnapshotMax; + CompactionLevel = meta.GetCompactionLevel(); DeletionsCount = meta.GetDeletionsCount(); TierName = meta.GetTierNameOptional(); if (meta.Produced != NPortion::EProduced::UNSPECIFIED) { @@ -38,10 +42,12 @@ TPortionMeta TPortionMetaConstructor::Build() { AFL_VERIFY(FirstAndLastPK); AFL_VERIFY(RecordSnapshotMin); AFL_VERIFY(RecordSnapshotMax); + AFL_VERIFY(CompactionLevel); TPortionMeta result(*FirstAndLastPK, *RecordSnapshotMin, *RecordSnapshotMax); if (TierName) { result.TierName = *TierName; } + result.CompactionLevel = *CompactionLevel; AFL_VERIFY(DeletionsCount); result.DeletionsCount = *DeletionsCount; AFL_VERIFY(Produced); @@ -62,6 +68,7 @@ bool TPortionMetaConstructor::LoadMetadata(const NKikimrTxColumnShard::TIndexPor } else { DeletionsCount = 0; } + CompactionLevel = portionMeta.GetCompactionLevel(); if (portionMeta.GetIsInserted()) { Produced = TPortionMeta::EProduced::INSERTED; } else if (portionMeta.GetIsCompacted()) { diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.h b/ydb/core/tx/columnshard/engines/portions/constructor_meta.h index 87b808a282fd..4c55771fdf4b 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_meta.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor_meta.h @@ -15,14 +15,19 @@ class TPortionMetaConstructor { std::optional RecordSnapshotMin; std::optional RecordSnapshotMax; std::optional Produced; + std::optional CompactionLevel; std::optional DeletionsCount; friend class TPortionInfoConstructor; - void FillMetaInfo(const NArrow::TFirstLastSpecialKeys& primaryKeys, const ui32 deletionsCount, const NArrow::TMinMaxSpecialKeys& snapshotKeys, const TIndexInfo& indexInfo); + void FillMetaInfo(const NArrow::TFirstLastSpecialKeys& primaryKeys, const ui32 deletionsCount, const std::optional& snapshotKeys, const TIndexInfo& indexInfo); public: TPortionMetaConstructor() = default; TPortionMetaConstructor(const TPortionMeta& meta); + void SetCompactionLevel(const ui64 level) { + CompactionLevel = level; + } + void SetTierName(const TString& tierName); void ResetTierName(const TString& tierName) { TierName.reset(); diff --git a/ydb/core/tx/columnshard/engines/portions/meta.cpp b/ydb/core/tx/columnshard/engines/portions/meta.cpp index 9d7e374ec8f1..7019b5770577 100644 --- a/ydb/core/tx/columnshard/engines/portions/meta.cpp +++ b/ydb/core/tx/columnshard/engines/portions/meta.cpp @@ -11,6 +11,7 @@ namespace NKikimr::NOlap { NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto() const { NKikimrTxColumnShard::TIndexPortionMeta portionMeta; portionMeta.SetTierName(TierName); + portionMeta.SetCompactionLevel(CompactionLevel); portionMeta.SetDeletionsCount(DeletionsCount); switch (Produced) { case TPortionMeta::EProduced::UNSPECIFIED: diff --git a/ydb/core/tx/columnshard/engines/portions/meta.h b/ydb/core/tx/columnshard/engines/portions/meta.h index c29dc431d5d3..e45156b5d529 100644 --- a/ydb/core/tx/columnshard/engines/portions/meta.h +++ b/ydb/core/tx/columnshard/engines/portions/meta.h @@ -16,6 +16,7 @@ struct TPortionMeta { NArrow::TFirstLastSpecialKeys ReplaceKeyEdges; // first and last PK rows YDB_READONLY_DEF(TString, TierName); YDB_READONLY(ui32, DeletionsCount, 0); + YDB_READONLY(ui32, CompactionLevel, 0); friend class TPortionMetaConstructor; friend class TPortionInfo; TPortionMeta(NArrow::TFirstLastSpecialKeys& pk, const TSnapshot& min, const TSnapshot& max) @@ -31,6 +32,14 @@ struct TPortionMeta { TSnapshot RecordSnapshotMax; public: + const NArrow::TFirstLastSpecialKeys& GetFirstLastPK() const { + return ReplaceKeyEdges; + } + + void ResetCompactionLevel(const ui32 level) { + CompactionLevel = level; + } + using EProduced = NPortion::EProduced; NArrow::TReplaceKey IndexKeyStart; diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 855faf00d936..3aea7cf48b22 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -50,22 +50,14 @@ ui64 TPortionInfo::GetColumnBlobBytes(const std::set& entityIds, const boo return sum; } -ui64 TPortionInfo::GetColumnRawBytes(const bool validation) const { - ui64 sum = 0; - const auto aggr = [&](const TColumnRecord& r) { - sum += r.GetMeta().GetRawBytes(); - }; - AggregateIndexChunksData(aggr, Records, nullptr, validation); - return sum; +ui64 TPortionInfo::GetColumnRawBytes() const { + AFL_VERIFY(Precalculated); + return PrecalculatedColumnRawBytes; } -ui64 TPortionInfo::GetColumnBlobBytes(const bool validation) const { - ui64 sum = 0; - const auto aggr = [&](const TColumnRecord& r) { - sum += r.GetBlobRange().GetSize(); - }; - AggregateIndexChunksData(aggr, Records, nullptr, validation); - return sum; +ui64 TPortionInfo::GetColumnBlobBytes() const { + AFL_VERIFY(Precalculated); + return PrecalculatedColumnBlobBytes; } ui64 TPortionInfo::GetIndexRawBytes(const std::set& entityIds, const bool validation) const { @@ -91,7 +83,8 @@ TString TPortionInfo::DebugString(const bool withDetails) const { sb << "(portion_id:" << Portion << ";" << "path_id:" << PathId << ";records_count:" << NumRows() << ";" "min_schema_snapshot:(" << MinSnapshotDeprecated.DebugString() << ");" - "schema_version:" << SchemaVersion.value_or(0) << ";"; + "schema_version:" << SchemaVersion.value_or(0) << ";" + "level:" << GetMeta().GetCompactionLevel() << ";"; if (withDetails) { sb << "records_snapshot_min:(" << RecordSnapshotMin().DebugString() << ");" << @@ -293,6 +286,7 @@ TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDat } Indexes.emplace_back(std::move(parse.DetachResult())); } + Precalculate(); return TConclusionStatus::Success(); } @@ -734,13 +728,30 @@ NKikimr::NOlap::NSplitter::TEntityGroups TPortionInfo::GetEntityGroupsByStorageI } } -std::shared_ptr TPortionInfo::TPreparedColumn::AssembleAccessor() const { +void TPortionInfo::Precalculate() { + AFL_VERIFY(!Precalculated); + Precalculated = true; + { + PrecalculatedColumnRawBytes = 0; + PrecalculatedColumnBlobBytes = 0; + const auto aggr = [&](const TColumnRecord& r) { + PrecalculatedColumnRawBytes += r.GetMeta().GetRawBytes(); + PrecalculatedColumnBlobBytes += r.BlobRange.GetSize(); + }; + AggregateIndexChunksData(aggr, Records, nullptr, true); + } +} + +TConclusion> TPortionInfo::TPreparedColumn::AssembleAccessor() const { Y_ABORT_UNLESS(!Blobs.empty()); NArrow::NAccessor::TCompositeChunkedArray::TBuilder builder(GetField()->type()); for (auto& blob : Blobs) { auto chunkedArray = blob.BuildRecordBatch(*Loader); - builder.AddChunk(chunkedArray); + if (chunkedArray.IsFail()) { + return chunkedArray; + } + builder.AddChunk(chunkedArray.DetachResult()); } return builder.Finish(); } @@ -776,7 +787,7 @@ NArrow::NAccessor::TDeserializeChunkedArray::TChunk TPortionInfo::TAssembleBlobI } } -std::shared_ptr TPortionInfo::TAssembleBlobInfo::BuildRecordBatch(const TColumnLoader& loader) const { +TConclusion> TPortionInfo::TAssembleBlobInfo::BuildRecordBatch(const TColumnLoader& loader) const { if (DefaultRowsCount) { Y_ABORT_UNLESS(!Data); if (NeedCache) { @@ -788,11 +799,11 @@ std::shared_ptr TPortionInfo::TAssembleBlobInf } } else { AFL_VERIFY(ExpectedRowsCount); - return loader.ApplyVerified(Data, *ExpectedRowsCount); + return loader.ApplyConclusion(Data, *ExpectedRowsCount); } } -std::shared_ptr TPortionInfo::TPreparedBatchData::AssembleToGeneralContainer( +TConclusion> TPortionInfo::TPreparedBatchData::AssembleToGeneralContainer( const std::set& sequentialColumnIds) const { std::vector> columns; std::vector> fields; @@ -801,7 +812,11 @@ std::shared_ptr TPortionInfo::TPreparedBatchData::Ass if (sequentialColumnIds.contains(i.GetColumnId())) { columns.emplace_back(i.AssembleForSeqAccess()); } else { - columns.emplace_back(i.AssembleAccessor()); + auto conclusion = i.AssembleAccessor(); + if (conclusion.IsFail()) { + return conclusion; + } + columns.emplace_back(conclusion.DetachResult()); } fields.emplace_back(i.GetField()); } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 09cc515fef90..22fb2ec1f9f6 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -61,6 +61,12 @@ class TPortionInfo { Optimized = 1 /* "optimized" */ }; private: + ui64 PrecalculatedColumnRawBytes = 0; + ui64 PrecalculatedColumnBlobBytes = 0; + bool Precalculated = false; + + void Precalculate(); + friend class TPortionInfoConstructor; TPortionInfo(TPortionMeta&& meta) : Meta(std::move(meta)) { @@ -127,6 +133,10 @@ class TPortionInfo { } } public: + ui32 GetCompactionLevel() const { + return GetMeta().GetCompactionLevel(); + } + ui64 GetMinMemoryForReadColumns(const std::optional>& columnIds) const; bool NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) const; @@ -593,10 +603,10 @@ class TPortionInfo { } ui64 GetColumnRawBytes(const std::set& columnIds, const bool validation = true) const; - ui64 GetColumnRawBytes(const bool validation = true) const; + ui64 GetColumnRawBytes() const; ui64 GetColumnBlobBytes(const std::set& columnIds, const bool validation = true) const; - ui64 GetColumnBlobBytes(const bool validation = true) const; + ui64 GetColumnBlobBytes() const; ui64 GetTotalBlobBytes() const noexcept { return GetIndexBlobBytes() + GetColumnBlobBytes(); @@ -656,7 +666,7 @@ class TPortionInfo { return DefaultRowsCount && !Data; } - std::shared_ptr BuildRecordBatch(const TColumnLoader& loader) const; + TConclusion> BuildRecordBatch(const TColumnLoader& loader) const; NArrow::NAccessor::TDeserializeChunkedArray::TChunk BuildDeserializeChunk(const std::shared_ptr& loader) const; }; @@ -684,7 +694,7 @@ class TPortionInfo { } std::shared_ptr AssembleForSeqAccess() const; - std::shared_ptr AssembleAccessor() const; + TConclusion> AssembleAccessor() const; }; class TPreparedBatchData { @@ -743,7 +753,7 @@ class TPortionInfo { , RowsCount(rowsCount) { } - std::shared_ptr AssembleToGeneralContainer(const std::set& sequentialColumnIds) const; + TConclusion> AssembleToGeneralContainer(const std::set& sequentialColumnIds) const; }; class TColumnAssemblingInfo { diff --git a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp index 67a070d11bb8..49be899b7e8d 100644 --- a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp +++ b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp @@ -13,7 +13,7 @@ void TReadPortionInfoWithBlobs::RestoreChunk(const std::shared_ptr TReadPortionInfoWithBlobs::RestoreBatch( +TConclusion> TReadPortionInfoWithBlobs::RestoreBatch( const ISnapshotSchema& data, const ISnapshotSchema& resultSchema, const std::set& seqColumns) const { THashMap blobs; for (auto&& i : PortionInfo.Records) { diff --git a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h index a9e24eb3c165..e2d25b08f0b7 100644 --- a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h +++ b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h @@ -38,7 +38,7 @@ class TReadPortionInfoWithBlobs: public TBasePortionInfoWithBlobs { static TReadPortionInfoWithBlobs RestorePortion(const TPortionInfo& portion, NBlobOperations::NRead::TCompositeReadBlobs& blobs, const TIndexInfo& indexInfo); - std::shared_ptr RestoreBatch(const ISnapshotSchema& data, const ISnapshotSchema& resultSchema, const std::set& seqColumns) const; + TConclusion> RestoreBatch(const ISnapshotSchema& data, const ISnapshotSchema& resultSchema, const std::set& seqColumns) const; static std::optional SyncPortion(TReadPortionInfoWithBlobs&& source, const ISnapshotSchema::TPtr& from, const ISnapshotSchema::TPtr& to, const TString& targetTier, const std::shared_ptr& storages, std::shared_ptr counters); diff --git a/ydb/core/tx/columnshard/engines/protos/portion_info.proto b/ydb/core/tx/columnshard/engines/protos/portion_info.proto index f7f38bb96ed7..b62f22790ced 100644 --- a/ydb/core/tx/columnshard/engines/protos/portion_info.proto +++ b/ydb/core/tx/columnshard/engines/protos/portion_info.proto @@ -19,6 +19,7 @@ message TIndexPortionMeta { optional TSnapshot RecordSnapshotMin = 7; optional TSnapshot RecordSnapshotMax = 8; optional uint32 DeletionsCount = 10; + optional uint64 CompactionLevel = 11 [default = 0]; } message TIndexColumnMeta { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index dcb8935f3227..ab93985e6d1d 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -199,7 +199,7 @@ void TPortionDataSource::DoAssembleColumns(const std::shared_ptr& c } auto batch = Portion->PrepareForAssemble(*blobSchema, columns->GetFilteredSchemaVerified(), MutableStageData().MutableBlobs(), ss) - .AssembleToGeneralContainer(SequentialEntityIds); + .AssembleToGeneralContainer(SequentialEntityIds).DetachResult(); MutableStageData().AddBatch(batch); } diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h index 356dfc446ed3..5f18d8b9ece2 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include namespace NKikimr::NOlap::NReader::NSysView::NAbstract { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 5d3132b99e06..344d6f370493 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -11,7 +11,7 @@ void TStatsIterator::AppendStats(const std::vectorsecond.GetView(); - const bool activity = !portion.IsRemovedFor(ReadMetadata->GetRequestSnapshot()); + const bool activity = !portion.HasRemoveSnapshot(); static const TString ConstantEntityIsColumn = "COL"; static const arrow::util::string_view ConstantEntityIsColumnView = arrow::util::string_view(ConstantEntityIsColumn.data(), ConstantEntityIsColumn.size()); diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp index 83b2306a1bff..1cd56af82894 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp @@ -16,7 +16,7 @@ void TStatsIterator::AppendStats(const std::vector(*builders[6], portion.GetColumnBlobBytes()); NArrow::Append(*builders[7], portion.GetIndexBlobBytes()); NArrow::Append(*builders[8], portion.GetPortionId()); - NArrow::Append(*builders[9], !portion.IsRemovedFor(ReadMetadata->GetRequestSnapshot())); + NArrow::Append(*builders[9], !portion.HasRemoveSnapshot()); auto tierName = portion.GetTierNameDef(NBlobOperations::TGlobal::DefaultStorageId); NArrow::Append(*builders[10], arrow::util::string_view(tierName.data(), tierName.size())); @@ -33,6 +33,16 @@ void TStatsIterator::AppendStats(const std::vector(*builders[11], arrow::util::string_view(statInfo.data(), statInfo.size())); NArrow::Append(*builders[12], portion.HasRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized)); + NArrow::Append(*builders[13], portion.GetMeta().GetCompactionLevel()); + { + NJson::TJsonValue details = NJson::JSON_MAP; + details.InsertValue("snapshot_min", portion.RecordSnapshotMin().SerializeToJson()); + details.InsertValue("snapshot_max", portion.RecordSnapshotMax().SerializeToJson()); + details.InsertValue("primary_key_min", portion.IndexKeyStart().DebugString()); + details.InsertValue("primary_key_max", portion.IndexKeyEnd().DebugString()); + const auto detailsInfo = details.GetStringRobust(); + NArrow::Append(*builders[14], arrow::util::string_view(detailsInfo.data(), detailsInfo.size())); + } } ui32 TStatsIterator::PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const { diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp index 46418998ea4a..643840074487 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp @@ -8,8 +8,8 @@ namespace NKikimr::NOlap { -std::shared_ptr IIndexInfo::GetColumnLoaderVerified(const ui32 columnId) const { - auto result = GetColumnLoaderOptional(columnId); +const std::shared_ptr& IIndexInfo::GetColumnLoaderVerified(const ui32 columnId) const { + const auto& result = GetColumnLoaderOptional(columnId); AFL_VERIFY(result); return result; } diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h index 954363204f08..9d38cb333cb3 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h @@ -147,8 +147,8 @@ class IIndexInfo { static std::shared_ptr GetColumnFieldOptional(const ui32 columnId); static std::shared_ptr GetColumnFieldVerified(const ui32 columnId); - virtual std::shared_ptr GetColumnLoaderOptional(const ui32 columnId) const = 0; - std::shared_ptr GetColumnLoaderVerified(const ui32 columnId) const; + virtual const std::shared_ptr& GetColumnLoaderOptional(const ui32 columnId) const = 0; + const std::shared_ptr& GetColumnLoaderVerified(const ui32 columnId) const; static void NormalizeDeletionColumn(NArrow::TGeneralContainer& batch); diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 0569724bb7ba..41ebd89c53a8 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -141,10 +141,10 @@ TColumnSaver TIndexInfo::GetColumnSaver(const ui32 columnId) const { return GetColumnFeaturesVerified(columnId).GetColumnSaver(); } -std::shared_ptr TIndexInfo::GetColumnLoaderOptional(const ui32 columnId) const { +const std::shared_ptr& TIndexInfo::GetColumnLoaderOptional(const ui32 columnId) const { const auto& cFeatures = GetColumnFeaturesOptional(columnId); if (!cFeatures) { - return nullptr; + return Default>(); } else { return cFeatures->GetLoader(); } diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index 48910722fc71..24d6152ef793 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -306,7 +306,7 @@ struct TIndexInfo: public IIndexInfo { std::shared_ptr GetColumnSchema(const ui32 columnId) const; std::shared_ptr GetColumnsSchema(const std::set& columnIds) const; TColumnSaver GetColumnSaver(const ui32 columnId) const; - virtual std::shared_ptr GetColumnLoaderOptional(const ui32 columnId) const override; + virtual const std::shared_ptr& GetColumnLoaderOptional(const ui32 columnId) const override; std::optional GetColumnNameOptional(const ui32 columnId) const { auto f = GetColumnFieldOptional(columnId); if (!f) { diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp index 9b833f8ed1cb..b77d74ee690b 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp @@ -331,10 +331,10 @@ TConclusion ISnapshotSchema::PrepareForWrite(c TWritePortionInfoWithBlobsConstructor::BuildByBlobs(std::move(blobs), {}, pathId, GetVersion(), GetSnapshot(), storagesManager); NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(GetIndexInfo().GetReplaceKey())); - NArrow::TMinMaxSpecialKeys snapshotKeys(NArrow::MakeEmptyBatch(TIndexInfo::ArrowSchemaSnapshot(), 1), TIndexInfo::ArrowSchemaSnapshot()); const ui32 deletionsCount = (mType == NEvWrite::EModificationType::Delete) ? incomingBatch->num_rows() : 0; - constructor.GetPortionConstructor().AddMetadata(*this, deletionsCount, primaryKeys, snapshotKeys); + constructor.GetPortionConstructor().AddMetadata(*this, deletionsCount, primaryKeys, std::nullopt); constructor.GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); + constructor.GetPortionConstructor().MutableMeta().SetCompactionLevel(0); constructor.GetPortionConstructor().MutableMeta().UpdateRecordsMeta(NPortion::EProduced::INSERTED); return TWritePortionInfoWithBlobsResult(std::move(constructor)); } diff --git a/ydb/core/tx/columnshard/engines/storage/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule.cpp deleted file mode 100644 index c8c704bfa40d..000000000000 --- a/ydb/core/tx/columnshard/engines/storage/granule.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "granule.h" diff --git a/ydb/core/tx/columnshard/engines/storage/granule.h b/ydb/core/tx/columnshard/engines/storage/granule.h deleted file mode 100644 index 49587a09545e..000000000000 --- a/ydb/core/tx/columnshard/engines/storage/granule.h +++ /dev/null @@ -1,2 +0,0 @@ -#pragma once -#include "granule/granule.h" diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp index 668ac539e781..2580264831f9 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp @@ -4,8 +4,6 @@ #include #include #include -#include -#include #include @@ -93,22 +91,18 @@ void TGranuleMeta::OnBeforeChangePortion(const std::shared_ptr por void TGranuleMeta::OnCompactionFinished() { AllowInsertionFlag = false; - Y_ABORT_UNLESS(Activity.erase(EActivity::GeneralCompaction)); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnCompactionFinished")("info", DebugString()); Stats->UpdateGranuleInfo(*this); } void TGranuleMeta::OnCompactionFailed(const TString& reason) { AllowInsertionFlag = false; - Y_ABORT_UNLESS(Activity.erase(EActivity::GeneralCompaction)); AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "OnCompactionFailed")("reason", reason)("info", DebugString()); Stats->UpdateGranuleInfo(*this); } void TGranuleMeta::OnCompactionStarted() { AllowInsertionFlag = false; - Y_ABORT_UNLESS(Activity.empty()); - Activity.emplace(EActivity::GeneralCompaction); } void TGranuleMeta::RebuildAdditiveMetrics() const { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.h b/ydb/core/tx/columnshard/engines/storage/granule/granule.h index deca161e69f1..ad0f50b0336b 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.h @@ -132,11 +132,6 @@ class TGranuleAdditiveSummary { }; class TGranuleMeta: TNonCopyable { -public: - enum class EActivity { - GeneralCompaction - }; - private: TMonotonic ModificationLastTime = TMonotonic::Now(); THashMap> Portions; @@ -146,7 +141,6 @@ class TGranuleMeta: TNonCopyable { void RebuildHardMetrics() const; void RebuildAdditiveMetrics() const; - std::set Activity; mutable bool AllowInsertionFlag = false; const ui64 PathId; const NColumnShard::TGranuleDataCounters Counters; @@ -169,6 +163,37 @@ class TGranuleMeta: TNonCopyable { ActualizationIndex->RefreshTiering(tiering, context); } + TConclusionStatus IsInnerPortion(const std::shared_ptr& portion) const { + if (!portion) { + return TConclusionStatus::Fail("empty portion pointer"); + } + auto it = Portions.find(portion->GetPortionId()); + if (it == Portions.end()) { + return TConclusionStatus::Fail("portion id is incorrect: " + ::ToString(portion->GetPortionId())); + } + if (portion->GetPathId() != GetPathId()) { + return TConclusionStatus::Fail("portion path_id is incorrect: " + ::ToString(portion->GetPathId()) + " != " + ::ToString(GetPathId())); + } + return TConclusionStatus::Success(); + } + + template + void ModifyPortionOnExecute(NTable::TDatabase& db, const std::shared_ptr& portion, const TModifier& modifier) const { + IsInnerPortion(portion).Validate("modify portion on execute"); + auto copy = *portion; + modifier(copy); + TDbWrapper wrapper(db, nullptr); + copy.SaveToDatabase(wrapper, 0, true); + } + + template + void ModifyPortionOnComplete(const std::shared_ptr& portion, const TModifier& modifier) { + IsInnerPortion(portion).Validate("modify portion on complete"); + OnBeforeChangePortion(portion); + modifier(portion); + OnAfterChangePortion(portion, nullptr); + } + void InsertPortionOnExecute( NTabletFlatExecutor::TTransactionContext& txc, const std::shared_ptr& portion) const { AFL_VERIFY(!InsertedPortions.contains(portion->GetInsertWriteIdVerified())); @@ -289,10 +314,6 @@ class TGranuleMeta: TNonCopyable { return OptimizerPlanner->GetUsefulMetric(); } - bool IsLockedOptimizer(const std::shared_ptr& dataLocksManager) const { - return OptimizerPlanner->IsLocked(dataLocksManager); - } - void ActualizeOptimizer(const TInstant currentInstant, const TDuration recalcLag) const { if (OptimizerPlanner->GetActualizationInstant() + recalcLag < currentInstant) { OptimizerPlanner->Actualize(currentInstant); @@ -300,7 +321,7 @@ class TGranuleMeta: TNonCopyable { } bool IsErasable() const { - return Activity.empty() && Portions.empty(); + return Portions.empty(); } void OnCompactionStarted(); diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp b/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp index 18f0f7043ff2..32b72c5ee9f8 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp @@ -1,51 +1,91 @@ #include "storage.h" + #include namespace NKikimr::NOlap { -std::shared_ptr TGranulesStorage::GetGranuleForCompaction(const std::shared_ptr& dataLocksManager) const { +namespace { +class TGranuleOrdered { +private: + NStorageOptimizer::TOptimizationPriority Priority; + YDB_READONLY_DEF(std::shared_ptr, Granule); + +public: + const NStorageOptimizer::TOptimizationPriority& GetPriority() const { + return Priority; + } + + TGranuleOrdered(const NStorageOptimizer::TOptimizationPriority& priority, const std::shared_ptr& meta) + : Priority(priority) + , Granule(meta) + { + + } + + bool operator<(const TGranuleOrdered& item) const { + return Priority < item.Priority; + } +}; +} // namespace + +std::optional TGranulesStorage::GetCompactionPriority( + const std::shared_ptr& dataLocksManager, const std::set& pathIds, + const std::optional waitingPriority, std::shared_ptr* granuleResult) const { const TInstant now = HasAppData() ? AppDataVerified().TimeProvider->Now() : TInstant::Now(); - std::map> granulesSorted; - ui32 countChecker = 0; + std::vector granulesSorted; std::optional priorityChecker; + std::shared_ptr maxPriorityGranule; const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetCompactionActualizationLag(); - for (auto&& i : Tables) { -// NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("path_id", i.first); - i.second->ActualizeOptimizer(now, actualizationLag); - auto gPriority = i.second->GetCompactionPriority(); - if (gPriority.IsZero() || (priorityChecker && gPriority < *priorityChecker)) { - continue; + const auto actor = [&](const ui64 /*pathId*/, const std::shared_ptr& granule) { + // NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("path_id", i.first); + if (pathIds.empty()) { + granule->ActualizeOptimizer(now, actualizationLag); } - granulesSorted.emplace(gPriority, i.second); - if (++countChecker % 100 == 0) { - for (auto&& it = granulesSorted.rbegin(); it != granulesSorted.rend(); ++it) { - if (!it->second->IsLockedOptimizer(dataLocksManager)) { - priorityChecker = it->first; - break; - } - } + auto gPriority = granule->GetCompactionPriority(); + if (gPriority.IsZero() || (waitingPriority && gPriority.GetGeneralPriority() < *waitingPriority)) { + return; } - } - if (granulesSorted.empty()) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "no_granules"); - return nullptr; - } - for (auto&& it = granulesSorted.rbegin(); it != granulesSorted.rend(); ++it) { - if (priorityChecker && it->first < *priorityChecker) { - continue; + granulesSorted.emplace_back(gPriority, granule); + std::push_heap(granulesSorted.begin(), granulesSorted.end()); + }; + if (pathIds.size()) { + for (auto&& pathId : pathIds) { + auto it = Tables.find(pathId); + AFL_VERIFY(it != Tables.end()); + actor(it->first, it->second); } - NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("path_id", it->second->GetPathId()); - if (it->second->IsLockedOptimizer(dataLocksManager)) { - Counters.OnGranuleOptimizerLocked(); - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_optimizer_throught_lock")("priority", it->first.DebugString()); - } else { - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "granule_compaction_weight")("priority", it->first.DebugString()); - return it->second; + } else { + for (auto&& i : Tables) { + actor(i.first, i.second); + } + } + while (granulesSorted.size()) { + if (!dataLocksManager->IsLocked(*granulesSorted.front().GetGranule())) { + priorityChecker = granulesSorted.front().GetPriority(); + maxPriorityGranule = granulesSorted.front().GetGranule(); + break; } + std::pop_heap(granulesSorted.begin(), granulesSorted.end()); + granulesSorted.pop_back(); } + if (granuleResult) { + *granuleResult = maxPriorityGranule; + } + return priorityChecker; +} - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "all_significant_granules_locked")("count", granulesSorted.size()); - return nullptr; +std::shared_ptr TGranulesStorage::GetGranuleForCompaction(const std::shared_ptr& dataLocksManager) const { + std::shared_ptr granuleMaxPriority; + std::optional priorityChecker = + GetCompactionPriority(dataLocksManager, {}, std::nullopt, &granuleMaxPriority); + if (!granuleMaxPriority) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "no_granules"); + return nullptr; + } + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("path_id", granuleMaxPriority->GetPathId()); + AFL_VERIFY(!dataLocksManager->IsLocked(*granuleMaxPriority)); + AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "granule_compaction_weight")("priority", priorityChecker->DebugString()); + return granuleMaxPriority; } -} // namespace NKikimr::NOlap +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.h b/ydb/core/tx/columnshard/engines/storage/granule/storage.h index b925e42fd7fa..021ae1829d7c 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.h @@ -180,7 +180,9 @@ class TGranulesStorage { } std::shared_ptr GetGranuleForCompaction(const std::shared_ptr& locksManager) const; - + std::optional GetCompactionPriority(const std::shared_ptr& locksManager, + const std::set& pathIds = Default>(), const std::optional waitingPriority = std::nullopt, + std::shared_ptr* granuleResult = nullptr) const; }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h index 4bd196e552d0..1fceea178a89 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h @@ -32,6 +32,10 @@ class TOptimizationPriority { } public: + ui64 GetGeneralPriority() const { + return ((ui64)Level << 56) + InternalLevelWeight; + } + bool operator<(const TOptimizationPriority& item) const { return std::tie(Level, InternalLevelWeight) < std::tie(item.Level, item.InternalLevelWeight); } @@ -135,9 +139,6 @@ class IOptimizerPlanner { } virtual NArrow::NMerger::TIntervalPositions GetBucketPositions() const = 0; - bool IsLocked(const std::shared_ptr& dataLocksManager) const { - return DoIsLocked(dataLocksManager); - } NJson::TJsonValue SerializeToJsonVisual() const { return DoSerializeToJsonVisual(); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.cpp index 5d2501faad92..250c5b659ade 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.cpp @@ -1,18 +1,5 @@ #include "counters.h" -#include namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { -void TPortionCategoryCounters::AddPortion(const std::shared_ptr& p) { - RecordsCount->Add(p->NumRows()); - Count->Add(1); - Bytes->Add(p->GetTotalBlobBytes()); -} - -void TPortionCategoryCounters::RemovePortion(const std::shared_ptr& p) { - RecordsCount->Remove(p->NumRows()); - Count->Remove(1); - Bytes->Remove(p->GetTotalBlobBytes()); -} - } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.h index 64297bf21f79..e643a35cfa7c 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/counters.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace NKikimr::NOlap { class TPortionInfo; @@ -7,39 +8,8 @@ class TPortionInfo; namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { -class TPortionCategoryCounterAgents: public NColumnShard::TCommonCountersOwner { -private: - using TBase = NColumnShard::TCommonCountersOwner; -public: - const std::shared_ptr RecordsCount; - const std::shared_ptr Count; - const std::shared_ptr Bytes; - TPortionCategoryCounterAgents(NColumnShard::TCommonCountersOwner& base, const TString& categoryName) - : TBase(base, "category", categoryName) - , RecordsCount(TBase::GetValueAutoAggregations("ByGranule/Portions/RecordsCount")) - , Count(TBase::GetValueAutoAggregations("ByGranule/Portions/Count")) - , Bytes(TBase::GetValueAutoAggregations("ByGranule/Portions/Bytes")) - { - } -}; - -class TPortionCategoryCounters { -private: - std::shared_ptr RecordsCount; - std::shared_ptr Count; - std::shared_ptr Bytes; -public: - TPortionCategoryCounters(TPortionCategoryCounterAgents& agents) - { - RecordsCount = agents.RecordsCount->GetClient(); - Count = agents.Count->GetClient(); - Bytes = agents.Bytes->GetClient(); - } - - void AddPortion(const std::shared_ptr& p); - - void RemovePortion(const std::shared_ptr& p); -}; +using TPortionCategoryCounterAgents = NColumnShard::TPortionCategoryCounterAgents; +using TPortionCategoryCounters = NColumnShard::TPortionCategoryCounters; class TGlobalCounters: public NColumnShard::TCommonCountersOwner { private: @@ -54,6 +24,7 @@ class TGlobalCounters: public NColumnShard::TCommonCountersOwner { std::shared_ptr OldestCriticalActuality; std::shared_ptr MergeCoefficient; + public: NMonitoring::TDynamicCounters::TCounterPtr FinalBucketTaskCounter; NMonitoring::TDynamicCounters::TCounterPtr MiddleBucketTaskCounter; @@ -65,8 +36,7 @@ class TGlobalCounters: public NColumnShard::TCommonCountersOwner { NMonitoring::TDynamicCounters::TCounterPtr OptimizersCount; TGlobalCounters() - : TBase("BucketsStorageOptimizer") - { + : TBase("BucketsStorageOptimizer") { PortionsForMerge = std::make_shared(*this, "for_merge"); PortionsAlone = std::make_shared(*this, "alone"); SmallPortions = std::make_shared(*this, "small"); @@ -119,12 +89,12 @@ class TGlobalCounters: public NColumnShard::TCommonCountersOwner { static std::shared_ptr BuildFuturePortionsAggregation() { return std::make_shared(*Singleton()->FuturePortions); } - }; class TCounters { private: std::shared_ptr OldestCriticalActuality; + public: const std::shared_ptr PortionsForMerge; const std::shared_ptr PortionsAlone; @@ -140,11 +110,13 @@ class TCounters { if (isFinalBucket) { Singleton()->FinalBucketTaskCounter->Add(1); Singleton()->HistogramFinalBucketTask->Collect((Now() - youngestSnapshot).MilliSeconds() * 0.001, 1); - Singleton()->HistogramFinalBucketTaskSnapshotsDiff->Collect((youngestSnapshot - oldestSnapshot).MilliSeconds() * 0.001, 1); + Singleton()->HistogramFinalBucketTaskSnapshotsDiff->Collect( + (youngestSnapshot - oldestSnapshot).MilliSeconds() * 0.001, 1); } else { Singleton()->MiddleBucketTaskCounter->Add(1); Singleton()->HistogramMiddleBucketTask->Collect((Now() - youngestSnapshot).MilliSeconds() * 0.001, 1); - Singleton()->HistogramMiddleBucketTaskSnapshotsDiff->Collect((youngestSnapshot - oldestSnapshot).MilliSeconds() * 0.001, 1); + Singleton()->HistogramMiddleBucketTaskSnapshotsDiff->Collect( + (youngestSnapshot - oldestSnapshot).MilliSeconds() * 0.001, 1); } } @@ -157,15 +129,13 @@ class TCounters { , MergeCoefficient(TGlobalCounters::BuildMergeCoefficientAggregation()) , HistogramDiffSnapshots(TGlobalCounters::BuildHistogramDiffSnapshots()) , BucketsForMerge(Singleton()->BucketsForMerge->GetClient()) - , OptimizersCount(std::make_shared(Singleton()->OptimizersCount)) - { + , OptimizersCount(std::make_shared(Singleton()->OptimizersCount)) { OldestCriticalActuality = TGlobalCounters::BuildOldestCriticalActualityAggregation(); } void OnMinProblemSnapshot(const TDuration d) { OldestCriticalActuality->SetValue(d.MilliSeconds(), TInstant::Now() + TDuration::Seconds(10)); } - }; -} +} // namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp new file mode 100644 index 000000000000..ed41f5de42f7 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp @@ -0,0 +1,36 @@ +#include "constructor.h" +#include + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +NKikimr::TConclusion> TOptimizerPlannerConstructor::DoBuildPlanner(const TBuildContext& context) const { + return std::make_shared(context.GetPathId(), context.GetStorages(), context.GetPKSchema()); +} + +bool TOptimizerPlannerConstructor::DoApplyToCurrentObject(IOptimizerPlanner& current) const { + auto* itemClass = dynamic_cast(¤t); + if (!itemClass) { + return false; + } + return true; +} + +bool TOptimizerPlannerConstructor::DoIsEqualTo(const IOptimizerPlannerConstructor& item) const { + const auto* itemClass = dynamic_cast(&item); + AFL_VERIFY(itemClass); + return true; +} + +void TOptimizerPlannerConstructor::DoSerializeToProto(TProto& proto) const { + *proto.MutableLCBuckets() = NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TLCOptimizer(); +} + +bool TOptimizerPlannerConstructor::DoDeserializeFromProto(const TProto& proto) { + if (!proto.HasLCBuckets()) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse lc-buckets optimizer from proto")("proto", proto.DebugString()); + return false; + } + return true; +} + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.h new file mode 100644 index 000000000000..f1d47481c177 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.h @@ -0,0 +1,31 @@ +#pragma once +#include + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +class TOptimizerPlannerConstructor: public IOptimizerPlannerConstructor { +public: + static TString GetClassNameStatic() { + return "lc-buckets"; + } +private: + static inline const TFactory::TRegistrator Registrator = TFactory::TRegistrator(GetClassNameStatic()); + + virtual void DoSerializeToProto(TProto& proto) const override; + + virtual bool DoDeserializeFromProto(const TProto& proto) override; + virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& /*jsonInfo*/) override { + return TConclusionStatus::Success(); + } + virtual bool DoApplyToCurrentObject(IOptimizerPlanner& current) const override; + + virtual TConclusion> DoBuildPlanner(const TBuildContext& context) const override; + virtual bool DoIsEqualTo(const IOptimizerPlannerConstructor& item) const override; +public: + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } + +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/ya.make new file mode 100644 index 000000000000..f95d3abf7469 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + GLOBAL constructor.cpp +) + +PEERDIR( + contrib/libs/apache/arrow + ydb/core/protos + ydb/core/formats/arrow + ydb/core/tx/columnshard/engines/changes/abstract +) + +END() diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.cpp new file mode 100644 index 000000000000..6055c30c18c9 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.cpp @@ -0,0 +1,57 @@ +#include "abstract.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +NKikimr::NArrow::NMerger::TIntervalPositions TCompactionTaskData::GetCheckPositions( + const std::shared_ptr& pkSchema, const bool withMoved) { + NArrow::NMerger::TIntervalPositions result; + for (auto&& i : GetFinishPoints(withMoved)) { + result.AddPosition(NArrow::NMerger::TSortableBatchPosition(i.ToBatch(pkSchema), 0, pkSchema->field_names(), {}, false), false); + } + return result; +} + +std::vector TCompactionTaskData::GetFinishPoints(const bool withMoved) { + std::vector points; + if (MemoryUsage > ((ui64)1 << 30)) { + for (auto&& i : Portions) { + if (!CurrentLevelPortionIds.contains(i->GetPortionId())) { + points.emplace_back(i->IndexKeyStart()); + } + } + std::sort(points.begin(), points.end()); + return points; + } + + THashSet middlePortions; + for (auto&& i : Chains) { + for (auto&& p : i.GetPortions()) { + middlePortions.emplace(p->GetPortionId()); + } + } + THashSet endPortions; + for (auto&& i : Chains) { + if (!i.GetNotIncludedNextPortion()) { + continue; + } + if (middlePortions.contains(i.GetNotIncludedNextPortion()->GetPortionId())) { + continue; + } + if (!endPortions.emplace(i.GetNotIncludedNextPortion()->GetPortionId()).second) { + continue; + } + points.emplace_back(i.GetNotIncludedNextPortion()->IndexKeyStart()); + } + if (withMoved) { + for (auto&& i : GetMovePortions()) { + points.emplace_back(i->IndexKeyStart()); + } + } + if (StopSeparation) { + points.emplace_back(*StopSeparation); + } + std::sort(points.begin(), points.end()); + return points; +} + +} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.h new file mode 100644 index 000000000000..971917eb2125 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/abstract.h @@ -0,0 +1,374 @@ +#pragma once +#include "counters.h" + +#include +#include + +#include + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +class TOrderedPortion { +private: + std::shared_ptr Portion; + NArrow::TReplaceKey Start; + ui64 PortionId; + NArrow::NMerger::TSortableBatchPosition StartPosition; + +public: + const std::shared_ptr& GetPortion() const { + AFL_VERIFY(Portion); + return Portion; + } + + const NArrow::TReplaceKey& GetStart() const { + return Start; + } + + const NArrow::NMerger::TSortableBatchPosition& GetStartPosition() const { + AFL_VERIFY(Portion); + return StartPosition; + } + + TOrderedPortion(const std::shared_ptr& portion) + : Portion(portion) + , Start(portion->IndexKeyStart()) + , PortionId(portion->GetPortionId()) + , StartPosition(Portion->GetMeta().GetFirstLastPK().GetBatch(), 0, false) { + } + + TOrderedPortion(const NArrow::TReplaceKey& start) + : Start(start) + , PortionId(Max()) { + } + + bool operator<(const TOrderedPortion& item) const { + auto cmp = Start.CompareNotNull(item.Start); + if (cmp == std::partial_ordering::equivalent) { + return PortionId < item.PortionId; + } else { + return cmp == std::partial_ordering::less; + } + } +}; + +class TChainAddress { +private: + YDB_READONLY(ui64, FromPortionId, 0); + YDB_READONLY(ui64, ToPortionId, 0); + bool LastIsSeparator = false; + +public: + TChainAddress(const ui64 from, const ui64 to, const bool lastIsSeparator) + : FromPortionId(from) + , ToPortionId(to) + , LastIsSeparator(lastIsSeparator) { + } + + bool operator<(const TChainAddress& item) const { + return std::tie(FromPortionId, ToPortionId, LastIsSeparator) < std::tie(item.FromPortionId, item.ToPortionId, item.LastIsSeparator); + } + + TString DebugString() const { + return TStringBuilder() << FromPortionId << "-" << ToPortionId << ":" << LastIsSeparator; + } +}; + +class TPortionsChain { +private: + std::vector> Portions; + + std::shared_ptr NotIncludedNextPortion; + +public: + const std::vector>& GetPortions() const { + return Portions; + } + + const std::shared_ptr& GetNotIncludedNextPortion() const { + return NotIncludedNextPortion; + } + + TChainAddress GetAddress() const { + if (Portions.size()) { + return TChainAddress(Portions.front()->GetPortionId(), + NotIncludedNextPortion ? NotIncludedNextPortion->GetPortionId() : Portions.back()->GetPortionId(), !!NotIncludedNextPortion); + } else { + AFL_VERIFY(NotIncludedNextPortion); + return TChainAddress(NotIncludedNextPortion->GetPortionId(), NotIncludedNextPortion->GetPortionId(), true); + } + } + + TPortionsChain(const std::vector>& portions, const std::shared_ptr& notIncludedNextPortion) + : Portions(portions) + , NotIncludedNextPortion(notIncludedNextPortion) { + AFL_VERIFY(Portions.size() || !!NotIncludedNextPortion); + } +}; + +class TCompactionTaskData { +private: + YDB_ACCESSOR_DEF(std::vector>, Portions); + const ui64 TargetCompactionLevel = 0; + std::shared_ptr Predictor = + NCompaction::TGeneralCompactColumnEngineChanges::BuildMemoryPredictor(); + ui64 MemoryUsage = 0; + THashSet UsedPortionIds; + THashSet RepackPortionIds; + + TSimplePortionsGroupInfo CurrentLevelPortionsInfo; + TSimplePortionsGroupInfo TargetLevelPortionsInfo; + + std::set NextLevelChainIds; + THashSet NextLevelPortionIds; + THashSet CurrentLevelPortionIds; + std::vector Chains; + std::optional StopSeparation; + +public: + ui64 GetTargetCompactionLevel() const { + if (MemoryUsage > ((ui64)1 << 30)) { + AFL_VERIFY(TargetCompactionLevel); + return TargetCompactionLevel - 1; + } else { + return TargetCompactionLevel; + } + } + + void SetStopSeparation(const NArrow::TReplaceKey& point) { + AFL_VERIFY(!StopSeparation); + StopSeparation = point; + } + + std::vector> GetRepackPortions(const ui32 levelIdx) const { + std::vector> result; + if (MemoryUsage > ((ui64)1 << 30)) { + auto predictor = NCompaction::TGeneralCompactColumnEngineChanges::BuildMemoryPredictor(); + for (auto&& i : Portions) { + if (CurrentLevelPortionIds.contains(i->GetPortionId())) { + if (predictor->AddPortion(*i) < MemoryUsage || result.size() < 2) { + result.emplace_back(i); + } else { + break; + } + } + } + return result; + } else if (levelIdx == 0) { + return Portions; + } + auto moveIds = GetMovePortionIds(); + for (auto&& i : Portions) { + if (!moveIds.contains(i->GetPortionId())) { + result.emplace_back(i); + } + } + return result; + } + + std::vector> GetMovePortions() const { + if (MemoryUsage > ((ui64)1 << 30)) { + return {}; + } + auto moveIds = GetMovePortionIds(); + std::vector> result; + for (auto&& i : Portions) { + if (moveIds.contains(i->GetPortionId())) { + result.emplace_back(i); + } + } + return result; + } + + ui64 GetRepackPortionsVolume() const { + return TargetLevelPortionsInfo.GetRawBytes(); + } + + THashSet GetMovePortionIds() const { + auto movePortionIds = CurrentLevelPortionIds; + for (auto&& i : RepackPortionIds) { + movePortionIds.erase(i); + } + return movePortionIds; + } + + TString DebugString() const { + TStringBuilder sb; + sb << "target_level_chains:["; + for (auto&& i : NextLevelChainIds) { + sb << i.DebugString() << ","; + } + sb << "];target_level_portions:[" << JoinSeq(",", NextLevelPortionIds) << "];current_level_portions_info:{" + << CurrentLevelPortionsInfo.DebugString() << "};target_level_portions_info:{" << TargetLevelPortionsInfo.DebugString() << "};"; + sb << "move_portion_ids:[" << JoinSeq(",", GetMovePortionIds()) << "]"; + return sb; + } + + TCompactionTaskData() = default; + + const THashSet& GetPortionIds() const { + return UsedPortionIds; + } + + bool Contains(const ui64 portionId) const { + return UsedPortionIds.contains(portionId); + } + + bool IsEmpty() const { + return !Portions.size(); + } + + NArrow::NMerger::TIntervalPositions GetCheckPositions(const std::shared_ptr& pkSchema, const bool withMoved); + std::vector GetFinishPoints(const bool withMoved); + + void AddCurrentLevelPortion(const std::shared_ptr& portion, std::optional&& chain, const bool repackMoved) { + AFL_VERIFY(UsedPortionIds.emplace(portion->GetPortionId()).second); + AFL_VERIFY(CurrentLevelPortionIds.emplace(portion->GetPortionId()).second); + Portions.emplace_back(portion); + CurrentLevelPortionsInfo.AddPortion(portion); + if (repackMoved || (chain && chain->GetPortions().size())) { + MemoryUsage = Predictor->AddPortion(*portion); + } + + if (chain) { + if (chain->GetPortions().size()) { + RepackPortionIds.emplace(portion->GetPortionId()); + } + if (NextLevelChainIds.emplace(chain->GetAddress()).second) { + Chains.emplace_back(std::move(*chain)); + for (auto&& i : Chains.back().GetPortions()) { + if (!UsedPortionIds.emplace(i->GetPortionId()).second) { + AFL_VERIFY(NextLevelPortionIds.contains(i->GetPortionId())); + continue; + } + TargetLevelPortionsInfo.AddPortion(i); + Portions.emplace_back(i); + MemoryUsage = Predictor->AddPortion(*i); + AFL_VERIFY(NextLevelPortionIds.emplace(i->GetPortionId()).second); + } + } + } + } + + bool CanTakeMore() const { + return MemoryUsage < (((ui64)512) << 20) && Portions.size() < 10000; + } + + TCompactionTaskData(const ui64 targetCompactionLevel) + : TargetCompactionLevel(targetCompactionLevel) { + } +}; + +class IPortionsLevel { +private: + virtual void DoModifyPortions( + const std::vector>& add, const std::vector>& remove) = 0; + virtual ui64 DoGetWeight() const = 0; + virtual NArrow::NMerger::TIntervalPositions DoGetBucketPositions(const std::shared_ptr& pkSchema) const = 0; + virtual TCompactionTaskData DoGetOptimizationTask() const = 0; + virtual std::optional DoGetAffectedPortions(const NArrow::TReplaceKey& from, const NArrow::TReplaceKey& to) const = 0; + virtual ui64 DoGetAffectedPortionBytes(const NArrow::TReplaceKey& from, const NArrow::TReplaceKey& to) const = 0; + + virtual NJson::TJsonValue DoSerializeToJson() const { + return NJson::JSON_MAP; + } + + virtual TString DoDebugString() const { + return ""; + } + + YDB_READONLY(ui64, LevelId, 0); + +protected: + std::shared_ptr NextLevel; + TSimplePortionsGroupInfo PortionsInfo; + mutable std::optional PredOptimization = TInstant::Now(); + +public: + bool HasData() const { + return PortionsInfo.GetCount(); + } + + virtual std::optional GetPackKff() const { + if (PortionsInfo.GetRawBytes()) { + return 1.0 * PortionsInfo.GetBlobBytes() / PortionsInfo.GetRawBytes(); + } else if (!NextLevel) { + return std::nullopt; + } else { + return NextLevel->GetPackKff(); + } + } + + const TSimplePortionsGroupInfo& GetPortionsInfo() const { + return PortionsInfo; + } + + const std::shared_ptr& GetNextLevel() const { + return NextLevel; + } + + virtual ~IPortionsLevel() = default; + IPortionsLevel(const ui64 levelId, const std::shared_ptr& nextLevel) + : LevelId(levelId) + , NextLevel(nextLevel) { + } + + bool CanTakePortion(const std::shared_ptr& portion) const { + auto chain = GetAffectedPortions(portion->IndexKeyStart(), portion->IndexKeyEnd()); + if (chain && chain->GetPortions().size()) { + return false; + } + return true; + } + + virtual bool IsLocked(const std::shared_ptr& locksManager) const = 0; + + virtual TTaskDescription GetTaskDescription() const { + TTaskDescription result(0); + result.SetWeight(GetWeight()); + result.SetDetails(SerializeToJson().GetStringRobust()); + return result; + } + + NJson::TJsonValue SerializeToJson() const { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("level", LevelId); + result.InsertValue("weight", GetWeight()); + result.InsertValue("portions", PortionsInfo.SerializeToJson()); + result.InsertValue("details", DoSerializeToJson()); + return result; + } + + TString DebugString() const { + return DoDebugString(); + } + + std::optional GetAffectedPortions(const NArrow::TReplaceKey& from, const NArrow::TReplaceKey& to) const { + return DoGetAffectedPortions(from, to); + } + + ui64 GetAffectedPortionBytes(const NArrow::TReplaceKey& from, const NArrow::TReplaceKey& to) const { + return DoGetAffectedPortionBytes(from, to); + } + + void ModifyPortions(const std::vector>& add, const std::vector>& remove) { + return DoModifyPortions(add, remove); + } + + ui64 GetWeight() const { + return DoGetWeight(); + } + + NArrow::NMerger::TIntervalPositions GetBucketPositions(const std::shared_ptr& pkSchema) const { + return DoGetBucketPositions(pkSchema); + } + + TCompactionTaskData GetOptimizationTask() const { + AFL_VERIFY(NextLevel); + TCompactionTaskData result = DoGetOptimizationTask(); + AFL_VERIFY(!result.IsEmpty()); + return result; + } +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/accumulation_level.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/accumulation_level.cpp new file mode 100644 index 000000000000..fcd7fcbb9bb2 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/accumulation_level.cpp @@ -0,0 +1,5 @@ +#include "accumulation_level.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/accumulation_level.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/accumulation_level.h new file mode 100644 index 000000000000..a8598871358e --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/accumulation_level.h @@ -0,0 +1,119 @@ +#pragma once +#include "abstract.h" +#include "counters.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +class TAccumulationLevelPortions: public IPortionsLevel { +private: + using TBase = IPortionsLevel; + const TLevelCounters LevelCounters; + + std::set Portions; + + virtual std::optional DoGetAffectedPortions( + const NArrow::TReplaceKey& /*from*/, const NArrow::TReplaceKey& /*to*/) const override { + return std::nullopt; + } + + virtual ui64 DoGetAffectedPortionBytes(const NArrow::TReplaceKey& /*from*/, const NArrow::TReplaceKey& /*to*/) const override { + return 0; + } + + virtual ui64 DoGetWeight() const override { + if (PortionsInfo.GetCount() <= 1) { + return 0; + } + + THashSet portionIds; + auto targetLevel = GetNextLevel(); + + + const ui64 affectedRawBytes = targetLevel->GetAffectedPortionBytes( + Portions.begin()->GetPortion()->IndexKeyStart(), Portions.rbegin()->GetPortion()->IndexKeyEnd()); + /* + auto chain = + targetLevel->GetAffectedPortions(Portions.begin()->GetPortion()->IndexKeyStart(), Portions.rbegin()->GetPortion()->IndexKeyEnd()); + if (chain) { + auto it = Portions.begin(); + auto itNext = chain->GetPortions().begin(); + while (it != Portions.end() && itNext != chain->GetPortions().end()) { + const auto& nextLevelPortion = *itNext; + if (nextLevelPortion->IndexKeyEnd() < it->GetPortion()->IndexKeyStart()) { + ++itNext; + } else if (it->GetPortion()->IndexKeyEnd() < nextLevelPortion->IndexKeyStart()) { + ++it; + } else { + if (portionIds.emplace(nextLevelPortion->GetPortionId()).second) { + affectedRawBytes += nextLevelPortion->GetTotalRawBytes(); + } + ++itNext; + } + } + } +*/ + + const ui64 mb = (affectedRawBytes + PortionsInfo.GetRawBytes()) / 1000000 + 1; + return 1000000000.0 * PortionsInfo.GetCount() * PortionsInfo.GetCount() / mb; + } + +public: + TAccumulationLevelPortions(const ui64 levelId, const std::shared_ptr& nextLevel, const TLevelCounters& levelCounters) + : TBase(levelId, nextLevel) + , LevelCounters(levelCounters) { + } + + virtual bool IsLocked(const std::shared_ptr& locksManager) const override { + for (auto&& i : Portions) { + if (locksManager->IsLocked(*i.GetPortion())) { + return true; + } + } + return false; + } + + virtual void DoModifyPortions( + const std::vector>& add, const std::vector>& remove) override { + for (auto&& i : remove) { + auto it = Portions.find(i); + AFL_VERIFY(it != Portions.end()); + AFL_VERIFY(it->GetPortion()->GetPortionId() == i->GetPortionId()); + PortionsInfo.RemovePortion(i); + Portions.erase(it); + LevelCounters.Portions->RemovePortion(i); + } + for (auto&& i : add) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "add_accum")("portion_id", i->GetPortionId())( + "blob_size", i->GetTotalBlobBytes()); + AFL_VERIFY(Portions.emplace(i).second); + PortionsInfo.AddPortion(i); + LevelCounters.Portions->AddPortion(i); + } + } + + virtual TCompactionTaskData DoGetOptimizationTask() const override { + AFL_VERIFY(Portions.size()); + std::shared_ptr targetLevel = GetNextLevel(); + AFL_VERIFY(targetLevel); + TCompactionTaskData result(targetLevel->GetLevelId()); + { + for (auto&& i : Portions) { + result.AddCurrentLevelPortion( + i.GetPortion(), targetLevel->GetAffectedPortions(i.GetPortion()->IndexKeyStart(), i.GetPortion()->IndexKeyEnd()), true); + if (!result.CanTakeMore()) { + result.SetStopSeparation(i.GetPortion()->IndexKeyStart()); + break; + } + } + } + return result; + } + + virtual NArrow::NMerger::TIntervalPositions DoGetBucketPositions(const std::shared_ptr& /*pkSchema*/) const override { + AFL_VERIFY(false); + NArrow::NMerger::TIntervalPositions result; + return result; + } +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/common_level.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/common_level.cpp new file mode 100644 index 000000000000..3de3bd02abba --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/common_level.cpp @@ -0,0 +1,78 @@ +#include "common_level.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +void TLevelPortions::DoModifyPortions( + const std::vector>& add, const std::vector>& remove) { + for (auto&& i : remove) { + auto it = Portions.find(i); + AFL_VERIFY(it != Portions.end()); + AFL_VERIFY(it->GetPortion()->GetPortionId() == i->GetPortionId()); + PortionsInfo.RemovePortion(i); + Portions.erase(it); + LevelCounters.Portions->RemovePortion(i); + } + TStringBuilder sb; + for (auto&& i : add) { + sb << i->GetPortionId() << ","; + auto info = Portions.emplace(i); + i->AddRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized); + AFL_VERIFY(info.second); + PortionsInfo.AddPortion(i); + if (StrictOneLayer) { + { + auto it = info.first; + ++it; + if (it != Portions.end()) { + AFL_VERIFY(i->IndexKeyEnd() < it->GetStart())("start", i->IndexKeyStart().DebugString())("end", i->IndexKeyEnd().DebugString())( + "next", it->GetStart().DebugString())("next1", it->GetStart().DebugString())( + "next2", it->GetPortion()->IndexKeyEnd().DebugString())("level_id", GetLevelId())( + "portion_id_new", i->GetPortionId())("portion_id_old", it->GetPortion()->GetPortionId())( + "portion_old", it->GetPortion()->DebugString())("add", sb); + } + } + { + auto it = info.first; + if (it != Portions.begin()) { + --it; + AFL_VERIFY(it->GetPortion()->IndexKeyEnd() < i->IndexKeyStart()) + ("start", i->IndexKeyStart().DebugString())("finish", i->IndexKeyEnd().DebugString())("pred_start", + it->GetPortion()->IndexKeyStart().DebugString())("pred_finish", it->GetPortion()->IndexKeyEnd().DebugString())("level_id", GetLevelId())( + "portion_id_new", i->GetPortionId())("portion_id_old", it->GetPortion()->GetPortionId())("add", sb); + } + } + } + LevelCounters.Portions->AddPortion(i); + } +} + +TCompactionTaskData TLevelPortions::DoGetOptimizationTask() const { + AFL_VERIFY(GetNextLevel()); + ui64 compactedData = 0; + TCompactionTaskData result(GetNextLevel()->GetLevelId()); + auto itFwd = Portions.begin(); + AFL_VERIFY(itFwd != Portions.end()); + auto itBkwd = itFwd; + if (itFwd != Portions.begin()) { + --itBkwd; + } + while (GetLevelBlobBytesLimit() * 0.5 + compactedData < (ui64)PortionsInfo.GetBlobBytes() && + (itBkwd != Portions.begin() || itFwd != Portions.end()) && result.CanTakeMore()) { + if (itFwd != Portions.end() && + (itBkwd == Portions.begin() || itBkwd->GetPortion()->GetTotalBlobBytes() <= itFwd->GetPortion()->GetTotalBlobBytes())) { + auto portion = itFwd->GetPortion(); + compactedData += portion->GetTotalBlobBytes(); + result.AddCurrentLevelPortion(portion, GetNextLevel()->GetAffectedPortions(portion->IndexKeyStart(), portion->IndexKeyEnd()), false); + ++itFwd; + } else if (itBkwd != Portions.begin() && + (itFwd == Portions.end() || itFwd->GetPortion()->GetTotalBlobBytes() < itBkwd->GetPortion()->GetTotalBlobBytes())) { + auto portion = itBkwd->GetPortion(); + compactedData += portion->GetTotalBlobBytes(); + result.AddCurrentLevelPortion(portion, GetNextLevel()->GetAffectedPortions(portion->IndexKeyStart(), portion->IndexKeyEnd()), false); + --itBkwd; + } + } + return result; +} + +} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/common_level.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/common_level.h new file mode 100644 index 000000000000..3f3c56b426a8 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/common_level.h @@ -0,0 +1,131 @@ +#pragma once +#include "abstract.h" +#include "counters.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +class TLevelPortions: public IPortionsLevel { +private: + using TBase = IPortionsLevel; + + std::set Portions; + const TLevelCounters LevelCounters; + const double BytesLimitFraction = 1; + const ui64 ExpectedPortionSize = (1 << 20); + const bool StrictOneLayer = true; + std::shared_ptr SummaryPortionsInfo; + + ui64 GetLevelBlobBytesLimit() const { + const ui32 discrete = SummaryPortionsInfo->GetBlobBytes() / (150 << 20); + return (discrete + 1) * (150 << 20) * BytesLimitFraction; + } + + virtual NJson::TJsonValue DoSerializeToJson() const override { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("expected_portion_size", ExpectedPortionSize); + result.InsertValue("bytes_limit", GetLevelBlobBytesLimit()); + result.InsertValue("total_bytes", SummaryPortionsInfo->GetBlobBytes()); + result.InsertValue("fraction", BytesLimitFraction); + return result; + } + + virtual std::optional DoGetAffectedPortions(const NArrow::TReplaceKey& from, const NArrow::TReplaceKey& to) const override { + if (Portions.empty()) { + return std::nullopt; + } + std::vector> result; + auto itFrom = Portions.upper_bound(from); + auto itTo = Portions.upper_bound(to); + if (itFrom != Portions.begin()) { + auto it = itFrom; + --it; + if (from <= it->GetPortion()->IndexKeyEnd()) { + result.insert(result.begin(), it->GetPortion()); + } + } + for (auto it = itFrom; it != itTo; ++it) { + result.emplace_back(it->GetPortion()); + } + if (itTo != Portions.end()) { + return TPortionsChain(std::move(result), itTo->GetPortion()); + } else if (result.size()) { + return TPortionsChain(std::move(result), nullptr); + } else { + return std::nullopt; + } + } + + virtual ui64 DoGetWeight() const override { + if (!GetNextLevel()) { + return 0; + } + if ((ui64)PortionsInfo.GetBlobBytes() > GetLevelBlobBytesLimit() && PortionsInfo.GetCount() > 2 && + (ui64)PortionsInfo.GetBlobBytes() > ExpectedPortionSize * 2) { + return ((ui64)GetLevelId() << 48) + PortionsInfo.GetBlobBytes() - GetLevelBlobBytesLimit(); + } else { + return 0; + } + } + +public: + TLevelPortions(const ui64 levelId, const double bytesLimitFraction, const ui64 expectedPortionSize, + const std::shared_ptr& nextLevel, const std::shared_ptr& summaryPortionsInfo, + const TLevelCounters& levelCounters, const bool strictOneLayer = true) + : TBase(levelId, nextLevel) + , LevelCounters(levelCounters) + , BytesLimitFraction(bytesLimitFraction) + , ExpectedPortionSize(expectedPortionSize) + , StrictOneLayer(strictOneLayer) + , SummaryPortionsInfo(summaryPortionsInfo) + { + } + + ui64 GetExpectedPortionSize() const { + return ExpectedPortionSize; + } + + virtual bool IsLocked(const std::shared_ptr& locksManager) const override { + for (auto&& i : Portions) { + if (locksManager->IsLocked(*i.GetPortion())) { + return true; + } + } + return false; + } + + virtual ui64 DoGetAffectedPortionBytes(const NArrow::TReplaceKey& from, const NArrow::TReplaceKey& to) const override { + if (Portions.empty()) { + return 0; + } + ui64 result = 0; + auto itFrom = Portions.upper_bound(from); + auto itTo = Portions.upper_bound(to); + if (itFrom != Portions.begin()) { + auto it = itFrom; + --it; + if (from <= it->GetPortion()->IndexKeyEnd()) { + result += it->GetPortion()->GetTotalRawBytes(); + } + } + for (auto it = itFrom; it != itTo; ++it) { + result += it->GetPortion()->GetTotalRawBytes(); + } + return result; + } + + virtual void DoModifyPortions( + const std::vector>& add, const std::vector>& remove) override; + + virtual TCompactionTaskData DoGetOptimizationTask() const override; + + virtual NArrow::NMerger::TIntervalPositions DoGetBucketPositions(const std::shared_ptr& pkSchema) const override { + NArrow::NMerger::TIntervalPositions result; + const auto& sortingColumns = pkSchema->field_names(); + for (auto&& i : Portions) { + result.AddPosition(i.GetStartPosition(), false); + } + return result; + } +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/counters.cpp new file mode 100644 index 000000000000..a429ee576fdd --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/counters.cpp @@ -0,0 +1,5 @@ +#include "counters.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/counters.h new file mode 100644 index 000000000000..95d4924b3ece --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/counters.h @@ -0,0 +1,78 @@ +#pragma once +#include +#include + +#include + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +using TPortionCategoryCounterAgents = NColumnShard::TPortionCategoryCounterAgents; +using TPortionCategoryCounters = NColumnShard::TPortionCategoryCounters; + +class TLevelAgents { +private: + const ui32 LevelId; + +public: + const std::shared_ptr Portions; + + TLevelAgents(const ui32 levelId, NColumnShard::TCommonCountersOwner& baseOwner) + : LevelId(levelId) + , Portions(std::make_shared(baseOwner, "level=" + ::ToString(LevelId))) { + } +}; + +class TGlobalCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + std::vector> Levels; + +public: + TGlobalCounters() + : TBase("LeveledCompactionOptimizer") { + for (ui32 i = 0; i <= 10; ++i) { + Levels.emplace_back(std::make_shared(i, *this)); + } + } + + static std::shared_ptr BuildPortionsCounter(const ui32 levelId) { + AFL_VERIFY(levelId < Singleton()->Levels.size()); + return std::make_shared(*Singleton()->Levels[levelId]->Portions); + } +}; + +class TLevelCounters { +public: + const std::shared_ptr Portions; + TLevelCounters(const ui32 levelId) + : Portions(TGlobalCounters::BuildPortionsCounter(levelId)) { + } +}; + +class TCounters { +public: + std::vector Levels; + + TCounters() { + for (ui32 i = 0; i < 10; ++i) { + Levels.emplace_back(i); + } + } + + const TLevelCounters& GetLevelCounters(const ui32 levelIdx) const { + AFL_VERIFY(levelIdx < Levels.size())("idx", levelIdx)("count", Levels.size()); + return Levels[levelIdx]; + } + + void AddPortion(const ui32 levelId, const std::shared_ptr& portion) { + AFL_VERIFY(levelId < Levels.size()); + Levels[levelId].Portions->AddPortion(portion); + } + + void RemovePortion(const ui32 levelId, const std::shared_ptr& portion) { + AFL_VERIFY(levelId < Levels.size()); + Levels[levelId].Portions->RemovePortion(portion); + } +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp new file mode 100644 index 000000000000..871263e43e2d --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp @@ -0,0 +1,60 @@ +#include "accumulation_level.h" +#include "common_level.h" +#include "optimizer.h" +#include "zero_level.h" + +#include + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +TOptimizerPlanner::TOptimizerPlanner( + const ui64 pathId, const std::shared_ptr& storagesManager, const std::shared_ptr& primaryKeysSchema) + : TBase(pathId) + , Counters(std::make_shared()) + , StoragesManager(storagesManager) + , PrimaryKeysSchema(primaryKeysSchema) { + std::shared_ptr nextLevel; +/* + const ui64 maxPortionBlobBytes = (ui64)1 << 20; + Levels.emplace_back( + std::make_shared(2, 0.9, maxPortionBlobBytes, nullptr, PortionsInfo, Counters->GetLevelCounters(2))); + Levels.emplace_back( + std::make_shared(1, 0.1, maxPortionBlobBytes, Levels.back(), PortionsInfo, Counters->GetLevelCounters(1))); +*/ + Levels.emplace_back(std::make_shared(1, nullptr, Counters->GetLevelCounters(2))); + Levels.emplace_back(std::make_shared(0, Levels.back(), Counters->GetLevelCounters(0))); + std::reverse(Levels.begin(), Levels.end()); + RefreshWeights(); +} + +std::shared_ptr TOptimizerPlanner::DoGetOptimizationTask( + std::shared_ptr granule, const std::shared_ptr& locksManager) const { + AFL_VERIFY(LevelsByWeight.size()); + auto level = LevelsByWeight.begin()->second; + auto data = level->GetOptimizationTask(); + TSaverContext saverContext(StoragesManager); + std::shared_ptr result; + if (level->GetLevelId() == 0) { + result = std::make_shared( + granule, data.GetRepackPortions(level->GetLevelId()), saverContext); + } else { + result = std::make_shared( + granule, data.GetRepackPortions(level->GetLevelId()), saverContext); + result->AddMovePortions(data.GetMovePortions()); + } + result->SetTargetCompactionLevel(data.GetTargetCompactionLevel()); + auto levelPortions = std::dynamic_pointer_cast(Levels[data.GetTargetCompactionLevel()]); + if (levelPortions) { + result->SetPortionExpectedSize(levelPortions->GetExpectedPortionSize()); + } + auto positions = data.GetCheckPositions(PrimaryKeysSchema, level->GetLevelId() > 1); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("task_id", result->GetTaskIdentifier())("positions", positions.DebugString())( + "level", level->GetLevelId())("target", data.GetTargetCompactionLevel())("data", data.DebugString()); + result->SetCheckPoints(std::move(positions)); + for (auto&& i : result->SwitchedPortions) { + AFL_VERIFY(!locksManager->IsLocked(i)); + } + return result; +} + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h new file mode 100644 index 000000000000..1935fcaaa178 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h @@ -0,0 +1,151 @@ +#pragma once +#include "abstract.h" +#include "counters.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +class TOptimizerPlanner: public IOptimizerPlanner { +private: + using TBase = IOptimizerPlanner; + std::shared_ptr Counters; + std::shared_ptr PortionsInfo = std::make_shared(); + TInstant LastActualization = TInstant::Now(); + + std::vector> Levels; + class TReverseSorting { + public: + bool operator()(const ui64 l, const ui64 r) const { + return r < l; + } + }; + std::map, TReverseSorting> LevelsByWeight; + const std::shared_ptr StoragesManager; + const std::shared_ptr PrimaryKeysSchema; + virtual std::vector DoGetTasksDescription() const override { + std::vector result; + for (auto&& i : Levels) { + result.emplace_back(i->GetTaskDescription()); + } + return result; + } + + void RefreshWeights() { + LevelsByWeight.clear(); + for (ui32 i = 0; i < Levels.size(); ++i) { + LevelsByWeight.emplace(Levels[i]->GetWeight(), Levels[i]); + } + } + +protected: + virtual bool DoIsLocked(const std::shared_ptr& dataLocksManager) const override { + for (auto&& i : Levels) { + if (i->IsLocked(dataLocksManager)) { + return true; + } + } + return false; + } + + virtual void DoModifyPortions( + const THashMap>& add, const THashMap>& remove) override { + std::vector>> removePortionsByLevel; + removePortionsByLevel.resize(Levels.size()); + for (auto&& [_, i] : remove) { + if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") { + continue; + } + PortionsInfo->RemovePortion(i); + AFL_VERIFY(i->GetCompactionLevel() < Levels.size()); + removePortionsByLevel[i->GetCompactionLevel()].emplace_back(i); + } + for (ui32 i = 0; i < Levels.size(); ++i) { + Levels[i]->ModifyPortions({}, removePortionsByLevel[i]); + } + for (auto&& [_, i] : add) { + if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") { + continue; + } + PortionsInfo->AddPortion(i); + if (i->GetCompactionLevel() && (i->GetCompactionLevel() >= Levels.size() || !Levels[i->GetCompactionLevel()]->CanTakePortion(i))) { + i->MutableMeta().ResetCompactionLevel(0); + } + AFL_VERIFY(i->GetCompactionLevel() < Levels.size()); + if (i->GetMeta().GetCompactionLevel()) { + Levels[i->GetMeta().GetCompactionLevel()]->ModifyPortions({ i }, {}); + } + } + + for (auto&& [_, i] : add) { + if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") { + continue; + } + AFL_VERIFY(i->GetCompactionLevel() < Levels.size()); + if (i->GetCompactionLevel()) { + continue; + } + if (i->GetTotalBlobBytes() > 512 * 1024 && i->GetMeta().GetProduced() != NPortion::EProduced::INSERTED) { + for (i32 levelIdx = Levels.size() - 1; levelIdx >= 0; --levelIdx) { + if (Levels[levelIdx]->CanTakePortion(i)) { + Levels[levelIdx]->ModifyPortions({i}, {}); + i->MutableMeta().ResetCompactionLevel(levelIdx); + break; + } + } + } else { + Levels[0]->ModifyPortions({ i }, {}); + } + } + RefreshWeights(); + } + virtual std::shared_ptr DoGetOptimizationTask( + std::shared_ptr granule, const std::shared_ptr& locksManager) const override; + + virtual void DoActualize(const TInstant currentInstant) override { + if (currentInstant - LastActualization > TDuration::Seconds(180)) { + LastActualization = currentInstant; + } else { + return; + } + RefreshWeights(); + } + + virtual TOptimizationPriority DoGetUsefulMetric() const override { + AFL_VERIFY(LevelsByWeight.size()); + const ui64 levelPriority = LevelsByWeight.begin()->first; + if (levelPriority) { + return TOptimizationPriority::Critical(levelPriority); + } else { + return TOptimizationPriority::Zero(); + } + } + + virtual TString DoDebugString() const override { + TStringBuilder sb; + sb << "["; + for (auto&& i : Levels) { + sb << "{" << i->GetLevelId() << ":" << i->DebugString() << "},"; + } + sb << "]"; + return sb; + } + + virtual NJson::TJsonValue DoSerializeToJsonVisual() const override { + NJson::TJsonValue arr = NJson::JSON_MAP; + NJson::TJsonValue& arrLevels = arr.InsertValue("levels", NJson::JSON_ARRAY); + for (auto&& i : Levels) { + arrLevels.AppendValue(i->SerializeToJson()); + } + return arr; + } + +public: + virtual NArrow::NMerger::TIntervalPositions GetBucketPositions() const override { + NArrow::NMerger::TIntervalPositions result = Levels.back()->GetBucketPositions(PrimaryKeysSchema); + return result; + } + + TOptimizerPlanner( + const ui64 pathId, const std::shared_ptr& storagesManager, const std::shared_ptr& primaryKeysSchema); +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/ya.make new file mode 100644 index 000000000000..7eba1467e8b1 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +SRCS( + abstract.cpp + zero_level.cpp + common_level.cpp + GLOBAL optimizer.cpp + counters.cpp +) + +PEERDIR( + contrib/libs/apache/arrow + ydb/core/protos + ydb/core/formats/arrow + ydb/core/tx/columnshard/engines/changes/abstract +) + +END() diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/zero_level.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/zero_level.cpp new file mode 100644 index 000000000000..98c4d3f9bd58 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/zero_level.cpp @@ -0,0 +1,68 @@ +#include "zero_level.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +TCompactionTaskData TZeroLevelPortions::DoGetOptimizationTask() const { + AFL_VERIFY(Portions.size()); + TCompactionTaskData result(NextLevel->GetLevelId()); + for (auto&& i : Portions) { + result.AddCurrentLevelPortion( + i.GetPortion(), NextLevel->GetAffectedPortions(i.GetPortion()->IndexKeyStart(), i.GetPortion()->IndexKeyEnd()), true); + if (!result.CanTakeMore()) { +// result.SetStopSeparation(i.GetPortion()->IndexKeyStart()); + break; + } + } + if (result.CanTakeMore()) { + PredOptimization = TInstant::Now(); + } else { + PredOptimization = std::nullopt; + } + return result; +} + +ui64 TZeroLevelPortions::DoGetWeight() const { + if (!NextLevel || Portions.size() < 10) { + return 0; + } + if (TInstant::Now() - *PredOptimization < TDuration::Seconds(180)) { + if (PortionsInfo.GetCount() <= 100 || PortionsInfo.PredictPackedBlobBytes(GetPackKff()) < (1 << 20)) { + return 0; + } + } else { + if (PortionsInfo.PredictPackedBlobBytes(GetPackKff()) < (512 << 10)) { + return 0; + } + } + + THashSet portionIds; + const ui64 affectedRawBytes = + NextLevel->GetAffectedPortionBytes(Portions.begin()->GetPortion()->IndexKeyStart(), Portions.rbegin()->GetPortion()->IndexKeyEnd()); + /* + auto chain = + targetLevel->GetAffectedPortions(Portions.begin()->GetPortion()->IndexKeyStart(), Portions.rbegin()->GetPortion()->IndexKeyEnd()); + ui64 affectedRawBytes = 0; + if (chain) { + auto it = Portions.begin(); + auto itNext = chain->GetPortions().begin(); + while (it != Portions.end() && itNext != chain->GetPortions().end()) { + const auto& nextLevelPortion = *itNext; + if (nextLevelPortion->IndexKeyEnd() < it->GetPortion()->IndexKeyStart()) { + ++itNext; + } else if (it->GetPortion()->IndexKeyEnd() < nextLevelPortion->IndexKeyStart()) { + ++it; + } else { + if (portionIds.emplace(nextLevelPortion->GetPortionId()).second) { + affectedRawBytes += nextLevelPortion->GetTotalRawBytes(); + } + ++itNext; + } + } + } +*/ + + const ui64 mb = (affectedRawBytes + PortionsInfo.GetRawBytes()) / 1000000 + 1; + return 1000.0 * PortionsInfo.GetCount() * PortionsInfo.GetCount() / mb; +} + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/zero_level.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/zero_level.h new file mode 100644 index 000000000000..ba78cebbd436 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/zero_level.h @@ -0,0 +1,96 @@ +#pragma once +#include "abstract.h" +#include "counters.h" + +namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { + +class TZeroLevelPortions: public IPortionsLevel { +private: + using TBase = IPortionsLevel; + const TLevelCounters LevelCounters; + class TOrderedPortion { + private: + YDB_READONLY_DEF(std::shared_ptr, Portion); + + public: + TOrderedPortion(const std::shared_ptr& portion) + : Portion(portion) { + } + + bool operator==(const TOrderedPortion& item) const { + return item.Portion->GetPathId() == Portion->GetPathId() && item.Portion->GetPortionId() == Portion->GetPortionId(); + } + + bool operator<(const TOrderedPortion& item) const { + auto cmp = Portion->IndexKeyStart().CompareNotNull(item.Portion->IndexKeyStart()); + if (cmp == std::partial_ordering::equivalent) { + return Portion->GetPortionId() < item.Portion->GetPortionId(); + } else { + return cmp == std::partial_ordering::less; + } + } + }; + std::set Portions; + + virtual NArrow::NMerger::TIntervalPositions DoGetBucketPositions(const std::shared_ptr& /*pkSchema*/) const override { + return NArrow::NMerger::TIntervalPositions(); + } + + virtual std::optional DoGetAffectedPortions( + const NArrow::TReplaceKey& /*from*/, const NArrow::TReplaceKey& /*to*/) const override { + return std::nullopt; + } + + virtual ui64 DoGetAffectedPortionBytes(const NArrow::TReplaceKey& /*from*/, const NArrow::TReplaceKey& /*to*/) const override { + return 0; + } + + virtual void DoModifyPortions( + const std::vector>& add, const std::vector>& remove) override { + const bool constructionFlag = Portions.empty(); + if (constructionFlag) { + std::vector ordered; + ordered.reserve(add.size()); + for (auto&& i : add) { + ordered.emplace_back(i); + } + std::sort(ordered.begin(), ordered.end()); + AFL_VERIFY(std::unique(ordered.begin(), ordered.end()) == ordered.end()); + Portions = std::set(ordered.begin(), ordered.end()); + } + for (auto&& i : add) { + if (!constructionFlag) { + AFL_VERIFY(Portions.emplace(i).second); + } + PortionsInfo.AddPortion(i); + LevelCounters.Portions->AddPortion(i); + i->InitRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized, !NextLevel); + } + for (auto&& i : remove) { + AFL_VERIFY(Portions.erase(i)); + LevelCounters.Portions->RemovePortion(i); + PortionsInfo.RemovePortion(i); + } + } + + virtual bool IsLocked(const std::shared_ptr& locksManager) const override { + for (auto&& i : Portions) { + if (locksManager->IsLocked(*i.GetPortion())) { + return true; + } + } + return false; + } + + virtual ui64 DoGetWeight() const override; + + virtual TCompactionTaskData DoGetOptimizationTask() const override; + +public: + TZeroLevelPortions(const ui32 levelIdx, const std::shared_ptr& nextLevel, const TLevelCounters& levelCounters) + : TBase(levelIdx, nextLevel) + , LevelCounters(levelCounters) { + } +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/ya.make new file mode 100644 index 000000000000..8847ed7e2c81 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( +) + +PEERDIR( + ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner + ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor +) + +END() diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h index 56bddb8547fb..ed75da95f46c 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h @@ -35,7 +35,13 @@ class TPortionBuckets { const std::shared_ptr StoragesManager; std::map> Buckets; - std::map> BucketsByWeight; + struct TReverseComparator { + bool operator()(const i64 l, const i64 r) const { + return r < l; + } + }; + + std::map, TReverseComparator> BucketsByWeight; THashSet RatedBuckets; TInstant CurrentWeightInstant = TInstant::Now(); @@ -123,8 +129,8 @@ class TPortionBuckets { bool IsLocked(const std::shared_ptr& dataLocksManager) const { AFL_VERIFY(BucketsByWeight.size()); - AFL_VERIFY(BucketsByWeight.rbegin()->second.size()); - const auto bucket = BucketsByWeight.rbegin()->second.begin()->GetBucketVerified(); + AFL_VERIFY(BucketsByWeight.begin()->second.size()); + const auto bucket = BucketsByWeight.begin()->second.begin()->GetBucketVerified(); return bucket->IsLocked(dataLocksManager); } @@ -154,7 +160,7 @@ class TPortionBuckets { i64 GetWeight() const { AFL_VERIFY(BucketsByWeight.size()); - return BucketsByWeight.rbegin()->first; + return BucketsByWeight.begin()->first; } void RemovePortion(const std::shared_ptr& portion) { @@ -211,9 +217,9 @@ class TPortionBuckets { std::shared_ptr BuildOptimizationTask(std::shared_ptr granule, const std::shared_ptr& locksManager) const { AFL_VERIFY(BucketsByWeight.size()); - AFL_VERIFY(BucketsByWeight.rbegin()->first); - AFL_VERIFY(BucketsByWeight.rbegin()->second.size()); - const std::shared_ptr bucketForOptimization = BucketsByWeight.rbegin()->second.begin()->GetBucketVerified(); + AFL_VERIFY(BucketsByWeight.begin()->first); + AFL_VERIFY(BucketsByWeight.begin()->second.size()); + const std::shared_ptr bucketForOptimization = BucketsByWeight.begin()->second.begin()->GetBucketVerified(); auto it = Buckets.find(bucketForOptimization->GetStart()); AFL_VERIFY(it != Buckets.end()); ++it; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make index e5b1c1905986..c7739f8c8782 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make @@ -3,6 +3,7 @@ LIBRARY() PEERDIR( ydb/core/tx/columnshard/engines/storage/optimizer/abstract ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets + ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets ) diff --git a/ydb/core/tx/columnshard/engines/storage/storage.cpp b/ydb/core/tx/columnshard/engines/storage/storage.cpp deleted file mode 100644 index 85252e68069f..000000000000 --- a/ydb/core/tx/columnshard/engines/storage/storage.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "storage.h" diff --git a/ydb/core/tx/columnshard/engines/storage/storage.h b/ydb/core/tx/columnshard/engines/storage/storage.h deleted file mode 100644 index 7e9fe7bb73a8..000000000000 --- a/ydb/core/tx/columnshard/engines/storage/storage.h +++ /dev/null @@ -1,2 +0,0 @@ -#pragma once -#include "granule/storage.h" diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index 238638d9596a..699cd0bebc66 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -1,32 +1,34 @@ #include "broken_blobs.h" -#include -#include +#include #include +#include +#include +#include #include -#include - - namespace NKikimr::NOlap::NNormalizer::NBrokenBlobs { -class TNormalizerResult : public INormalizerChanges { +class TNormalizerResult: public INormalizerChanges { THashMap> BrokenPortions; std::shared_ptr> Schemas; + public: - TNormalizerResult(THashMap>&& portions, const std::shared_ptr>& schemas) + TNormalizerResult( + THashMap>&& portions, const std::shared_ptr>& schemas) : BrokenPortions(std::move(portions)) - , Schemas(schemas) - {} + , Schemas(schemas) { + } bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& normController) const override { NOlap::TBlobManagerDb blobManagerDb(txc.DB); - + TDbWrapper db(txc.DB, nullptr); for (auto&& [_, portionInfo] : BrokenPortions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "portion_removed_as_broken")("portion_id", portionInfo->GetAddress().DebugString()); + AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "portion_removed_as_broken")( + "portion_id", portionInfo->GetAddress().DebugString()); portionInfo->SetRemoveSnapshot(TSnapshot(1, 1)); portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false); } @@ -61,25 +63,49 @@ class TReadTask: public NOlap::NBlobOperations::NRead::ITask { std::shared_ptr> Schemas; THashMap>> PortionsByBlobId; THashMap> BrokenPortions; + public: TReadTask(const TNormalizationContext& nCtx, const std::vector>& actions, - std::shared_ptr> schemas, THashMap < TString, THashMap>>&& portionsByBlobId) + std::shared_ptr> schemas, + THashMap>>&& portionsByBlobId) : TBase(actions, "CS::NORMALIZER") , NormContext(nCtx) , Schemas(std::move(schemas)) - , PortionsByBlobId(portionsByBlobId) - { + , PortionsByBlobId(portionsByBlobId) { } protected: virtual void DoOnDataReady(const std::shared_ptr& /*resourcesGuard*/) override { + NBlobOperations::NRead::TCompositeReadBlobs blobs = ExtractBlobsData(); + + THashSet readyPortions; + for (auto&& i : BrokenPortions) { + readyPortions.emplace(i.first); + } + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("event", "broken_data_found"); + for (auto&& i : PortionsByBlobId) { + for (auto&& [_, p] : i.second) { + if (readyPortions.emplace(p->GetPortionId()).second) { + auto it = Schemas->find(p->GetPortionId()); + AFL_VERIFY(it != Schemas->end()); + auto restored = TReadPortionInfoWithBlobs::RestorePortion(*p, blobs, it->second->GetIndexInfo()); + auto restoredBatch = restored.RestoreBatch(*it->second, *it->second, {}); + if (restoredBatch.IsFail()) { + AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("portion", p->DebugString())("fail", restoredBatch.GetErrorMessage()); + BrokenPortions.emplace(p->GetPortionId(), p); + } + } + } + } + auto changes = std::make_shared(std::move(BrokenPortions), Schemas); - TActorContext::AsActorContext().Send(NormContext.GetShardActor(), std::make_unique(changes)); + TActorContext::AsActorContext().Send( + NormContext.GetShardActor(), std::make_unique(changes)); } virtual bool DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) override { - NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("blob_id", range.GetBlobId().ToStringNew()) - ("error", status.GetErrorMessage())("status", status.GetStatus())("event", "broken_blob_found")("storage_id", storageId); + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("blob_id", range.GetBlobId().ToStringNew())( + "error", status.GetErrorMessage())("status", status.GetStatus())("event", "broken_blob_found")("storage_id", storageId); AFL_VERIFY(status.GetStatus() == NKikimrProto::EReplyStatus::NODATA)("status", status.GetStatus()); auto itStorage = PortionsByBlobId.find(storageId); AFL_VERIFY(itStorage != PortionsByBlobId.end()); @@ -95,16 +121,18 @@ class TReadTask: public NOlap::NBlobOperations::NRead::ITask { }; class TBrokenBlobsTask: public INormalizerTask { - THashMap> Blobs; + THashMap> Blobs; THashMap>> PortionsByBlobId; const std::shared_ptr> Schemas; + public: - TBrokenBlobsTask(THashMap>&& blobs, THashMap>>&& portionsByBlobId, + TBrokenBlobsTask(THashMap>&& blobs, + THashMap>>&& portionsByBlobId, const std::shared_ptr>& schemas) : Blobs(std::move(blobs)) , PortionsByBlobId(portionsByBlobId) - , Schemas(schemas) - {} + , Schemas(schemas) { + } void Start(const TNormalizationController& controller, const TNormalizationContext& nCtx) override { ui64 memSize = 0; @@ -113,52 +141,49 @@ class TBrokenBlobsTask: public INormalizerTask { auto op = controller.GetStoragesManager()->GetOperatorVerified(storageId); actions.emplace_back(op->StartReadingAction(NBlobOperations::EConsumer::NORMALIZER)); for (auto&& b : data) { - memSize += b.BlobSize(); - actions.back()->AddRange(TBlobRange::FromBlobId(b)); + memSize += b.GetBlobSize(); + actions.back()->AddRange(b); } } NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( nCtx.GetResourceSubscribeActor(), std::make_shared( - std::make_shared(nCtx, actions, Schemas, std::move(PortionsByBlobId)), 0, memSize, "CS::NORMALIZER", controller.GetTaskSubscription())); + std::make_shared(nCtx, actions, Schemas, std::move(PortionsByBlobId)), 0, memSize, + "CS::NORMALIZER", controller.GetTaskSubscription())); } }; - bool TNormalizer::CheckPortion(const NColumnShard::TTablesManager& /*tablesManager*/, const TPortionInfo& /*portionInfo*/) const { return false; } -INormalizerTask::TPtr TNormalizer::BuildTask(std::vector>&& portions, std::shared_ptr> schemas) const { - THashMap> blobIds; +INormalizerTask::TPtr TNormalizer::BuildTask( + std::vector>&& portions, std::shared_ptr> schemas) const { + THashMap> blobIds; THashMap>> portionByBlobId; for (auto&& portion : portions) { auto schemaPtr = schemas->FindPtr(portion->GetPortionId()); - THashMap> blobsByStorage; - portion->FillBlobIdsByStorage(blobsByStorage, schemaPtr->get()->GetIndexInfo()); + THashMap> blobsByStorage; + portion->FillBlobRangesByStorage(blobsByStorage, schemaPtr->get()->GetIndexInfo()); if (blobsByStorage.size() > 1 || !blobsByStorage.contains(NBlobOperations::TGlobal::DefaultStorageId)) { continue; } - for (auto&& i: blobsByStorage) { + for (auto&& i : blobsByStorage) { + AFL_VERIFY(i.first == NBlobOperations::TGlobal::DefaultStorageId)("details", "Invalid storage for normalizer")( + "storage_id", i.first); for (auto&& b : i.second) { - AFL_VERIFY(portionByBlobId[i.first].emplace(b, portion).second); + portionByBlobId[i.first].emplace(b.BlobId, portion); + AFL_VERIFY(blobIds[i.first].emplace(b).second); } } } - for (auto&& [storageId, blobs] : portionByBlobId) { - AFL_VERIFY(storageId == NBlobOperations::TGlobal::DefaultStorageId)("details", "Invalid storage for normalizer")("storage_id", storageId); - for (auto&& [blobId, _] : blobs) { - AFL_VERIFY(blobIds[storageId].emplace(blobId).second); - } - } if (blobIds.empty()) { return nullptr; } return std::make_shared(std::move(blobIds), std::move(portionByBlobId), schemas); } - TConclusion TNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { +TConclusion TNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { return true; } - -} +} // namespace NKikimr::NOlap::NNormalizer::NBrokenBlobs diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp index f42f38061e45..6901760d5e55 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp @@ -60,7 +60,7 @@ class TRowsAndBytesChangesTask: public NConveyor::ITask { TPortionInfo::TAssembleBlobInfo assembleBlob(blobData); assembleBlob.SetExpectedRecordsCount(chunkInfo.GetRecordsCount()); - auto batch = assembleBlob.BuildRecordBatch(*columnLoader); + auto batch = assembleBlob.BuildRecordBatch(*columnLoader).DetachResult(); Y_ABORT_UNLESS(!!batch); chunkInfo.MutableUpdate().SetNumRows(batch->GetRecordsCount()); diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp index 63cea8b19952..cd903e567fc4 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp @@ -1,12 +1,14 @@ #include "normalizer.h" -#include -#include #include +#include +#include +#include namespace NKikimr::NOlap { -TConclusion> TPortionsNormalizerBase::DoInit(const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) { +TConclusion> TPortionsNormalizerBase::DoInit( + const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) { auto initRes = DoInitImpl(controller, txc); if (initRes.IsFail()) { @@ -35,6 +37,12 @@ TConclusion> TPortionsNormalizerBase::DoInit( THashMap portions; auto schemas = std::make_shared>(); + { + auto conclusion = InitPortions(tablesManager, db, portions); + if (conclusion.IsFail()) { + return conclusion; + } + } { auto conclusion = InitColumns(tablesManager, db, portions); if (conclusion.IsFail()) { @@ -58,7 +66,7 @@ TConclusion> TPortionsNormalizerBase::DoInit( ui64 brokenPortioncCount = 0; for (auto&& portionConstructor : portions) { auto portionInfo = std::make_shared(portionConstructor.second.Build(false)); - if (CheckPortion(tablesManager, *portionInfo)) { + if (CheckPortion(tablesManager, *portionInfo)) { continue; } ++brokenPortioncCount; @@ -83,6 +91,21 @@ TConclusion> TPortionsNormalizerBase::DoInit( return tasks; } +TConclusionStatus TPortionsNormalizerBase::InitPortions( + const NColumnShard::TTablesManager& tablesManager, NIceDb::TNiceDb& db, THashMap& constructors) { + TDbWrapper wrapper(db.GetDatabase(), nullptr); + if (!wrapper.LoadPortions([&](TPortionInfoConstructor&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + const TIndexInfo& indexInfo = + portion.GetSchema(tablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex())->GetIndexInfo(); + AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo)); + const ui64 portionId = portion.GetPortionIdVerified(); + AFL_VERIFY(constructors.emplace(portionId, std::move(portion)).second); + })) { + return TConclusionStatus::Fail("repeated read db"); + } + return TConclusionStatus::Success(); +} + TConclusionStatus TPortionsNormalizerBase::InitColumns( const NColumnShard::TTablesManager& tablesManager, NIceDb::TNiceDb& db, THashMap& portions) { using namespace NColumnShard; @@ -149,4 +172,4 @@ TConclusionStatus TPortionsNormalizerBase::InitIndexes(NIceDb::TNiceDb& db, THas return TConclusionStatus::Success(); } -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.h b/ydb/core/tx/columnshard/normalizer/portion/normalizer.h index 8c23395eba0b..e8cae47d5cda 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.h +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.h @@ -1,19 +1,16 @@ #pragma once -#include -#include #include - -#include -#include #include #include - #include - +#include +#include +#include +#include namespace NKikimr::NColumnShard { - class TTablesManager; +class TTablesManager; } namespace NKikimr::NOlap { @@ -26,18 +23,19 @@ class TReadPortionsTask: public NOlap::NBlobOperations::NRead::ITask { TNormalizationContext NormContext; public: - TReadPortionsTask(const TNormalizationContext& nCtx, const std::vector>& actions, typename TConveyorTask::TDataContainer&& data, std::shared_ptr> schemas) + TReadPortionsTask(const TNormalizationContext& nCtx, const std::vector>& actions, + typename TConveyorTask::TDataContainer&& data, std::shared_ptr> schemas) : TBase(actions, "CS::NORMALIZER") , Data(std::move(data)) , Schemas(std::move(schemas)) - , NormContext(nCtx) - { + , NormContext(nCtx) { } protected: virtual void DoOnDataReady(const std::shared_ptr& resourcesGuard) override { NormContext.SetResourcesGuard(resourcesGuard); - std::shared_ptr task = std::make_shared(std::move(ExtractBlobsData()), NormContext, std::move(Data), Schemas); + std::shared_ptr task = + std::make_shared(std::move(ExtractBlobsData()), NormContext, std::move(Data), Schemas); NConveyor::TCompServiceOperator::SendTaskToExecute(task); } @@ -51,18 +49,20 @@ class TReadPortionsTask: public NOlap::NBlobOperations::NRead::ITask { }; template -class TPortionsNormalizerTask : public INormalizerTask { +class TPortionsNormalizerTask: public INormalizerTask { typename TConveyorTask::TDataContainer Package; std::shared_ptr> Schemas; + public: TPortionsNormalizerTask(typename TConveyorTask::TDataContainer&& package) - : Package(std::move(package)) - {} + : Package(std::move(package)) { + } - TPortionsNormalizerTask(typename TConveyorTask::TDataContainer&& package, const std::shared_ptr> schemas) + TPortionsNormalizerTask( + typename TConveyorTask::TDataContainer&& package, const std::shared_ptr> schemas) : Package(std::move(package)) - , Schemas(schemas) - {} + , Schemas(schemas) { + } void Start(const TNormalizationController& controller, const TNormalizationContext& nCtx) override { controller.GetCounters().CountObjects(Package.size()); @@ -72,19 +72,22 @@ class TPortionsNormalizerTask : public INormalizerTask { TConveyorTask::FillBlobRanges(readingAction, data); memSize += TConveyorTask::GetMemSize(data); } - std::vector> actions = {readingAction}; + std::vector> actions = { readingAction }; NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - nCtx.GetResourceSubscribeActor(),std::make_shared( - std::make_shared>(nCtx, actions, std::move(Package), Schemas), 1, memSize, "CS::NORMALIZER", controller.GetTaskSubscription())); + nCtx.GetResourceSubscribeActor(), std::make_shared( + std::make_shared>(nCtx, actions, std::move(Package), Schemas), + 1, memSize, "CS::NORMALIZER", controller.GetTaskSubscription())); } }; -class TPortionsNormalizerBase : public TNormalizationController::INormalizerComponent { +class TPortionsNormalizerBase: public TNormalizationController::INormalizerComponent { public: TPortionsNormalizerBase(const TNormalizationController::TInitContext& info) - : DsGroupSelector(info.GetStorageInfo()) - {} + : DsGroupSelector(info.GetStorageInfo()) { + } + TConclusionStatus InitPortions( + const NColumnShard::TTablesManager& tablesManager, NIceDb::TNiceDb& db, THashMap& portions); TConclusionStatus InitColumns( const NColumnShard::TTablesManager& tablesManager, NIceDb::TNiceDb& db, THashMap& portions); TConclusionStatus InitIndexes(NIceDb::TNiceDb& db, THashMap& portions); @@ -93,8 +96,9 @@ class TPortionsNormalizerBase : public TNormalizationController::INormalizerComp const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) override final; protected: - virtual INormalizerTask::TPtr BuildTask(std::vector>&& portions, std::shared_ptr> schemas) const = 0; - virtual TConclusion DoInitImpl(const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) = 0; + virtual INormalizerTask::TPtr BuildTask( + std::vector>&& portions, std::shared_ptr> schemas) const = 0; + virtual TConclusion DoInitImpl(const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) = 0; virtual bool CheckPortion(const NColumnShard::TTablesManager& tablesManager, const TPortionInfo& /*portionInfo*/) const = 0; @@ -106,4 +110,4 @@ class TPortionsNormalizerBase : public TNormalizationController::INormalizerComp NColumnShard::TBlobGroupSelector DsGroupSelector; }; -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h index 6e67abea8730..0fb190c0f7fc 100644 --- a/ydb/core/tx/columnshard/operations/write.h +++ b/ydb/core/tx/columnshard/operations/write.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -45,7 +46,7 @@ enum class EOperationBehaviour : ui32 { NoTxWrite = 6 }; -class TWriteOperation { +class TWriteOperation: public TMonitoringObjectsCounter { private: YDB_READONLY(ui64, PathId, 0); YDB_READONLY(EOperationStatus, Status, EOperationStatus::Draft); @@ -66,8 +67,8 @@ class TWriteOperation { const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool writePortions); - void Start(TColumnShard& owner, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, - const NOlap::TWritingContext& context); + void Start( + TColumnShard& owner, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const NOlap::TWritingContext& context); void OnWriteFinish( NTabletFlatExecutor::TTransactionContext& txc, const std::vector& insertWriteIds, const bool ephemeralFlag); void CommitOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc, const NOlap::TSnapshot& snapshot) const; diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 6d657cebcb6c..007e3c966500 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -153,7 +153,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { TSchemaPreset::TSchemaPresetVersionInfo info; Y_ABORT_UNLESS(info.ParseFromString(rowset.GetValue())); - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "load_preset")("preset_id", id)("snapshot", version)("version", info.HasSchema() ? info.GetSchema().GetVersion() : -1); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "load_preset")("preset_id", id)("snapshot", version)("version", info.HasSchema() ? info.GetSchema().GetVersion() : -1); preset.AddVersion(version, info); if (!rowset.Next()) { return false; diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index a0716bd0925b..cb8349bb4e70 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -2348,11 +2348,11 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } } Cerr << "compacted=" << sumCompactedRows << ";inserted=" << sumInsertedRows << ";expected=" << fullNumRows << ";" << Endl; - if (!sumInsertedRows && sumCompactedRows == fullNumRows) { + if (sumCompactedRows && sumInsertedRows + sumCompactedRows == fullNumRows) { success = true; RebootTablet(runtime, TTestTxConfig::TxTablet0, sender); UNIT_ASSERT(sumCompactedRows < sumCompactedBytes); - UNIT_ASSERT(sumInsertedBytes == 0); + UNIT_ASSERT(sumInsertedRows <= sumInsertedBytes); } else { Wakeup(runtime, sender, TTestTxConfig::TxTablet0); } diff --git a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp index aec0d0d8cd1a..d4ec6949f66f 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp @@ -218,8 +218,26 @@ class TEmptyPortionsCleaner: public NYDBTest::ILocalDBModifier { using namespace NColumnShard; NIceDb::TNiceDb db(txc.DB); for (size_t pathId = 100; pathId != 299; ++pathId) { - for (size_t portionId = 1000; portionId != 1199; ++portionId) { - db.Table().Key(pathId, portionId).Update(); + for (size_t portionId = pathId * 100000 + 1000; portionId != pathId * 100000 + 1199; ++portionId) { + NKikimrTxColumnShard::TIndexPortionMeta metaProto; + metaProto.SetDeletionsCount(0); + metaProto.SetIsInserted(true); + + const auto schema = std::make_shared( + arrow::FieldVector({ std::make_shared("key1", arrow::uint64()), std::make_shared("key2", arrow::uint64()) })); + auto batch = NArrow::MakeEmptyBatch(schema, 1); + NArrow::TFirstLastSpecialKeys keys(batch); + metaProto.SetPrimaryKeyBorders(keys.SerializePayloadToString()); + metaProto.MutableRecordSnapshotMin()->SetPlanStep(0); + metaProto.MutableRecordSnapshotMin()->SetTxId(0); + metaProto.MutableRecordSnapshotMax()->SetPlanStep(0); + metaProto.MutableRecordSnapshotMax()->SetTxId(0); + db.Table() + .Key(pathId, portionId) + .Update(NIceDb::TUpdate(1), + NIceDb::TUpdate(metaProto.SerializeAsString()), + NIceDb::TUpdate(10), + NIceDb::TUpdate(10)); } } } diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index f1f4df107ffe..f20664cf838e 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -60,6 +60,7 @@ PEERDIR( ydb/core/tx/columnshard/blobs_action/storages_manager ydb/core/tx/tiering ydb/core/tx/conveyor/usage + ydb/core/tx/priorities/service ydb/core/tx/tracing ydb/core/tx/long_tx_service/public ydb/core/util diff --git a/ydb/core/tx/priorities/service/counters.cpp b/ydb/core/tx/priorities/service/counters.cpp new file mode 100644 index 000000000000..f0448e9f02f3 --- /dev/null +++ b/ydb/core/tx/priorities/service/counters.cpp @@ -0,0 +1,19 @@ +#include "counters.h" + +namespace NKikimr::NPrioritiesQueue { + + TCounters::TCounters(const TString& queueName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals) + : TBase("Priorities/" + queueName, baseSignals) + , UsedCount(TBase::GetValue("UsedCount")) + , Ask(TBase::GetDeriviative("Ask")) + , AskMax(TBase::GetDeriviative("AskMax")) + , Free(TBase::GetDeriviative("Free")) + , FreeNoClient(TBase::GetDeriviative("FreeNoClient")) + , Register(TBase::GetDeriviative("Register")) + , Unregister(TBase::GetDeriviative("Unregister")) + , QueueSize(TBase::GetValue("QueueSize")) + , Clients(TBase::GetDeriviative("Clients")) + , Limit(TBase::GetValue("Limit")) { +} + +} diff --git a/ydb/core/tx/priorities/service/counters.h b/ydb/core/tx/priorities/service/counters.h new file mode 100644 index 000000000000..7eb2202d9556 --- /dev/null +++ b/ydb/core/tx/priorities/service/counters.h @@ -0,0 +1,27 @@ +#pragma once +#include + +#include + +namespace NKikimr::NPrioritiesQueue { + +class TCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + +public: + const ::NMonitoring::TDynamicCounters::TCounterPtr UsedCount; + const ::NMonitoring::TDynamicCounters::TCounterPtr Ask; + const ::NMonitoring::TDynamicCounters::TCounterPtr AskMax; + const ::NMonitoring::TDynamicCounters::TCounterPtr Free; + const ::NMonitoring::TDynamicCounters::TCounterPtr FreeNoClient; + const ::NMonitoring::TDynamicCounters::TCounterPtr Register; + const ::NMonitoring::TDynamicCounters::TCounterPtr Unregister; + const ::NMonitoring::TDynamicCounters::TCounterPtr QueueSize; + const ::NMonitoring::TDynamicCounters::TCounterPtr Clients; + const ::NMonitoring::TDynamicCounters::TCounterPtr Limit; + + TCounters(const TString& queueName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals); +}; + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/service/manager.cpp b/ydb/core/tx/priorities/service/manager.cpp new file mode 100644 index 000000000000..219c39de1df6 --- /dev/null +++ b/ydb/core/tx/priorities/service/manager.cpp @@ -0,0 +1,103 @@ +#include "manager.h" + +#include +#include + +#include + +namespace NKikimr::NPrioritiesQueue { + +void TManager::AllocateNext() { + while (WaitingQueue.size() && UsedCount + WaitingQueue.begin()->second.GetSize() <= Config.GetLimit()) { + auto& waitRequest = WaitingQueue.begin()->second; + auto it = Clients.find(waitRequest.GetClientId()); + AFL_VERIFY(it != Clients.end()); + UsedCount += waitRequest.GetSize(); + it->second.MutableCount() += waitRequest.GetSize(); + it->second.SetLastPriority(std::nullopt); + waitRequest.GetRequest()->OnAllocated(std::make_shared(ServiceActorId, waitRequest.GetClientId(), waitRequest.GetSize())); + WaitingQueue.erase(WaitingQueue.begin()); + } + Counters->QueueSize->Set(WaitingQueue.size()); + Counters->UsedCount->Set(UsedCount); +} + +void TManager::RemoveFromQueue(const TClientStatus& client) { + if (!client.GetLastPriority()) { + return; + } + AFL_VERIFY(WaitingQueue.erase(*client.GetLastPriority())); + Counters->QueueSize->Set(WaitingQueue.size()); +} + +void TManager::Free(const ui64 clientId, const ui32 count) { + auto it = Clients.find(clientId); + if (it == Clients.end()) { + Counters->FreeNoClient->Inc(); + return; + } + Counters->Free->Inc(); + AFL_VERIFY(it->second.GetCount() <= UsedCount); + AFL_VERIFY(count <= it->second.GetCount()); + it->second.MutableCount() -= count; + UsedCount -= count; + AllocateNext(); +} + +TManager::TClientStatus& TManager::GetClientVerified(const ui64 clientId) { + auto it = Clients.find(clientId); + AFL_VERIFY(it != Clients.end()); + return it->second; +} + +void TManager::Ask(const ui64 clientId, const ui32 count, const std::shared_ptr& request, const ui64 extPriority) { + AFL_VERIFY(request); + Counters->Ask->Inc(); + AskImpl(GetClientVerified(clientId), extPriority, TAskRequest(clientId, request, count)); +} + +void TManager::AskMax(const ui64 clientId, const ui32 count, const std::shared_ptr& request, const ui64 extPriority) { + AFL_VERIFY(request); + Counters->AskMax->Inc(); + auto& client = GetClientVerified(clientId); + if (client.GetLastPriority() && extPriority < client.GetLastPriority()->GetExternalPriority()) { + return; + } + AskImpl(client, extPriority, TAskRequest(clientId, request, count)); +} + +void TManager::AskImpl(TClientStatus& client, const ui64 extPriority, TAskRequest&& request) { + RemoveFromQueue(client); + AFL_VERIFY(request.GetSize() <= Config.GetLimit())("requested", request.GetSize())("limit", Config.GetLimit()); + TPriority priority(extPriority); + client.SetLastPriority(priority); + AFL_VERIFY(WaitingQueue.emplace(priority, std::move(request)).second); + AllocateNext(); +} + +void TManager::RegisterClient(const ui64 clientId) { + Counters->Register->Inc(); + AFL_VERIFY(Clients.emplace(clientId, clientId).second); + Counters->Clients->Set(Clients.size()); +} + +void TManager::UnregisterClient(const ui64 clientId) { + Counters->Unregister->Inc(); + auto it = Clients.find(clientId); + AFL_VERIFY(it != Clients.end()); + AFL_VERIFY(it->second.GetCount() <= UsedCount); + UsedCount -= it->second.GetCount(); + RemoveFromQueue(it->second); + Clients.erase(it); + AllocateNext(); + Counters->Clients->Set(Clients.size()); +} + +TManager::TManager(const std::shared_ptr& counters, const TConfig& config, const NActors::TActorId& serviceActorId) + : Counters(counters) + , Config(config) + , ServiceActorId(serviceActorId) { + AFL_VERIFY(Counters); +} + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/service/manager.h b/ydb/core/tx/priorities/service/manager.h new file mode 100644 index 000000000000..6c37c61c2a8e --- /dev/null +++ b/ydb/core/tx/priorities/service/manager.h @@ -0,0 +1,91 @@ +#pragma once +#include "counters.h" + +#include +#include + +#include + +namespace NKikimr::NPrioritiesQueue { + +class TManager { +private: + std::shared_ptr Counters; + const TConfig Config; + const NActors::TActorId ServiceActorId; + + class TPriority { + private: + ui64 ExternalPriority; + static inline TAtomicCounter Counter = 0; + ui64 Sequence = Counter.Inc(); + + public: + TPriority(const ui64 priority) + : ExternalPriority(priority) { + } + + ui64 GetExternalPriority() const { + return ExternalPriority; + } + + bool operator<(const TPriority& item) const { + if (item.ExternalPriority < ExternalPriority) { + return true; + } else if (ExternalPriority < item.ExternalPriority) { + return false; + } else { + return item.Sequence < Sequence; + } + } + }; + + class TClientStatus: TNonCopyable { + private: + YDB_READONLY(ui64, ClientId, 0); + YDB_ACCESSOR(ui32, Count, 0); + YDB_ACCESSOR_DEF(std::optional, LastPriority); + + public: + TClientStatus(const ui64 clientId) + : ClientId(clientId) { + } + }; + + THashMap Clients; + + class TAskRequest { + private: + YDB_READONLY(ui64, ClientId, 0); + YDB_READONLY_DEF(std::shared_ptr, Request); + YDB_READONLY(ui32, Size, 0); + + public: + TAskRequest(const ui64 clientId, const std::shared_ptr& request, const ui32 size) + : ClientId(clientId) + , Request(request) + , Size(size) { + } + }; + + ui32 UsedCount = 0; + std::map WaitingQueue; + + void AllocateNext(); + + void RemoveFromQueue(const TClientStatus& client); + void AskImpl(TClientStatus& client, const ui64 extPriority, TAskRequest&& request); + TClientStatus& GetClientVerified(const ui64 clientId); + +public: + TManager(const std::shared_ptr& counters, const TConfig& config, const NActors::TActorId& serviceActorId); + + void Ask(const ui64 client, const ui32 count, const std::shared_ptr& request, const ui64 extPriority); + void AskMax(const ui64 client, const ui32 count, const std::shared_ptr& request, const ui64 extPriority); + void Free(const ui64 client, const ui32 count); + + void RegisterClient(const ui64 clientId); + void UnregisterClient(const ui64 clientId); +}; + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/service/service.cpp b/ydb/core/tx/priorities/service/service.cpp new file mode 100644 index 000000000000..be41623e7f8c --- /dev/null +++ b/ydb/core/tx/priorities/service/service.cpp @@ -0,0 +1,19 @@ +#include "service.h" +#include +#include + +namespace NKikimr::NPrioritiesQueue { + +void TDistributor::Bootstrap() { + Counters->Limit->Set(Config.GetLimit()); + Manager = std::make_unique(Counters, Config, SelfId()); + Become(&TDistributor::StateMain); +} + +TDistributor::TDistributor(const TConfig& config, const TString& queueName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals) + : Counters(std::make_shared(queueName, baseSignals)) + , QueueName(queueName) + , Config(config) { +} + +} diff --git a/ydb/core/tx/priorities/service/service.h b/ydb/core/tx/priorities/service/service.h new file mode 100644 index 000000000000..663f3519d971 --- /dev/null +++ b/ydb/core/tx/priorities/service/service.h @@ -0,0 +1,59 @@ +#pragma once +#include "counters.h" +#include "manager.h" + +#include + +#include +#include + +namespace NKikimr::NPrioritiesQueue { + +class TDistributor: public TActorBootstrapped { +private: + std::shared_ptr Counters; + const TString QueueName = "common"; + const TConfig Config; + std::unique_ptr Manager; + + void Handle(TEvExecution::TEvRegisterClient::TPtr& ev) { + Manager->RegisterClient(ev->Get()->GetClientId()); + } + + void Handle(TEvExecution::TEvUnregisterClient::TPtr& ev) { + Manager->UnregisterClient(ev->Get()->GetClientId()); + } + + void Handle(TEvExecution::TEvAsk::TPtr& ev) { + Manager->Ask(ev->Get()->GetClientId(), ev->Get()->GetCount(), ev->Get()->GetRequest(), ev->Get()->GetPriority()); + } + + void Handle(TEvExecution::TEvAskMax::TPtr& ev) { + Manager->AskMax(ev->Get()->GetClientId(), ev->Get()->GetCount(), ev->Get()->GetRequest(), ev->Get()->GetPriority()); + } + + void Handle(TEvExecution::TEvFree::TPtr& ev) { + Manager->Free(ev->Get()->GetClientId(), ev->Get()->GetCount()); + } + +public: + STATEFN(StateMain) { + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("name", QueueName)("actor_id", SelfId()); + switch (ev->GetTypeRewrite()) { + hFunc(TEvExecution::TEvRegisterClient, Handle); + hFunc(TEvExecution::TEvUnregisterClient, Handle); + hFunc(TEvExecution::TEvAsk, Handle); + hFunc(TEvExecution::TEvAskMax, Handle); + hFunc(TEvExecution::TEvFree, Handle); + default: + AFL_ERROR(NKikimrServices::TX_PRIORITIES_QUEUE)("problem", "unexpected event for task executor")("ev_type", ev->GetTypeName()); + break; + } + } + + TDistributor(const TConfig& config, const TString& queueName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals); + + void Bootstrap(); +}; + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/service/ya.make b/ydb/core/tx/priorities/service/ya.make new file mode 100644 index 000000000000..172fd7e35eda --- /dev/null +++ b/ydb/core/tx/priorities/service/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + service.cpp + manager.cpp + counters.cpp +) + +PEERDIR( + ydb/core/tx/priorities/usage + ydb/core/protos +) + +END() diff --git a/ydb/core/tx/priorities/usage/abstract.cpp b/ydb/core/tx/priorities/usage/abstract.cpp new file mode 100644 index 000000000000..f62d66331ebd --- /dev/null +++ b/ydb/core/tx/priorities/usage/abstract.cpp @@ -0,0 +1,24 @@ +#include "abstract.h" +#include "events.h" + +#include +#include + +namespace NKikimr::NPrioritiesQueue { + +TAllocationGuard::~TAllocationGuard() { + if (!Released) { + Release(); + } +} + +void TAllocationGuard::Release() { + AFL_VERIFY(!Released); + if (TlsActivationContext) { + auto& context = NActors::TActorContext::AsActorContext(); + context.Send(ServiceActorId, new TEvExecution::TEvFree(ClientId, Count)); + } + Released = true; +} + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/usage/abstract.h b/ydb/core/tx/priorities/usage/abstract.h new file mode 100644 index 000000000000..8d5a46e3884d --- /dev/null +++ b/ydb/core/tx/priorities/usage/abstract.h @@ -0,0 +1,37 @@ +#pragma once +#include + +namespace NKikimr::NPrioritiesQueue { + +class TAllocationGuard { +private: + const NActors::TActorId ServiceActorId; + const ui64 ClientId; + const ui32 Count; + bool Released = false; + +public: + TAllocationGuard(const NActors::TActorId& serviceActorId, const ui64 clientId, const ui32 count) + : ServiceActorId(serviceActorId) + , ClientId(clientId) + , Count(count) { + } + + ~TAllocationGuard(); + + void Release(); +}; + +class IRequest { +protected: + virtual void DoOnAllocated(const std::shared_ptr& guard) = 0; + +public: + virtual ~IRequest() = default; + + void OnAllocated(const std::shared_ptr& guard) { + return DoOnAllocated(guard); + } +}; + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/usage/config.cpp b/ydb/core/tx/priorities/usage/config.cpp new file mode 100644 index 000000000000..6b2647ac2269 --- /dev/null +++ b/ydb/core/tx/priorities/usage/config.cpp @@ -0,0 +1,25 @@ +#include "config.h" +#include + +namespace NKikimr::NPrioritiesQueue { + +bool TConfig::DeserializeFromProto(const NKikimrConfig::TPrioritiesQueueConfig& config) { + if (!config.HasEnabled()) { + EnabledFlag = true; + } else { + EnabledFlag = config.GetEnabled(); + } + if (config.HasLimit()) { + Limit = config.GetLimit(); + } + return true; +} + +TString TConfig::DebugString() const { + TStringBuilder sb; + sb << "Limit=" << Limit << ";"; + sb << "Enabled=" << EnabledFlag << ";"; + return sb; +} + +} diff --git a/ydb/core/tx/priorities/usage/config.h b/ydb/core/tx/priorities/usage/config.h new file mode 100644 index 000000000000..b423ec08a24f --- /dev/null +++ b/ydb/core/tx/priorities/usage/config.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +namespace NKikimr::NPrioritiesQueue { + +class TConfig { +private: + YDB_READONLY(ui32, Limit, 32); + YDB_READONLY_FLAG(Enabled, true); +public: + bool DeserializeFromProto(const NKikimrConfig::TPrioritiesQueueConfig& config); + TString DebugString() const; +}; + +} diff --git a/ydb/core/tx/priorities/usage/events.cpp b/ydb/core/tx/priorities/usage/events.cpp new file mode 100644 index 000000000000..f96de64e62f6 --- /dev/null +++ b/ydb/core/tx/priorities/usage/events.cpp @@ -0,0 +1,25 @@ +#include "events.h" + +#include + +namespace NKikimr::NPrioritiesQueue { + +TEvExecution::TEvAsk::TEvAsk(const ui64 clientId, const ui32 count, const std::shared_ptr& request, const ui64 priority) + : ClientId(clientId) + , Count(count) + , Request(request) + , Priority(priority) { + AFL_VERIFY(Request); + AFL_VERIFY(Count); +} + +TEvExecution::TEvAskMax::TEvAskMax(const ui64 clientId, const ui32 count, const std::shared_ptr& request, const ui64 priority) + : ClientId(clientId) + , Count(count) + , Request(request) + , Priority(priority) { + AFL_VERIFY(Request); + AFL_VERIFY(Count); +} + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/usage/events.h b/ydb/core/tx/priorities/usage/events.h new file mode 100644 index 000000000000..47f3c4f973af --- /dev/null +++ b/ydb/core/tx/priorities/usage/events.h @@ -0,0 +1,93 @@ +#pragma once +#include "abstract.h" + +#include + +#include +#include +#include +#include + +namespace NKikimr::NPrioritiesQueue { + +struct TEvExecution { + enum EEv { + EvRegisterClient = EventSpaceBegin(TKikimrEvents::ES_PRIORITY_QUEUE), + EvUnregisterClient, + EvAsk, + EvAskMax, + EvFree, + EvAllocated, + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIORITY_QUEUE), "expected EvEnd < EventSpaceEnd"); + + class TEvRegisterClient: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ClientId, 0); + + public: + TEvRegisterClient(const ui64 clientId) + : ClientId(clientId) { + } + }; + + class TEvUnregisterClient: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ClientId, 0); + + public: + TEvUnregisterClient(const ui64 clientId) + : ClientId(clientId) { + } + }; + + class TEvAsk: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ClientId, 0); + YDB_READONLY(ui32, Count, 0); + YDB_READONLY_DEF(std::shared_ptr, Request); + YDB_READONLY(ui64, Priority, 0); + + public: + TEvAsk(const ui64 clientId, const ui32 count, const std::shared_ptr& request, const ui64 priority); + }; + + class TEvAskMax: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ClientId, 0); + YDB_READONLY(ui32, Count, 0); + YDB_READONLY_DEF(std::shared_ptr, Request); + YDB_READONLY(ui64, Priority, 0); + + public: + TEvAskMax(const ui64 clientId, const ui32 count, const std::shared_ptr& request, const ui64 priority); + }; + + class TEvFree: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ClientId, 0); + YDB_READONLY(ui32, Count, 0); + + public: + TEvFree(const ui64 clientId, const ui32 count) + : ClientId(clientId) + , Count(count) { + } + }; + + class TEvAllocated: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, RequestId, 0); + YDB_READONLY_DEF(std::shared_ptr, Guard); + + public: + TEvAllocated(const ui32 requestId, const std::shared_ptr& guard) + : RequestId(requestId) + , Guard(guard) { + } + }; +}; + +} // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/usage/service.cpp b/ydb/core/tx/priorities/usage/service.cpp new file mode 100644 index 000000000000..9345e792cb0f --- /dev/null +++ b/ydb/core/tx/priorities/usage/service.cpp @@ -0,0 +1,5 @@ +#include "service.h" + +namespace NKikimr::NPrioritiesQueue { + +} diff --git a/ydb/core/tx/priorities/usage/service.h b/ydb/core/tx/priorities/usage/service.h new file mode 100644 index 000000000000..79149e918eba --- /dev/null +++ b/ydb/core/tx/priorities/usage/service.h @@ -0,0 +1,77 @@ +#pragma once +#include "config.h" +#include +#include +#include +#include + +namespace NKikimr::NPrioritiesQueue { + +template +class TServiceOperatorImpl { +private: + using TSelf = TServiceOperatorImpl; + std::atomic IsEnabledFlag = false; + static void Register(const TConfig& serviceConfig) { + Singleton()->IsEnabledFlag = serviceConfig.IsEnabled(); + } + static const TString& GetQueueName() { + Y_ABORT_UNLESS(TQueuePolicy::Name.size() == 4); + return TQueuePolicy::Name; + } +public: + [[nodiscard]] static ui64 RegisterClient() { + static TAtomicCounter Counter = 0; + const ui64 id = Counter.Inc(); + if (TSelf::IsEnabled()) { + auto& context = NActors::TActorContext::AsActorContext(); + context.Send(MakeServiceId(), new TEvExecution::TEvRegisterClient(id)); + } + return id; + } + static void UnregisterClient(const ui64 clientId) { + auto& context = NActors::TActorContext::AsActorContext(); + if (TSelf::IsEnabled()) { + context.Send(MakeServiceId(), new TEvExecution::TEvUnregisterClient(clientId)); + } + } + static void Ask(const ui64 clientId, const ui64 priority, const std::shared_ptr& request, const ui32 count = 1) { + AFL_VERIFY(request); + if (TSelf::IsEnabled()) { + NActors::TActorContext::AsActorContext().Send(MakeServiceId(), new TEvExecution::TEvAsk(clientId, count, request, priority)); + } else { + request->OnAllocated(std::make_shared(NActors::TActorId(), clientId, count)); + } + } + static void AskMax(const ui64 clientId, const ui64 priority, const std::shared_ptr& request, const ui32 count = 1) { + AFL_VERIFY(request); + if (TSelf::IsEnabled()) { + NActors::TActorContext::AsActorContext().Send(MakeServiceId(), new TEvExecution::TEvAskMax(clientId, count, request, priority)); + } else { + request->OnAllocated(std::make_shared(NActors::TActorId(), clientId, count)); + } + } + static bool IsEnabled() { + return Singleton()->IsEnabledFlag; + } + static NActors::TActorId MakeServiceId() { + return NActors::TActorId(NActors::TActorContext::AsActorContext().SelfID.NodeId(), "SrvcPrqe" + GetQueueName()); + } + static NActors::TActorId MakeServiceId(const ui64 nodeId) { + return NActors::TActorId(nodeId, "SrvcPrqe" + GetQueueName()); + } + static NActors::IActor* CreateService(const TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> queueSignals) { + Register(config); + return new TDistributor(config, GetQueueName(), queueSignals); + } + +}; + +class TCompConveyorPolicy { +public: + static const inline TString Name = "Comp"; +}; + +using TCompServiceOperator = TServiceOperatorImpl; + +} diff --git a/ydb/core/tx/priorities/usage/ya.make b/ydb/core/tx/priorities/usage/ya.make new file mode 100644 index 000000000000..ba7690de7307 --- /dev/null +++ b/ydb/core/tx/priorities/usage/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +SRCS( + abstract.cpp + events.cpp + config.cpp + service.cpp +) + +PEERDIR( + ydb/library/actors/core + ydb/core/protos +) + +END() diff --git a/ydb/library/services/services.proto b/ydb/library/services/services.proto index 65d72b0961ba..759fa4393027 100644 --- a/ydb/library/services/services.proto +++ b/ydb/library/services/services.proto @@ -401,6 +401,8 @@ enum EServiceKikimr { GROUPED_MEMORY_LIMITER = 2700; DATA_INTEGRITY = 3000; + + TX_PRIORITIES_QUEUE = 3100; }; message TActivity {