diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 426f8950b79f..384e0019d211 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -28,6 +28,7 @@ ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingModuloN ydb/core/kqp/ut/olap KqpOlapBlobsSharing.UpsertWhileSplitTest ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL ydb/core/kqp/ut/olap KqpOlapWrite.TierDraftsGCWithRestart +ydb/core/kqp/ut/olap KqpOlapIndexes.Indexes* ydb/core/kqp/ut/olap [*/*] chunk chunk ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable+ColumnStore ydb/core/kqp/ut/query KqpLimits.QueryExecTimeoutCancel diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index 8d9c96bbd7e2..410adeddb67f 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -15,6 +15,7 @@ namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapTiering) { Y_UNIT_TEST(Eviction) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetSkipSpecialCheckForEvict(true); TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/ya.make b/ydb/core/tx/columnshard/blobs_action/abstract/ya.make index b3b4c20028c8..1ce1d711629e 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/ya.make +++ b/ydb/core/tx/columnshard/blobs_action/abstract/ya.make @@ -18,6 +18,7 @@ PEERDIR( contrib/libs/apache/arrow ydb/core/tablet_flat ydb/core/tx/tiering/abstract + ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/blobs_action/common ydb/core/tx/columnshard/data_sharing/protos ydb/core/tx/columnshard/blobs_action/events diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 636327c597fc..b65558221511 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -3,6 +3,8 @@ #include "bg_tasks/manager/manager.h" #include "blobs_reader/actor.h" #include "counters/aggregation/table_stats.h" +#include "data_accessor/actor.h" +#include "data_accessor/manager.h" #include "engines/column_engine_logs.h" #include "engines/writer/buffer/actor.h" #include "hooks/abstract/abstract.h" @@ -30,6 +32,7 @@ void TColumnShard::CleanupActors(const TActorContext& ctx) { } ctx.Send(ResourceSubscribeActor, new TEvents::TEvPoisonPill); ctx.Send(BufferizationWriteActorId, new TEvents::TEvPoisonPill); + ctx.Send(DataAccessorsControlActorId, new TEvents::TEvPoisonPill); if (PrioritizationClientId) { NPrioritiesQueue::TCompServiceOperator::UnregisterClient(PrioritizationClientId); } @@ -105,6 +108,9 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { Settings.RegisterControls(icb); ResourceSubscribeActor = ctx.Register(new NOlap::NResourceBroker::NSubscribe::TActor(TabletID(), SelfId())); BufferizationWriteActorId = ctx.Register(new NColumnShard::NWriting::TActor(TabletID(), SelfId())); + DataAccessorsControlActorId = ctx.Register(new NOlap::NDataAccessorControl::TActor(TabletID(), SelfId())); + DataAccessorsManager = std::make_shared(DataAccessorsControlActorId), + PrioritizationClientId = NPrioritiesQueue::TCompServiceOperator::RegisterClient(); Execute(CreateTxInitSchema(), ctx); } @@ -299,8 +305,7 @@ void TColumnShard::UpdateIndexCounters() { LOG_S_DEBUG("Index: tables " << stats.Tables << " inserted " << stats.GetInsertedStats().DebugString() << " compacted " << stats.GetCompactedStats().DebugString() << " s-compacted " << stats.GetSplitCompactedStats().DebugString() << " inactive " << stats.GetInactiveStats().DebugString() << " evicted " - << stats.GetEvictedStats().DebugString() << " at tablet " - << TabletID()); + << stats.GetEvictedStats().DebugString() << " at tablet " << TabletID()); } ui64 TColumnShard::MemoryUsage() const { diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index ad2db7e1a672..088294c6896b 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -1,45 +1,51 @@ #include "columnshard_impl.h" -#include "columnshard_ttl.h" #include "columnshard_private_events.h" #include "columnshard_schema.h" +#include "columnshard_ttl.h" + +#include "bg_tasks/adapter/adapter.h" +#include "bg_tasks/manager/manager.h" #include "blobs_action/storages_manager/manager.h" -#include "hooks/abstract/abstract.h" +#include "data_accessor/manager.h" #include "engines/column_engine_logs.h" -#include "bg_tasks/manager/manager.h" -#include "bg_tasks/adapter/adapter.h" -#include -#include +#include "hooks/abstract/abstract.h" +#include "loading/stages.h" +#include "tx_reader/abstract.h" +#include "tx_reader/composite.h" #include +#include #include - +#include namespace NKikimr::NColumnShard { using namespace NTabletFlatExecutor; -class TTxInit : public TTransactionBase { +class TTxInit: public TTransactionBase { private: const TMonotonic StartInstant = TMonotonic::Now(); public: TTxInit(TColumnShard* self) - : TBase(self) - {} + : TBase(self) { + } bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_INIT; } + TTxType GetTxType() const override { + return TXTYPE_INIT; + } private: - bool Precharge(TTransactionContext& txc); + std::shared_ptr StartReader; void SetDefaults(); - bool ReadEverything(TTransactionContext& txc, const TActorContext& ctx); + std::shared_ptr BuildReader(); }; void TTxInit::SetDefaults() { Self->CurrentSchemeShardId = 0; - Self->LastSchemaSeqNo = { }; + Self->LastSchemaSeqNo = {}; Self->ProcessingParams.reset(); Self->LastPlannedStep = 0; Self->LastPlannedTxId = 0; @@ -50,213 +56,39 @@ void TTxInit::SetDefaults() { Self->LongTxWritesByUniqueId.clear(); } -bool TTxInit::Precharge(TTransactionContext& txc) { - NIceDb::TNiceDb db(txc.DB); - - bool ready = true; - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); - - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::CurrentSchemeShardId, Self->CurrentSchemeShardId); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastSchemaSeqNoGeneration, Self->LastSchemaSeqNo.Generation); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastSchemaSeqNoRound, Self->LastSchemaSeqNo.Round); - ready = ready && Schema::GetSpecialProtoValue(db, Schema::EValueIds::ProcessingParams, Self->ProcessingParams); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastPlannedStep, Self->LastPlannedStep); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastPlannedTxId, Self->LastPlannedTxId); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastExportNumber, Self->LastExportNo); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::OwnerPathId, Self->OwnerPathId); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::OwnerPath, Self->OwnerPath); - - - { - ui64 lastCompletedStep = 0; - ui64 lastCompletedTx = 0; - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastCompletedStep, lastCompletedStep); - ready = ready && Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastCompletedTxId, lastCompletedTx); - Self->LastCompletedTx = NOlap::TSnapshot(lastCompletedStep, lastCompletedTx); - } - - if (!ready) { - return false; - } - return true; -} - -bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) { - TTablesManager tManagerLocal(Self->StoragesManager, Self->TabletID()); - { - TLoadTimeSignals::TLoadTimer timer = tManagerLocal.GetLoadTimeCounters()->PrechargeTimeCounters.StartGuard(); - if (!Precharge(txc)) { - timer.AddLoadingFail(); - return false; - } - } - - NIceDb::TNiceDb db(txc.DB); - TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - { - ACFL_DEBUG("step", "TTablesManager::Load_Start"); - { - TMemoryProfileGuard g("TTxInit/TTablesManager"); - if (!tManagerLocal.InitFromDB(db)) { - ACFL_ERROR("step", "TTablesManager::InitFromDB_Fails"); - return false; - } - } - { - TMemoryProfileGuard g("TTxInit/LoadIndex"); - if (!tManagerLocal.LoadIndex(dbTable)) { - ACFL_ERROR("step", "TTablesManager::LoadIndex_Fails"); - return false; - } - } - Self->TablesManager = std::move(tManagerLocal); - - Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, Self->TablesManager.GetTables().size()); - Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_PRESETS, Self->TablesManager.GetSchemaPresets().size()); - Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_TTLS, Self->TablesManager.GetTtl().PathsCount()); - ACFL_DEBUG("step", "TTablesManager::Load_Finish"); - } - - { - ACFL_DEBUG("step", "TInsertTable::Load_Start"); - TMemoryProfileGuard g("TTxInit/InsertTable"); - auto localInsertTable = std::make_unique(); - for (auto&& i : Self->TablesManager.GetTables()) { - localInsertTable->RegisterPathInfo(i.first); - } - if (!localInsertTable->Load(db, dbTable, TAppData::TimeProvider->Now())) { - ACFL_ERROR("step", "TInsertTable::Load_Fails"); - return false; - } - ACFL_DEBUG("step", "TInsertTable::Load_Finish"); - Self->InsertTable.swap(localInsertTable); - } - - { - ACFL_DEBUG("step", "TTxController::Load_Start"); - TMemoryProfileGuard g("TTxInit/TTxController"); - auto localTxController = std::make_unique(*Self); - if (!localTxController->Load(txc)) { - ACFL_ERROR("step", "TTxController::Load_Fails"); - return false; - } - ACFL_DEBUG("step", "TTxController::Load_Finish"); - Self->ProgressTxController.swap(localTxController); - } - - { - ACFL_DEBUG("step", "TOperationsManager::Load_Start"); - TMemoryProfileGuard g("TTxInit/TOperationsManager"); - auto localOperationsManager = std::make_unique(); - if (!localOperationsManager->Load(txc)) { - ACFL_ERROR("step", "TOperationsManager::Load_Fails"); - return false; - } - ACFL_DEBUG("step", "TOperationsManager::Load_Finish"); - Self->OperationsManager.swap(localOperationsManager); - } - - { - ACFL_DEBUG("step", "TStoragesManager::Load_Start"); - AFL_VERIFY(Self->StoragesManager); - TMemoryProfileGuard g("TTxInit/NDataSharing::TStoragesManager"); - if (!Self->StoragesManager->LoadIdempotency(txc.DB)) { - return false; - } - ACFL_DEBUG("step", "TStoragesManager::Load_Finish"); - } - - { - TMemoryProfileGuard g("TTxInit/LongTxWrites"); - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const TInsertWriteId writeId = (TInsertWriteId)rowset.GetValue(); - const ui32 writePartId = rowset.GetValue(); - NKikimrLongTxService::TLongTxId proto; - Y_ABORT_UNLESS(proto.ParseFromString(rowset.GetValue())); - const auto longTxId = NLongTxService::TLongTxId::FromProto(proto); - - std::optional granuleShardingVersion; - if (rowset.HaveValue() && rowset.GetValue()) { - granuleShardingVersion = rowset.GetValue(); - } - - Self->LoadLongTxWrite(writeId, writePartId, longTxId, granuleShardingVersion); - - if (!rowset.Next()) { - return false; - } - } - } - { - TMemoryProfileGuard g("TTxInit/LocksDB"); - if (txc.DB.GetScheme().GetTableInfo(Schema::Locks::TableId)) { - TColumnShardLocksDb locksDb(*Self, txc); - if (!Self->SysLocks.Load(locksDb)) { - return false; - } - } - } - - { - TMemoryProfileGuard g("TTxInit/NDataSharing::TBackgroundSessionsManager"); - if (!Self->BackgroundSessionsManager->LoadIdempotency(txc)) { - return false; - } - } - - { - TMemoryProfileGuard g("TTxInit/NDataSharing::TSessionsManager"); - auto local = std::make_shared(); - if (!local->Load(txc.DB, Self->TablesManager.GetPrimaryIndexAsOptional())) { - return false; - } - Self->SharingSessionsManager = local; - } - { - TMemoryProfileGuard g("TTxInit/TInFlightReadsTracker"); - TInFlightReadsTracker local(Self->StoragesManager, Self->Counters.GetRequestsTracingCounters()); - if (!local.LoadFromDatabase(txc.DB)) { - return false; - } - Self->InFlightReadsTracker = std::move(local); - } - - Self->UpdateInsertTableCounters(); - Self->UpdateIndexCounters(); - Self->UpdateResourceMetrics(ctx, {}); - return true; +std::shared_ptr TTxInit::BuildReader() { + auto result = std::make_shared("composite_init"); + result->AddChildren(std::make_shared("special_values", Self)); + result->AddChildren(std::make_shared("tables_manager", Self)); + result->AddChildren(std::make_shared("insert_table", Self)); + result->AddChildren(std::make_shared("tx_controller", Self)); + result->AddChildren(std::make_shared("operations_manager", Self)); + result->AddChildren(std::make_shared("storages_manager", Self)); + result->AddChildren(std::make_shared("long_tx", Self)); + result->AddChildren(std::make_shared("db_locks", Self)); + result->AddChildren(std::make_shared("bg_sessions", Self)); + result->AddChildren(std::make_shared("sharing_sessions", Self)); + result->AddChildren(std::make_shared("in_flight_reads", Self)); + return result; } bool TTxInit::Execute(TTransactionContext& txc, const TActorContext& ctx) { - NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); + NActors::TLogContextGuard gLogging = + NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); LOG_S_DEBUG("TTxInit.Execute at tablet " << Self->TabletID()); try { - SetDefaults(); - return ReadEverything(txc, ctx); + if (!StartReader) { + SetDefaults(); + StartReader = BuildReader(); + } + if (!StartReader->Execute(txc, ctx)) { + return false; + } + StartReader = nullptr; + Self->UpdateInsertTableCounters(); + Self->UpdateIndexCounters(); + Self->UpdateResourceMetrics(ctx, {}); } catch (const TNotReadyTabletException&) { ACFL_ERROR("event", "tablet not ready"); return false; @@ -277,22 +109,25 @@ void TTxInit::Complete(const TActorContext& ctx) { NYDBTest::TControllers::GetColumnShardController()->OnTabletInitCompleted(*Self); } -class TTxUpdateSchema : public TTransactionBase { +class TTxUpdateSchema: public TTransactionBase { std::vector NormalizerTasks; const TMonotonic StartInstant = TMonotonic::Now(); public: TTxUpdateSchema(TColumnShard* self) - : TBase(self) - {} + : TBase(self) { + } bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_UPDATE_SCHEMA; } + TTxType GetTxType() const override { + return TXTYPE_UPDATE_SCHEMA; + } }; bool TTxUpdateSchema::Execute(TTransactionContext& txc, const TActorContext&) { - NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); + NActors::TLogContextGuard gLogging = + NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); ACFL_INFO("step", "TTxUpdateSchema.Execute_Start")("details", Self->NormalizerController.DebugString()); while (!Self->NormalizerController.IsNormalizationFinished()) { @@ -336,23 +171,26 @@ void TTxUpdateSchema::Complete(const TActorContext& ctx) { } } -class TTxApplyNormalizer : public TTransactionBase { +class TTxApplyNormalizer: public TTransactionBase { public: TTxApplyNormalizer(TColumnShard* self, NOlap::INormalizerChanges::TPtr changes) : TBase(self) - , Changes(changes) - {} + , Changes(changes) { + } bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_APPLY_NORMALIZER; } + TTxType GetTxType() const override { + return TXTYPE_APPLY_NORMALIZER; + } private: NOlap::INormalizerChanges::TPtr Changes; }; bool TTxApplyNormalizer::Execute(TTransactionContext& txc, const TActorContext&) { - NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); + NActors::TLogContextGuard gLogging = + NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("step", "TTxApplyNormalizer.Execute")("details", Self->NormalizerController.DebugString()); if (!Changes->ApplyOnExecute(txc, Self->NormalizerController)) { return false; @@ -368,7 +206,8 @@ bool TTxApplyNormalizer::Execute(TTransactionContext& txc, const TActorContext&) void TTxApplyNormalizer::Complete(const TActorContext& ctx) { AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("step", "TTxApplyNormalizer.Complete")("tablet_id", Self->TabletID())("event", "initialize_shard"); AFL_VERIFY(!Self->NormalizerController.IsNormalizationFinished())("details", Self->NormalizerController.DebugString()); - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "apply_normalizer_changes")("details", Self->NormalizerController.DebugString())("size", Changes->GetSize()); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "apply_normalizer_changes")( + "details", Self->NormalizerController.DebugString())("size", Changes->GetSize()); Changes->ApplyOnComplete(Self->NormalizerController); Self->NormalizerController.GetNormalizer()->OnResultReady(); if (Self->NormalizerController.GetNormalizer()->HasActiveTasks()) { @@ -384,18 +223,20 @@ void TTxApplyNormalizer::Complete(const TActorContext& ctx) { } /// Create local database on tablet start if none -class TTxInitSchema : public TTransactionBase { +class TTxInitSchema: public TTransactionBase { private: const TMonotonic StartInstant = TMonotonic::Now(); public: TTxInitSchema(TColumnShard* self) - : TBase(self) - {} + : TBase(self) { + } bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_INIT_SCHEMA; } + TTxType GetTxType() const override { + return TXTYPE_INIT_SCHEMA; + } }; bool TTxInitSchema::Execute(TTransactionContext& txc, const TActorContext&) { @@ -428,21 +269,16 @@ bool TTxInitSchema::Execute(TTransactionContext& txc, const TActorContext&) { // Enable compression for the SmallBlobs table const auto* smallBlobsDefaultColumnFamily = txc.DB.GetScheme().DefaultFamilyFor(Schema::SmallBlobs::TableId); - if (!smallBlobsDefaultColumnFamily || - smallBlobsDefaultColumnFamily->Codec != NTable::TAlter::ECodec::LZ4) - { - txc.DB.Alter().SetFamily(Schema::SmallBlobs::TableId, 0, - NTable::TAlter::ECache::None, NTable::TAlter::ECodec::LZ4); + if (!smallBlobsDefaultColumnFamily || smallBlobsDefaultColumnFamily->Codec != NTable::TAlter::ECodec::LZ4) { + txc.DB.Alter().SetFamily(Schema::SmallBlobs::TableId, 0, NTable::TAlter::ECache::None, NTable::TAlter::ECodec::LZ4); } // SmallBlobs table has compaction policy suitable for a big table const auto* smallBlobsTable = txc.DB.GetScheme().GetTableInfo(Schema::SmallBlobs::TableId); NLocalDb::TCompactionPolicyPtr bigTableCompactionPolicy = NLocalDb::CreateDefaultUserTablePolicy(); bigTableCompactionPolicy->MinDataPageSize = 32 * 1024; - if (!smallBlobsTable || - !smallBlobsTable->CompactionPolicy || - smallBlobsTable->CompactionPolicy->Generations.size() != bigTableCompactionPolicy->Generations.size()) - { + if (!smallBlobsTable || !smallBlobsTable->CompactionPolicy || + smallBlobsTable->CompactionPolicy->Generations.size() != bigTableCompactionPolicy->Generations.size()) { txc.DB.Alter().SetCompactionPolicy(Schema::SmallBlobs::TableId, *bigTableCompactionPolicy); } @@ -463,4 +299,4 @@ void TColumnShard::Handle(TEvPrivate::TEvNormalizerResult::TPtr& ev, const TActo Execute(new TTxApplyNormalizer(this, ev->Get()->GetChanges()), ctx); } -} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index bb192aef33a2..1aa77ca21c29 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -19,8 +19,11 @@ #include "blobs_action/transaction/tx_gc_indexed.h" #include "blobs_action/transaction/tx_gc_insert_table.h" #include "blobs_action/transaction/tx_remove_blobs.h" -#include "blobs_reader/actor.h" -#include "data_sharing/common/transactions/tx_extension.h" +#include "blobs_action/transaction/tx_gc_insert_table.h" +#include "blobs_action/transaction/tx_gc_indexed.h" +#include "bg_tasks/events/events.h" + +#include "data_accessor/manager.h" #include "data_sharing/destination/session/destination.h" #include "data_sharing/source/session/source.h" #include "engines/changes/cleanup_portions.h" @@ -78,7 +81,7 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod()) , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval()) , InFlightReadsTracker(StoragesManager, Counters.GetRequestsTracingCounters()) - , TablesManager(StoragesManager, info->TabletID) + , TablesManager(StoragesManager, std::make_shared(), info->TabletID) , Subscribers(std::make_shared(*this)) , PipeClientCache(NTabletPipe::CreateBoundedClientCache(new NTabletPipe::TBoundedClientCacheConfig(), GetPipeClientConfig())) , InsertTable(std::make_unique()) @@ -604,6 +607,56 @@ class TChangesReadTask: public NOlap::NBlobOperations::NRead::ITask { } }; +class TDataAccessorsSubscriber: public NOlap::IDataAccessorRequestsSubscriber { +protected: + const NActors::TActorId ShardActorId; + std::shared_ptr Changes; + std::shared_ptr VersionedIndex; + + virtual void DoOnRequestsFinishedImpl() = 0; + + virtual void DoOnRequestsFinished(NOlap::TDataAccessorsResult&& result) override final { + Changes->SetFetchedDataAccessors(std::move(result), NOlap::TDataAccessorsInitializationContext(VersionedIndex)); + DoOnRequestsFinishedImpl(); + } + +public: + TDataAccessorsSubscriber(const NActors::TActorId& shardActorId, const std::shared_ptr& changes, + const std::shared_ptr& versionedIndex) + : ShardActorId(shardActorId) + , Changes(changes) + , VersionedIndex(versionedIndex) { + } +}; + +class TDataAccessorsSubscriberWithRead: public TDataAccessorsSubscriber { +private: + using TBase = TDataAccessorsSubscriber; + +protected: + const bool CacheDataAfterWrite = false; + const ui64 ShardTabletId; + TIndexationCounters Counters; + NOlap::TSnapshot SnapshotModification; + const NActors::TActorId ResourceSubscribeActor; + const NOlap::NResourceBroker::NSubscribe::TTaskContext TaskSubscriptionContext; + +public: + TDataAccessorsSubscriberWithRead(const NActors::TActorId& resourceSubscribeActor, const std::shared_ptr& changes, + const std::shared_ptr& versionedIndex, const bool cacheAfterWrite, const NActors::TActorId& shardActorId, + const ui64 shardTabletId, const TIndexationCounters& counters, const NOlap::TSnapshot& snapshotModification, + const NOlap::NResourceBroker::NSubscribe::TTaskContext& taskSubscriptionContext) + : TBase(shardActorId, changes, versionedIndex) + , CacheDataAfterWrite(cacheAfterWrite) + , ShardTabletId(shardTabletId) + , Counters(counters) + , SnapshotModification(snapshotModification) + , ResourceSubscribeActor(resourceSubscribeActor) + , TaskSubscriptionContext(taskSubscriptionContext) + { + } +}; + class TInsertChangesReadTask: public TChangesReadTask, public TMonitoringObjectsCounter { private: using TBase = TChangesReadTask; @@ -720,7 +773,8 @@ class TCompactionAllocated: public NPrioritiesQueue::IRequest { : TabletActorId(tabletActorId) { } }; -} // namespace + +} // namespace void TColumnShard::SetupCompaction(const std::set& pathIds) { if (!AppDataVerified().ColumnShardConfig.GetCompactionEnabled() || @@ -744,6 +798,26 @@ void TColumnShard::SetupCompaction(const std::set& pathIds) { } } +class TCompactionDataAccessorsSubscriber: public TDataAccessorsSubscriberWithRead { +private: + using TBase = TDataAccessorsSubscriberWithRead; + +protected: + virtual void DoOnRequestsFinishedImpl() override { + const TString externalTaskId = Changes->GetTaskIdentifier(); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "compaction")("external_task_id", externalTaskId); + + auto ev = std::make_unique(VersionedIndex, Changes, CacheDataAfterWrite); + auto readSubscriber = std::make_shared( + std::make_shared(std::move(ev), ShardActorId, ShardTabletId, Counters, SnapshotModification), 0, + Changes->CalcMemoryForUsage(), externalTaskId, TaskSubscriptionContext); + NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription(ResourceSubscribeActor, readSubscriber); + } + +public: + using TBase::TBase; +}; + void TColumnShard::StartCompaction(const std::shared_ptr& guard) { Counters.GetCSCounters().OnSetupCompaction(); BackgroundController.ResetWaitingPriority(); @@ -756,21 +830,50 @@ void TColumnShard::StartCompaction(const std::shared_ptr(indexChanges); compaction->SetQueueGuard(guard); - indexChanges->Start(*this); + compaction->Start(*this); auto actualIndexInfo = std::make_shared(TablesManager.GetPrimaryIndex()->GetVersionedIndex()); - auto ev = std::make_unique(actualIndexInfo, indexChanges, Settings.CacheDataAfterCompaction); - const TString externalTaskId = indexChanges->GetTaskIdentifier(); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "compaction")("external_task_id", externalTaskId); + auto request = compaction->ExtractDataAccessorsRequest(); + request->RegisterSubscriber(std::make_shared(ResourceSubscribeActor, indexChanges, actualIndexInfo, + Settings.CacheDataAfterCompaction, SelfId(), TabletID(), Counters.GetCompactionCounters(), GetLastCompletedTx(), + CompactTaskSubscription)); + TablesManager.GetPrimaryIndex()->FetchDataAccessors(request); +} - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - ResourceSubscribeActor, std::make_shared( - std::make_shared( - std::move(ev), SelfId(), TabletID(), Counters.GetCompactionCounters(), GetLastCompletedTx()), - 0, indexChanges->CalcMemoryForUsage(), externalTaskId, CompactTaskSubscription)); +class TWriteEvictPortionsDataAccessorsSubscriber: public TDataAccessorsSubscriberWithRead { +private: + using TBase = TDataAccessorsSubscriberWithRead; - LOG_S_DEBUG("ActiveCompactions: " << BackgroundController.GetCompactionsCount() << " at tablet " << TabletID()); -} +protected: + virtual void DoOnRequestsFinishedImpl() override { + ACFL_DEBUG("background", "ttl")("need_writes", true); + auto ev = std::make_unique(VersionedIndex, Changes, false); + auto readSubscriber = std::make_shared( + std::make_shared(std::move(ev), ShardActorId, ShardTabletId, Counters, SnapshotModification), 0, + Changes->CalcMemoryForUsage(), Changes->GetTaskIdentifier(), TaskSubscriptionContext); + + NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription(ResourceSubscribeActor, readSubscriber); + } + +public: + using TBase::TBase; +}; + +class TNoWriteEvictPortionsDataAccessorsSubscriber: public TDataAccessorsSubscriber { +private: + using TBase = TDataAccessorsSubscriber; + +protected: + virtual void DoOnRequestsFinishedImpl() override { + ACFL_DEBUG("background", "ttl")("need_writes", false); + auto ev = std::make_unique(VersionedIndex, Changes, false); + ev->SetPutStatus(NKikimrProto::OK); + NActors::TActivationContext::Send(ShardActorId, std::move(ev)); + } + +public: + using TBase::TBase; +}; bool TColumnShard::SetupTtl(const THashMap& pathTtls) { if (!AppDataVerified().ColumnShardConfig.GetTTLEnabled() || !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::TTL)) { @@ -793,24 +896,36 @@ bool TColumnShard::SetupTtl(const THashMap& pathTtls) { auto actualIndexInfo = std::make_shared(TablesManager.GetPrimaryIndex()->GetVersionedIndex()); for (auto&& i : indexChanges) { - const TString externalTaskId = i->GetTaskIdentifier(); - const bool needWrites = i->NeedConstruction(); - ACFL_DEBUG("background", "ttl")("need_writes", needWrites); i->Start(*this); - auto ev = std::make_unique(actualIndexInfo, i, false); - if (needWrites) { - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - ResourceSubscribeActor, std::make_shared( - std::make_shared(std::move(ev), SelfId(), TabletID(), Counters.GetCompactionCounters(), GetLastCompletedTx()), - 0, i->CalcMemoryForUsage(), externalTaskId, TTLTaskSubscription)); + auto request = i->ExtractDataAccessorsRequest(); + if (i->NeedConstruction()) { + request->RegisterSubscriber(std::make_shared(ResourceSubscribeActor, i, + actualIndexInfo, Settings.CacheDataAfterCompaction, SelfId(), TabletID(), Counters.GetEvictionCounters(), GetLastCompletedTx(), + TTLTaskSubscription)); } else { - ev->SetPutStatus(NKikimrProto::OK); - ActorContext().Send(SelfId(), std::move(ev)); + request->RegisterSubscriber(std::make_shared(SelfId(), i, actualIndexInfo)); } + TablesManager.GetPrimaryIndex()->FetchDataAccessors(request); } return true; } +class TCleanupPortionsDataAccessorsSubscriber: public TDataAccessorsSubscriber { +private: + using TBase = TDataAccessorsSubscriber; + +protected: + virtual void DoOnRequestsFinishedImpl() override { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("background", "cleanup")("changes_info", Changes->DebugString()); + auto ev = std::make_unique(VersionedIndex, Changes, false); + ev->SetPutStatus(NKikimrProto::OK); // No new blobs to write + NActors::TActivationContext::Send(ShardActorId, std::move(ev)); + } + +public: + using TBase::TBase; +}; + void TColumnShard::SetupCleanupPortions() { Counters.GetCSCounters().OnSetupCleanup(); if (!AppDataVerified().ColumnShardConfig.GetCleanupEnabled() || !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::Cleanup)) { @@ -828,15 +943,12 @@ void TColumnShard::SetupCleanupPortions() { ACFL_DEBUG("background", "cleanup")("skip_reason", "no_changes"); return; } - - ACFL_DEBUG("background", "cleanup")("changes_info", changes->DebugString()); - auto actualIndexInfo = std::make_shared(TablesManager.GetPrimaryIndex()->GetVersionedIndex()); - auto ev = std::make_unique(actualIndexInfo, changes, false); - ev->SetPutStatus(NKikimrProto::OK); // No new blobs to write - changes->Start(*this); - Send(SelfId(), ev.release()); + auto request = changes->ExtractDataAccessorsRequest(); + auto actualIndexInfo = std::make_shared(TablesManager.GetPrimaryIndex()->GetVersionedIndex()); + request->RegisterSubscriber(std::make_shared(SelfId(), changes, actualIndexInfo)); + TablesManager.GetPrimaryIndex()->FetchDataAccessors(request); } void TColumnShard::SetupCleanupTables() { @@ -1103,6 +1215,76 @@ void TColumnShard::Handle(NOlap::NDataSharing::NEvents::TEvFinishedFromSource::T } }; +class TTxAskPortionChunks: public TTransactionBase { +private: + using TBase = TTransactionBase; + std::shared_ptr Request; + THashMap> PortionsByPath; + THashMap> Accessors; + NOlap::TDataAccessorsResult Result; + +public: + TTxAskPortionChunks(TColumnShard* self, const std::shared_ptr& request) + : TBase(self) + , Request(request) { + for (auto&& i : Request->GetPathIds()) { + PortionsByPath.emplace(i, Request->StartFetching(i)); + } + } + + bool Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) override { + NIceDb::TNiceDb db(txc.DB); + TBlobGroupSelector selector(Self->Info()); + for (auto&& i : PortionsByPath) { + while (i.second.size()) { + auto p = i.second.back(); + auto rowset = db.Table().Prefix(p->GetPathId(), p->GetPortionId()).Select(); + NOlap::TPortionInfoConstructor constructor(*p, false, true, true); + { + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + NOlap::TColumnChunkLoadContextV1 chunkLoadContext(rowset); + constructor.LoadRecord(chunkLoadContext); + if (!rowset.Next()) { + return false; + } + } + } + { + auto rowset = db.Table().Prefix(p->GetPathId(), p->GetPortionId()).Select(); + if (!rowset.IsReady()) { + return false; + } + while (!rowset.EndOfSet()) { + NOlap::TIndexChunkLoadContext chunkLoadContext(rowset, &selector); + constructor.LoadIndex(chunkLoadContext); + if (!rowset.Next()) { + return false; + } + } + } + Accessors[i.first].emplace_back(constructor.Build(true)); + i.second.pop_back(); + } + } + return true; + } + void Complete(const TActorContext& /*ctx*/) override { + for (auto&& i : Accessors) { + Request->AddData(i.first, std::move(i.second)); + } + } + TTxType GetTxType() const override { + return TXTYPE_WRITE_INDEX; + } +}; + +void TColumnShard::Handle(NOlap::NDataAccessorControl::TEvAskDataAccessors::TPtr& ev, const TActorContext& /*ctx*/) { + Execute(new TTxAskPortionChunks(this, ev->Get()->GetRequest())); +} + void TColumnShard::Handle(NOlap::NDataSharing::NEvents::TEvAckFinishFromInitiator::TPtr& ev, const TActorContext& ctx) { AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("process", "BlobsSharing")("event", "TEvAckFinishFromInitiator"); auto currentSession = SharingSessionsManager->GetDestinationSession(ev->Get()->Record.GetSessionId()); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 3a082f1def96..e53d6317437a 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -97,6 +97,20 @@ class TOperationsManager; class TWaitEraseTablesTxSubscriber; class TTxBlobsWritingFinished; +namespace NLoading { +class TInsertTableInitializer; +class TTxControllerInitializer; +class TOperationsManagerInitializer; +class TStoragesManagerInitializer; +class TLongTxInitializer; +class TDBLocksInitializer; +class TBackgroundSessionsInitializer; +class TSharingSessionsInitializer; +class TInFlightReadsInitializer; +class TSpecialValuesInitializer; +class TTablesManagerInitializer; +} + extern bool gAllowLogBatchingDefaultValue; IActor* CreateWriteActor(ui64 tabletId, IWriteController::TPtr writeController, const TInstant deadline); @@ -198,6 +212,18 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa friend class IProposeTxOperator; friend class TSharingTransactionOperator; + friend class NLoading::TInsertTableInitializer; + friend class NLoading::TTxControllerInitializer; + friend class NLoading::TOperationsManagerInitializer; + friend class NLoading::TStoragesManagerInitializer; + friend class NLoading::TLongTxInitializer; + friend class NLoading::TDBLocksInitializer; + friend class NLoading::TBackgroundSessionsInitializer; + friend class NLoading::TSharingSessionsInitializer; + friend class NLoading::TInFlightReadsInitializer; + friend class NLoading::TSpecialValuesInitializer; + friend class NLoading::TTablesManagerInitializer; + class TTxProgressTx; class TTxProposeCancel; // proto @@ -255,6 +281,8 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void Handle(NOlap::NDataSharing::NEvents::TEvAckFinishToSource::TPtr& ev, const TActorContext& ctx); void Handle(NOlap::NDataSharing::NEvents::TEvAckFinishFromInitiator::TPtr& ev, const TActorContext& ctx); + void Handle(NOlap::NDataAccessorControl::TEvAskDataAccessors::TPtr& ev, const TActorContext& ctx); + ITransaction* CreateTxInitSchema(); void OnActivateExecutor(const TActorContext& ctx) override; @@ -413,6 +441,8 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa HFunc(NOlap::NDataSharing::NEvents::TEvFinishedFromSource, Handle); HFunc(NOlap::NDataSharing::NEvents::TEvAckFinishToSource, Handle); HFunc(NOlap::NDataSharing::NEvents::TEvAckFinishFromInitiator, Handle); + HFunc(NOlap::NDataAccessorControl::TEvAskDataAccessors, Handle); + default: if (!HandleDefaultEvents(ev, SelfId())) { LOG_S_WARN("TColumnShard.StateWork at " << TabletID() << " unhandled event type: " << ev->GetTypeRewrite() @@ -471,6 +501,9 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa TActorId ResourceSubscribeActor; TActorId BufferizationWriteActorId; + TActorId DataAccessorsControlActorId; + NOlap::NDataAccessorControl::TDataAccessorsManagerContainer DataAccessorsManager; + TActorId StatsReportPipe; std::vector ActorsToStop; @@ -559,6 +592,8 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void FillColumnTableStats(const TActorContext& ctx, std::unique_ptr& ev); public: + std::shared_ptr StartReader; + ui64 TabletTxCounter = 0; const TTablesManager& GetTablesManager() const { diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index a90647ed1e2c..f97535f78de9 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -58,6 +58,12 @@ struct TEvPrivate { EvWritePortionResult, EvStartCompaction, + EvRegisterGranuleDataAccessor, + EvUnregisterGranuleDataAccessor, + EvAskDataAccessors, + EvAddPortionDataAccessor, + EvRemovePortionDataAccessor, + EvEnd }; diff --git a/ydb/core/tx/columnshard/counters/scan.h b/ydb/core/tx/columnshard/counters/scan.h index 6d1202288514..d9a02e1fef49 100644 --- a/ydb/core/tx/columnshard/counters/scan.h +++ b/ydb/core/tx/columnshard/counters/scan.h @@ -264,7 +264,7 @@ class TScanCounters: public TCommonCountersOwner { void FillStats(::NKikimrTableStats::TTableStats& output) const; }; -class TCounterGuard: TNonCopyable { +class TCounterGuard: TMoveOnly { private: std::shared_ptr Counter; public: @@ -290,6 +290,7 @@ class TCounterGuard: TNonCopyable { class TConcreteScanCounters: public TScanCounters { private: using TBase = TScanCounters; + std::shared_ptr FetchAccessorsCount; std::shared_ptr MergeTasksCount; std::shared_ptr AssembleTasksCount; std::shared_ptr ReadTasksCount; @@ -297,6 +298,10 @@ class TConcreteScanCounters: public TScanCounters { public: TScanAggregations Aggregations; + TCounterGuard GetFetcherAcessorsGuard() const { + return TCounterGuard(FetchAccessorsCount); + } + TCounterGuard GetMergeTasksGuard() const { return TCounterGuard(MergeTasksCount); } @@ -314,7 +319,8 @@ class TConcreteScanCounters: public TScanCounters { } bool InWaiting() const { - return MergeTasksCount->Val() || AssembleTasksCount->Val() || ReadTasksCount->Val() || ResourcesAllocationTasksCount->Val(); + return MergeTasksCount->Val() || AssembleTasksCount->Val() || ReadTasksCount->Val() || ResourcesAllocationTasksCount->Val() || + FetchAccessorsCount->Val(); } void OnBlobsWaitDuration(const TDuration d, const TDuration fullScanDuration) const { @@ -324,6 +330,7 @@ class TConcreteScanCounters: public TScanCounters { TConcreteScanCounters(const TScanCounters& counters) : TBase(counters) + , FetchAccessorsCount(std::make_shared()) , MergeTasksCount(std::make_shared()) , AssembleTasksCount(std::make_shared()) , ReadTasksCount(std::make_shared()) diff --git a/ydb/core/tx/columnshard/data_accessor/actor.cpp b/ydb/core/tx/columnshard/data_accessor/actor.cpp new file mode 100644 index 000000000000..2fe2bc5ee3e2 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/actor.cpp @@ -0,0 +1,9 @@ +#include "actor.h" + +namespace NKikimr::NOlap::NDataAccessorControl { + +void TActor::Handle(TEvAskDataAccessors::TPtr& ev) { + Manager.AskData(ev->Get()->GetRequest()); +} + +} diff --git a/ydb/core/tx/columnshard/data_accessor/actor.h b/ydb/core/tx/columnshard/data_accessor/actor.h new file mode 100644 index 000000000000..bc3e78186cee --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/actor.h @@ -0,0 +1,62 @@ +#pragma once +#include "controller.h" +#include "events.h" +#include "manager.h" + +#include +#include + +namespace NKikimr::NOlap::NDataAccessorControl { + +class TActor: public TActorBootstrapped { +private: + const ui64 TabletId; + const NActors::TActorId Parent; + TLocalManager Manager; + + void StartStopping() { + PassAway(); + } + + void Handle(TEvRegisterController::TPtr& ev) { + Manager.RegisterController(ev->Get()->ExtractController()); + } + void Handle(TEvUnregisterController::TPtr& ev) { + Manager.UnregisterController(ev->Get()->GetPathId()); + } + void Handle(TEvAddPortion::TPtr& ev) { + Manager.AddPortion(ev->Get()->ExtractAccessor()); + } + void Handle(TEvRemovePortion::TPtr& ev) { + Manager.RemovePortion(ev->Get()->GetPortion()); + } + void Handle(TEvAskDataAccessors::TPtr& ev); + +public: + TActor(const ui64 tabletId, const TActorId& parent) + : TabletId(tabletId) + , Parent(parent) { + } + ~TActor() = default; + + void Bootstrap() { + Become(&TThis::StateWait); + } + + STFUNC(StateWait) { + TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletId)("parent", Parent)( + "ev_type", ev->GetTypeName())); + switch (ev->GetTypeRewrite()) { + cFunc(NActors::TEvents::TEvPoison::EventType, StartStopping); + hFunc(TEvRegisterController, Handle); + hFunc(TEvUnregisterController, Handle); + hFunc(TEvAskDataAccessors, Handle); + hFunc(TEvRemovePortion, Handle); + hFunc(TEvAddPortion, Handle); + default: + AFL_VERIFY(false); + } + } +}; + +} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/controller.cpp b/ydb/core/tx/columnshard/data_accessor/controller.cpp new file mode 100644 index 000000000000..9a87599ae60b --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/controller.cpp @@ -0,0 +1,10 @@ +#include "controller.h" +#include "events.h" + +namespace NKikimr::NOlap { + +void TTabletDataAccessor::DoAskData(const std::shared_ptr& request) { + NActors::TActivationContext::Send(TabletActorId, std::make_unique(request)); +} + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/data_accessor/controller.h b/ydb/core/tx/columnshard/data_accessor/controller.h new file mode 100644 index 000000000000..c4ff6120c65c --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/controller.h @@ -0,0 +1,76 @@ +#pragma once +#include "request.h" + +namespace NKikimr::NOlap { + +class IGranuleDataAccessor { +private: + const ui64 PathId; + + virtual void DoAskData(const std::shared_ptr& request) = 0; + virtual void DoModifyPortions(const std::vector& add, const std::vector& remove) = 0; + +public: + virtual ~IGranuleDataAccessor() = default; + + ui64 GetPathId() const { + return PathId; + } + + IGranuleDataAccessor(const ui64 pathId) + : PathId(pathId) { + } + + void AskData(const std::shared_ptr& request) { + AFL_VERIFY(request); + AFL_VERIFY(request->HasSubscriber()); + return DoAskData(request); + } + void ModifyPortions(const std::vector& add, const std::vector& remove) { + return DoModifyPortions(add, remove); + } +}; + +class TMemDataAccessor: public IGranuleDataAccessor { +private: + using TBase = IGranuleDataAccessor; + virtual void DoAskData(const std::shared_ptr& request) override { + std::vector accessors; + auto& portions = request->StartFetching(GetPathId()); + for (auto&& i : portions) { + accessors.emplace_back(TPortionDataAccessor(i)); + } + request->AddData(GetPathId(), std::move(accessors)); + } + virtual void DoModifyPortions(const std::vector& /*add*/, const std::vector& /*remove*/) override { + } + +public: + TPortionDataAccessor BuildAccessor(const TPortionInfo::TConstPtr& portion) const { + return TPortionDataAccessor(portion); + } + + TMemDataAccessor(const ui64 pathId) + : TBase(pathId) { + } +}; + +class TTabletDataAccessor: public IGranuleDataAccessor { +private: + const NActors::TActorId TabletActorId; + using TBase = IGranuleDataAccessor; + virtual void DoAskData(const std::shared_ptr& request) override; + virtual void DoModifyPortions(const std::vector& /*add*/, const std::vector& /*remove*/) override { + } + +public: + TPortionDataAccessor BuildAccessor(const TPortionInfo::TConstPtr& portion) const { + return TPortionDataAccessor(portion); + } + + TTabletDataAccessor(const ui64 pathId) + : TBase(pathId) { + } +}; + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/data_accessor/events.cpp b/ydb/core/tx/columnshard/data_accessor/events.cpp new file mode 100644 index 000000000000..a1ec74cd603d --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/events.cpp @@ -0,0 +1,5 @@ +#include "events.h" + +namespace NKikimr::NOlap::NDataAccessorControl { + +} diff --git a/ydb/core/tx/columnshard/data_accessor/events.h b/ydb/core/tx/columnshard/data_accessor/events.h new file mode 100644 index 000000000000..6e5061781335 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/events.h @@ -0,0 +1,77 @@ +#pragma once + +#include "controller.h" + +#include +#include + +#include +#include + +namespace NKikimr::NOlap { +class IGranuleDataAccessor; +class TDataAccessorsRequest; +} // namespace NKikimr::NOlap + +namespace NKikimr::NOlap::NDataAccessorControl { + +class TEvAddPortion: public NActors::TEventLocal { +private: + TPortionDataAccessor Accessor; + +public: + TPortionDataAccessor ExtractAccessor() { + return std::move(Accessor); + } + + explicit TEvAddPortion(const TPortionDataAccessor& accessor) + : Accessor(accessor) { + } +}; + +class TEvRemovePortion: public NActors::TEventLocal { +private: + YDB_READONLY_DEF(TPortionInfo::TConstPtr, Portion); + +public: + explicit TEvRemovePortion(const TPortionInfo::TConstPtr& portion) + : Portion(portion) { + } +}; + +class TEvRegisterController: public NActors::TEventLocal { +private: + std::unique_ptr Controller; + +public: + std::unique_ptr ExtractController() { + return std::move(Controller); + } + + explicit TEvRegisterController(std::unique_ptr&& accessor) + : Controller(std::move(accessor)) { + } +}; + +class TEvUnregisterController + : public NActors::TEventLocal { +private: + YDB_READONLY(ui64, PathId, 0); + +public: + explicit TEvUnregisterController(const ui64 pathId) + : PathId(pathId) { + } +}; + +class TEvAskDataAccessors: public NActors::TEventLocal { +private: + YDB_READONLY_DEF(std::shared_ptr, Request); + +public: + explicit TEvAskDataAccessors(const std::shared_ptr& request) + : Request(request) { + } +}; + +} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/manager.cpp b/ydb/core/tx/columnshard/data_accessor/manager.cpp new file mode 100644 index 000000000000..f3a4cde0ec8e --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/manager.cpp @@ -0,0 +1,5 @@ +#include "manager.h" + +namespace NKikimr::NOlap::NDataAccessorControl { + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/data_accessor/manager.h b/ydb/core/tx/columnshard/data_accessor/manager.h new file mode 100644 index 000000000000..e73c27ed8c75 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/manager.h @@ -0,0 +1,108 @@ +#pragma once +#include "controller.h" +#include "events.h" +#include "request.h" + +#include + +namespace NKikimr::NOlap::NDataAccessorControl { + +class IDataAccessorsManager { +private: + virtual void DoAskData(const std::shared_ptr& request) = 0; + virtual void DoRegisterController(std::unique_ptr&& controller) = 0; + virtual void DoUnregisterController(const ui64 pathId) = 0; + virtual void DoAddPortion(const TPortionDataAccessor& accessor) = 0; + virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portion) = 0; + +public: + virtual ~IDataAccessorsManager() = default; + + void AddPortion(const TPortionDataAccessor& accessor) { + DoAddPortion(accessor); + } + void RemovePortion(const TPortionInfo::TConstPtr& portion) { + DoRemovePortion(portion); + } + void AskData(const std::shared_ptr& request) { + AFL_VERIFY(request); + return DoAskData(request); + } + void RegisterController(std::unique_ptr&& controller) { + AFL_VERIFY(controller); + return DoRegisterController(std::move(controller)); + } + void UnregisterController(const ui64 pathId) { + return DoUnregisterController(pathId); + } +}; + +class TDataAccessorsManagerContainer: public NBackgroundTasks::TControlInterfaceContainer { +private: + using TBase = NBackgroundTasks::TControlInterfaceContainer; + +public: + using TBase::TBase; +}; + +class TActorAccessorsManager: public IDataAccessorsManager { +private: + const NActors::TActorId ActorId; + virtual void DoAskData(const std::shared_ptr& request) override { + NActors::TActivationContext::Send(ActorId, std::make_unique(request)); + } + virtual void DoRegisterController(std::unique_ptr&& controller) override { + NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(controller))); + } + virtual void DoUnregisterController(const ui64 pathId) override { + NActors::TActivationContext::Send(ActorId, std::make_unique(pathId)); + } + virtual void DoAddPortion(const TPortionDataAccessor& accessor) override { + NActors::TActivationContext::Send(ActorId, std::make_unique(accessor)); + } + virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portion) override { + NActors::TActivationContext::Send(ActorId, std::make_unique(portion)); + } + +public: + TActorAccessorsManager(const NActors::TActorId& actorId) + : ActorId(actorId) { + } +}; + +class TLocalManager: public IDataAccessorsManager { +private: + THashMap> Managers; + + virtual void DoAskData(const std::shared_ptr& request) override { + for (auto&& i : request->GetPathIds()) { + auto it = Managers.find(i); + if (it == Managers.end()) { + request->AddData(i, TConclusionStatus::Fail("incorrect pathId")); + } else { + it->second->AskData(request); + } + } + } + virtual void DoRegisterController(std::unique_ptr&& controller) override { + AFL_VERIFY(Managers.emplace(controller->GetPathId(), std::move(controller)).second); + } + virtual void DoUnregisterController(const ui64 pathId) override { + AFL_VERIFY(Managers.erase(pathId)); + } + virtual void DoAddPortion(const TPortionDataAccessor& accessor) override { + auto it = Managers.find(accessor.GetPortionInfo().GetPathId()); + AFL_VERIFY(it != Managers.end()); + it->second->ModifyPortions( { accessor }, {} ); + } + virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portionInfo) override { + auto it = Managers.find(portionInfo->GetPathId()); + AFL_VERIFY(it != Managers.end()); + it->second->ModifyPortions({}, { portionInfo->GetPortionId() }); + } + +public: + TLocalManager() = default; +}; + +} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/request.cpp b/ydb/core/tx/columnshard/data_accessor/request.cpp new file mode 100644 index 000000000000..bdf3f6cf2af8 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/request.cpp @@ -0,0 +1,10 @@ +#include "request.h" + +namespace NKikimr::NOlap { + +void IDataAccessorRequestsSubscriber::RegisterRequestId(const TDataAccessorsRequest& request) { + AFL_VERIFY(!request.IsFetched()); + AFL_VERIFY(RequestIds.emplace(request.GetRequestId()).second); +} + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/data_accessor/request.h b/ydb/core/tx/columnshard/data_accessor/request.h new file mode 100644 index 000000000000..be598fcf6896 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/request.h @@ -0,0 +1,258 @@ +#pragma once +#include +#include + +namespace NKikimr::NOlap { + +class TDataAccessorsRequest; + +class TDataAccessorsResult { +private: + THashMap ErrorsByPathId; + THashMap> AccessorsByPathId; + THashMap PortionsById; + std::vector Portions; + +public: + const std::vector& GetPortions() const { + return Portions; + } + + void Merge(TDataAccessorsResult&& result) { + for (auto&& i : result.ErrorsByPathId) { + AFL_VERIFY(ErrorsByPathId.emplace(i.first, i.second).second); + } + for (auto&& i : result.AccessorsByPathId) { + AFL_VERIFY(AccessorsByPathId.emplace(i.first, std::move(i.second)).second); + } + for (auto&& i : result.PortionsById) { + AFL_VERIFY(PortionsById.emplace(i.first, std::move(i.second)).second); + } + Portions.insert(Portions.end(), result.Portions.begin(), result.Portions.end()); + } + + const TPortionDataAccessor& GetPortionAccessorVerified(const ui64 portionId) const { + auto it = PortionsById.find(portionId); + AFL_VERIFY(it != PortionsById.end()); + return it->second; + } + + std::vector ExtractPortionsVector() { + return std::move(Portions); + } + + void AddData(const ui64 pathId, TConclusion>&& accessors) { + if (accessors.IsFail()) { + AFL_VERIFY(ErrorsByPathId.emplace(pathId, accessors.GetErrorMessage()).second); + } else { + auto info = AccessorsByPathId.emplace(pathId, accessors.DetachResult()); + AFL_VERIFY(info.second); + for (auto&& i : info.first->second) { + AFL_VERIFY(PortionsById.emplace(i.GetPortionInfo().GetPortionId(), i).second); + Portions.emplace_back(i); + } + } + } + + bool HasErrors() const { + return ErrorsByPathId.size(); + } +}; + +class IDataAccessorRequestsSubscriber { +private: + THashSet RequestIds; + + virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) = 0; + + void OnRequestsFinished(TDataAccessorsResult&& result) { + DoOnRequestsFinished(std::move(result)); + } + + void RegisterRequestId(const TDataAccessorsRequest& request); + + friend class TDataAccessorsRequest; + std::optional Result; + +public: + void OnResult(const ui32 requestId, TDataAccessorsResult&& result) { + AFL_VERIFY(RequestIds.erase(requestId)); + if (!Result) { + Result = std::move(result); + } else { + Result->Merge(std::move(result)); + } + if (RequestIds.empty()) { + OnRequestsFinished(std::move(*Result)); + } + } + + virtual ~IDataAccessorRequestsSubscriber() = default; +}; + +class TPathFetchingState { +public: + enum class EFetchStage { + Preparing, + Fetching, + Error, + Fetched + }; + +private: + const ui64 PathId; + + YDB_READONLY(EFetchStage, Stage, EFetchStage::Preparing); + YDB_READONLY_DEF(TString, ErrorMessage); + THashSet PortionIds; + std::vector Portions; + +public: + TPathFetchingState(const ui64 pathId) + : PathId(pathId) { + } + + const std::vector& GetPortions() const { + return Portions; + } + + void AddPortion(const TPortionInfo::TConstPtr& portion) { + AFL_VERIFY(Stage == EFetchStage::Preparing); + AFL_VERIFY(PortionIds.emplace(portion->GetPortionId()).second); + Portions.emplace_back(portion); + AFL_VERIFY(portion->GetPathId() == PathId); + } + + void StartFetch() { + AFL_VERIFY(Stage == EFetchStage::Preparing); + Stage = EFetchStage::Fetching; + } + + void OnError(const TString& errorMessage) { + AFL_VERIFY(Stage == EFetchStage::Fetching); + Stage = EFetchStage::Error; + ErrorMessage = errorMessage; + } + + void OnFetched() { + AFL_VERIFY(Stage == EFetchStage::Fetching); + Stage = EFetchStage::Fetched; + } +}; + +class TDataAccessorsRequest { +private: + static inline TAtomicCounter Counter = 0; + ui32 FetchStage = 0; + YDB_READONLY(ui64, RequestId, Counter.Inc()); + THashSet PortionIds; + THashMap PathIdStatus; + THashSet PathIds; + TDataAccessorsResult AccessorsByPathId; + std::optional> ColumnIds; + std::optional> IndexIds; + + TAtomicCounter PreparingCount = 0; + TAtomicCounter FetchingCount = 0; + TAtomicCounter ReadyCount = 0; + + std::shared_ptr Subscriber; + +public: + TDataAccessorsRequest() = default; + + bool HasSubscriber() const { + return !!Subscriber; + } + + ui32 GetSize() const { + return PortionIds.size(); + } + + const THashSet& GetPathIds() const { + return PathIds; + } + + bool IsEmpty() const { + return PortionIds.empty(); + } + + void RegisterSubscriber(const std::shared_ptr& subscriber) { + AFL_VERIFY(!Subscriber); + AFL_VERIFY(FetchStage == 0); + Subscriber = subscriber; + Subscriber->RegisterRequestId(*this); + } + + const std::vector& StartFetching(const ui64 pathId) { + AFL_VERIFY(!!Subscriber); + AFL_VERIFY(FetchStage <= 1); + FetchStage = 1; + + auto it = PathIdStatus.find(pathId); + AFL_VERIFY(it != PathIdStatus.end()); + it->second.StartFetch(); + + FetchingCount.Inc(); + AFL_VERIFY(PreparingCount.Dec() >= 0); + + return it->second.GetPortions(); + } + + void AddPortion(const TPortionInfo::TConstPtr& portion) { + AFL_VERIFY(portion); + AFL_VERIFY(FetchStage <= 1); + AFL_VERIFY(PortionIds.emplace(portion->GetPortionId()).second); + PathIds.emplace(portion->GetPathId()); + auto it = PathIdStatus.find(portion->GetPathId()); + if (it == PathIdStatus.end()) { + PreparingCount.Inc(); + it = PathIdStatus.emplace(portion->GetPathId(), portion->GetPathId()).first; + } + it->second.AddPortion(portion); + } + + bool IsFetched() const { + return FetchStage == 2; + } + + void AddData(const ui64 pathId, TConclusion>&& accessors) { + AFL_VERIFY(FetchStage == 1); + { + auto itStatus = PathIdStatus.find(pathId); + AFL_VERIFY(itStatus != PathIdStatus.end()); + if (accessors.IsFail()) { + itStatus->second.OnError(accessors.GetErrorMessage()); + for (auto&& p : itStatus->second.GetPortions()) { + AFL_VERIFY(PortionIds.erase(p->GetPortionId())); + } + } else { + AFL_VERIFY(itStatus->second.GetPortions().size() == accessors->size()); + itStatus->second.OnFetched(); + for (auto&& acc : *accessors) { + AFL_VERIFY(PortionIds.erase(acc.GetPortionInfo().GetPortionId())); + } + } + PathIdStatus.erase(itStatus); + AFL_VERIFY(FetchingCount.Dec() >= 0); + ReadyCount.Inc(); + AccessorsByPathId.AddData(pathId, std::move(accessors)); + } + AFL_VERIFY(PathIdStatus.empty() == PortionIds.empty()); + if (PathIdStatus.empty()) { + AFL_VERIFY(!PreparingCount.Val()); + AFL_VERIFY(!FetchingCount.Val()); + FetchStage = 2; + Subscriber->OnResult(RequestId, std::move(AccessorsByPathId)); + Subscriber = nullptr; + } + } + + void AddData(THashMap>&& accessors) { + for (auto&& i : accessors) { + AddData(i.first, std::move(i.second)); + } + } +}; + +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/data_accessor/ya.make b/ydb/core/tx/columnshard/data_accessor/ya.make new file mode 100644 index 000000000000..31cf21d438a0 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SRCS( + actor.cpp + events.cpp + controller.cpp + request.cpp + manager.cpp +) + +PEERDIR( + ydb/library/actors/core + ydb/core/tx/columnshard/engines/portions +) + +END() diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index 8ac215cb6f63..5350c8f2f928 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -187,6 +187,17 @@ class TConstructionContext: TNonCopyable { class TGranuleMeta; +class TDataAccessorsInitializationContext { +private: + YDB_READONLY_DEF(std::shared_ptr, VersionedIndex); + +public: + TDataAccessorsInitializationContext(const std::shared_ptr& versionedIndex) + : VersionedIndex(versionedIndex) { + AFL_VERIFY(VersionedIndex); + } +}; + class TColumnEngineChanges { public: enum class EStage: ui32 { @@ -205,6 +216,7 @@ class TColumnEngineChanges { const TString TaskIdentifier = TGUID::CreateTimebased().AsGuidString(); protected: + std::optional FetchedDataAccessors; virtual void DoDebugString(TStringOutput& out) const = 0; virtual void DoCompile(TFinalizationContext& context) = 0; virtual void DoOnAfterCompile() {} @@ -214,7 +226,7 @@ class TColumnEngineChanges { virtual bool NeedConstruction() const { return true; } - virtual void DoStart(NColumnShard::TColumnShard& self) = 0; + virtual void DoStart(NColumnShard::TColumnShard& context) = 0; virtual TConclusionStatus DoConstructBlobs(TConstructionContext& context) noexcept = 0; virtual void OnAbortEmergency() { } @@ -229,7 +241,35 @@ class TColumnEngineChanges { return DoBuildDataLock(); } + std::shared_ptr PortionsToAccess = std::make_shared(); + virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& context) = 0; + public: + std::shared_ptr ExtractDataAccessorsRequest() const { + AFL_VERIFY(!!PortionsToAccess); + return std::move(PortionsToAccess); + } + + const TPortionDataAccessor& GetPortionDataAccessor(const ui64 portionId) const { + AFL_VERIFY(FetchedDataAccessors); + return FetchedDataAccessors->GetPortionAccessorVerified(portionId); + } + + std::vector GetPortionDataAccessors(const std::vector& portions) const { + AFL_VERIFY(FetchedDataAccessors); + std::vector result; + for (auto&& i : portions) { + result.emplace_back(GetPortionDataAccessor(i->GetPortionId())); + } + return result; + } + + void SetFetchedDataAccessors(TDataAccessorsResult&& result, const TDataAccessorsInitializationContext& context) { + AFL_VERIFY(!FetchedDataAccessors); + FetchedDataAccessors = std::move(result); + OnDataAccessorsInitialized(context); + } + class IMemoryPredictor { public: virtual ui64 AddPortion(const TPortionInfo::TConstPtr& portionInfo) = 0; diff --git a/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp b/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp index 8dfbbe0010c9..2eecd8ed6464 100644 --- a/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp +++ b/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp @@ -65,7 +65,7 @@ bool TTieringProcessContext::AddPortion( } else { Counters.OnPortionToEvict(info->GetTotalBlobBytes(), *dWait); } - it->second.back().GetTask()->AddPortionToEvict(TPortionDataAccessor(info), std::move(features)); + it->second.back().GetTask()->AddPortionToEvict(info, std::move(features)); AFL_VERIFY(!it->second.back().GetTask()->HasPortionsToRemove())("rw", features.GetRWAddress().DebugString())("f", it->first.DebugString()); } return true; diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index 2160b2734689..2b8f5573f1ad 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -12,7 +12,7 @@ void TCleanupPortionsColumnEngineChanges::DoDebugString(TStringOutput& out) cons if (ui32 dropped = PortionsToDrop.size()) { out << "drop " << dropped << " portions"; for (auto& portionInfo : PortionsToDrop) { - out << portionInfo.GetPortionInfo().DebugString(); + out << portionInfo->DebugString(); } } } @@ -23,7 +23,7 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC return; } THashMap> blobIdsByStorage; - for (auto&& p : PortionsToDrop) { + for (auto&& p : FetchedDataAccessors->GetPortions()) { p.RemoveFromDatabase(context.DBWrapper); p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPortionInfo().GetPathId()); @@ -38,15 +38,14 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) { for (auto& portionInfo : PortionsToDrop) { - if (!context.EngineLogs.ErasePortion(portionInfo.GetPortionInfo())) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "Cannot erase portion")("portion", portionInfo.GetPortionInfo().DebugString()); + if (!context.EngineLogs.ErasePortion(*portionInfo)) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "Cannot erase portion")("portion", portionInfo->DebugString()); } } if (self) { self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_PORTIONS_ERASED, PortionsToDrop.size()); for (auto&& p : PortionsToDrop) { - self->Counters.GetTabletCounters()->OnDropPortionEvent( - p.GetPortionInfo().GetTotalRawBytes(), p.GetPortionInfo().GetTotalBlobBytes(), p.GetPortionInfo().GetRecordsCount()); + self->Counters.GetTabletCounters()->OnDropPortionEvent(p->GetTotalRawBytes(), p->GetTotalBlobBytes(), p->GetRecordsCount()); } } } diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h index 5448d8e9dfeb..ebc3b7dcee7a 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h @@ -6,9 +6,13 @@ namespace NKikimr::NOlap { class TCleanupPortionsColumnEngineChanges: public TColumnEngineChanges { private: using TBase = TColumnEngineChanges; - THashMap> BlobsToForget; THashMap>> StoragePortions; + std::vector PortionsToDrop; + protected: + virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& /*context*/) override { + } + virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; virtual void DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) override; @@ -37,7 +41,14 @@ class TCleanupPortionsColumnEngineChanges: public TColumnEngineChanges { } - std::vector PortionsToDrop; + const std::vector& GetPortionsToDrop() const { + return PortionsToDrop; + } + + void AddPortionToDrop(const TPortionInfo::TConstPtr& portion) { + PortionsToDrop.emplace_back(portion); + PortionsToAccess->AddPortion(portion); + } virtual ui32 GetWritePortionsCount() const override { return 0; diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_tables.h b/ydb/core/tx/columnshard/engines/changes/cleanup_tables.h index 33c7fe34cb1d..97fc10e22beb 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_tables.h +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_tables.h @@ -10,6 +10,10 @@ class TCleanupTablesColumnEngineChanges: public TColumnEngineChanges { virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; virtual void DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) override; + virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& /*context*/) override { + + } + virtual void DoStart(NColumnShard::TColumnShard& self) override; virtual void DoOnFinish(NColumnShard::TColumnShard& self, TChangesFinishContext& context) override; virtual void DoDebugString(TStringOutput& out) const override; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index d45d7f350db5..e9db018e50cd 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -14,7 +14,7 @@ void TCompactColumnEngineChanges::DoDebugString(TStringOutput& out) const { if (ui32 switched = SwitchedPortions.size()) { out << "switch " << switched << " portions:("; for (auto& portionInfo : SwitchedPortions) { - out << portionInfo.GetPortionInfo().DebugString(false); + out << portionInfo->DebugString(false); } out << "); "; } @@ -32,19 +32,6 @@ void TCompactColumnEngineChanges::DoCompile(TFinalizationContext& context) { void TCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { TBase::DoStart(self); - THashMap> blobRanges; - auto& index = self.GetIndexAs().GetVersionedIndex(); - for (const auto& p : SwitchedPortions) { - p.FillBlobRangesByStorage(blobRanges, index); - } - - for (const auto& p : blobRanges) { - auto action = BlobsAction.GetReading(p.first); - for (auto&& b : p.second) { - action->AddRange(b); - } - } - self.BackgroundController.StartCompaction(NKikimr::NOlap::TPlanCompactionInfo(GranuleMeta->GetPathId())); NeedGranuleStatusProvide = true; GranuleMeta->OnCompactionStarted(); @@ -69,17 +56,16 @@ void TCompactColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, T } TCompactColumnEngineChanges::TCompactColumnEngineChanges( - std::shared_ptr granule, const std::vector& portions, const TSaverContext& saverContext) + std::shared_ptr granule, const std::vector& portions, const TSaverContext& saverContext) : TBase(saverContext, NBlobOperations::EConsumer::GENERAL_COMPACTION) , GranuleMeta(granule) { Y_ABORT_UNLESS(GranuleMeta); - SwitchedPortions.reserve(portions.size()); for (const auto& portionInfo : portions) { - Y_ABORT_UNLESS(!portionInfo.GetPortionInfo().HasRemoveSnapshot()); + Y_ABORT_UNLESS(!portionInfo->HasRemoveSnapshot()); SwitchedPortions.emplace_back(portionInfo); - AddPortionToRemove(portionInfo.GetPortionInfoPtr()); - Y_ABORT_UNLESS(portionInfo.GetPortionInfo().GetPathId() == GranuleMeta->GetPathId()); + AddPortionToRemove(portionInfo); + Y_ABORT_UNLESS(portionInfo->GetPathId() == GranuleMeta->GetPathId()); } // Y_ABORT_UNLESS(SwitchedPortions.size()); } diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.h b/ydb/core/tx/columnshard/engines/changes/compaction.h index fd080ad1b130..61b8a67d6faa 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction.h @@ -12,6 +12,7 @@ class TCompactColumnEngineChanges: public TChangesWithAppend { using TBase = TChangesWithAppend; bool NeedGranuleStatusProvide = false; protected: + std::vector SwitchedPortions; // Portions that would be replaced by new ones std::shared_ptr GranuleMeta; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; @@ -29,12 +30,34 @@ class TCompactColumnEngineChanges: public TChangesWithAppend { return std::make_shared(TypeString() + "::" + GetTaskIdentifier(), pathIds); } -public: - std::vector SwitchedPortions; // Portions that would be replaced by new ones + virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& context) override { + TBase::OnDataAccessorsInitialized(context); + THashMap> blobRanges; + for (const auto& p : SwitchedPortions) { + GetPortionDataAccessor(p->GetPortionId()).FillBlobRangesByStorage(blobRanges, *context.GetVersionedIndex()); + } + + for (const auto& p : blobRanges) { + auto action = BlobsAction.GetReading(p.first); + for (auto&& b : p.second) { + action->AddRange(b); + } + } + } - TCompactColumnEngineChanges(std::shared_ptr granule, const std::vector& portions, const TSaverContext& saverContext); +public: + TCompactColumnEngineChanges(std::shared_ptr granule, const std::vector& portions, const TSaverContext& saverContext); ~TCompactColumnEngineChanges(); + const std::vector& GetSwitchedPortions() const { + return SwitchedPortions; + } + + void AddSwitchedPortion(const TPortionInfo::TConstPtr& portion) { + SwitchedPortions.emplace_back(portion); + PortionsToAccess->AddPortion(portion); + } + static TString StaticTypeName() { return "CS::GENERAL"; } diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 0cf2b989e531..4e76243b0410 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -1,4 +1,3 @@ - #include "general_compaction.h" #include "compaction/merger.h" @@ -105,18 +104,19 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks( { THashMap schemas; for (auto& portion : SwitchedPortions) { - auto dataSchema = portion.GetPortionInfo().GetSchema(context.SchemaVersions); + auto dataSchema = portion->GetSchema(context.SchemaVersions); schemas.emplace(dataSchema->GetVersion(), dataSchema); } dataColumnIds = ISnapshotSchema::GetColumnsWithDifferentDefaults(schemas, resultSchema); } for (auto&& i : SwitchedPortions) { - stats->Merge(i.GetSerializationStat(*resultSchema)); - if (i.GetPortionInfo().GetMeta().GetDeletionsCount()) { + const auto& accessor = GetPortionDataAccessor(i->GetPortionId()); + stats->Merge(accessor.GetSerializationStat(*resultSchema)); + if (i->GetMeta().GetDeletionsCount()) { dataColumnIds.emplace((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG); } if (dataColumnIds.size() != resultSchema->GetColumnsCount()) { - for (auto id : i.GetColumnIds()) { + for (auto id : accessor.GetColumnIds()) { if (resultSchema->HasColumnId(id)) { dataColumnIds.emplace(id); } @@ -166,11 +166,11 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc TSimplePortionsGroupInfo compactedPortions; THashMap portionGroups; for (auto&& i : SwitchedPortions) { - portionGroups[i.GetPortionInfo().GetMeta().GetCompactionLevel()].AddPortion(i.GetPortionInfoPtr()); - if (i.GetPortionInfo().GetMeta().GetProduced() == TPortionMeta::EProduced::INSERTED) { - insertedPortions.AddPortion(i.GetPortionInfoPtr()); - } else if (i.GetPortionInfo().GetMeta().GetProduced() == TPortionMeta::EProduced::SPLIT_COMPACTED) { - compactedPortions.AddPortion(i.GetPortionInfoPtr()); + portionGroups[i->GetMeta().GetCompactionLevel()].AddPortion(i); + if (i->GetMeta().GetProduced() == TPortionMeta::EProduced::INSERTED) { + insertedPortions.AddPortion(i); + } else if (i->GetMeta().GetProduced() == TPortionMeta::EProduced::SPLIT_COMPACTED) { + compactedPortions.AddPortion(i); } else { AFL_VERIFY(false); } @@ -184,7 +184,7 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc { std::vector portions = - TReadPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs, context.SchemaVersions); + TReadPortionInfoWithBlobs::RestorePortions(GetPortionDataAccessors(SwitchedPortions), Blobs, context.SchemaVersions); BuildAppendedPortionsByChunks(context, std::move(portions)); } @@ -192,7 +192,7 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc TStringBuilder sbSwitched; sbSwitched << ""; for (auto&& p : SwitchedPortions) { - sbSwitched << p.GetPortionInfo().DebugString() << ";"; + sbSwitched << p->DebugString() << ";"; } sbSwitched << ""; @@ -238,39 +238,7 @@ std::shared_ptr TGeneralCo ui64 TGeneralCompactColumnEngineChanges::TMemoryPredictorChunkedPolicy::AddPortion(const TPortionInfo::TConstPtr& portionInfo) { SumMemoryFix += portionInfo->GetRecordsCount() * (2 * sizeof(ui64) + sizeof(ui32) + sizeof(ui16)) + portionInfo->GetTotalBlobBytes(); - ++PortionsCount; - SumMemoryDelta = 0; - - auto it = MaxMemoryByColumnChunk.begin(); - const auto advanceIterator = [&](const ui32 columnId, const ui64 maxColumnChunkRawBytes) { - while (it != MaxMemoryByColumnChunk.end() && it->ColumnId < columnId) { - ++it; - } - if (it == MaxMemoryByColumnChunk.end() || columnId < it->ColumnId) { - it = MaxMemoryByColumnChunk.insert(it, TColumnInfo(columnId)); - } - it->MemoryUsage += maxColumnChunkRawBytes; - SumMemoryDelta = std::max(SumMemoryDelta, it->MemoryUsage); - }; - ui32 columnId = 0; - ui64 maxChunkSize = 0; - for (auto&& i : TPortionDataAccessor(portionInfo).GetRecords()) { - if (columnId != i.GetColumnId()) { - if (columnId) { - advanceIterator(columnId, maxChunkSize); - } - columnId = i.GetColumnId(); - maxChunkSize = 0; - } - if (maxChunkSize < i.GetMeta().GetRawBytes()) { - maxChunkSize = i.GetMeta().GetRawBytes(); - } - } - advanceIterator(columnId, maxChunkSize); - - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("memory_prediction_after", SumMemoryFix + SumMemoryDelta)( - "portion_info", portionInfo->DebugString()); - return SumMemoryFix + SumMemoryDelta; + return SumMemoryFix + ((ui64)500 << 20); } } // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.h b/ydb/core/tx/columnshard/engines/changes/general_compaction.h index 0c20c123f91c..ee20088f0e39 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.h @@ -36,7 +36,7 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { auto predictor = BuildMemoryPredictor(); ui64 result = 0; for (auto& p : SwitchedPortions) { - result = predictor->AddPortion(p.GetPortionInfoPtr()); + result = predictor->AddPortion(p); } return result; } @@ -49,21 +49,7 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { class TMemoryPredictorChunkedPolicy: public IMemoryPredictor { private: - ui64 SumMemoryDelta = 0; ui64 SumMemoryFix = 0; - ui32 PortionsCount = 0; - class TColumnInfo { - public: - const ui32 ColumnId; - ui64 MemoryUsage = 0; - TColumnInfo(const ui32 columnId) - : ColumnId(columnId) - { - - } - }; - std::list MaxMemoryByColumnChunk; - public: virtual ui64 AddPortion(const TPortionInfo::TConstPtr& portionInfo) override; }; diff --git a/ydb/core/tx/columnshard/engines/changes/ttl.cpp b/ydb/core/tx/columnshard/engines/changes/ttl.cpp index a1f8276e68ab..72f49e67e4f4 100644 --- a/ydb/core/tx/columnshard/engines/changes/ttl.cpp +++ b/ydb/core/tx/columnshard/engines/changes/ttl.cpp @@ -17,19 +17,7 @@ void TTTLColumnEngineChanges::DoDebugString(TStringOutput& out) const { void TTTLColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { Y_ABORT_UNLESS(PortionsToEvict.size() || HasPortionsToRemove()); - THashMap> blobRanges; - auto& engine = self.MutableIndexAs(); - auto& index = engine.GetVersionedIndex(); - for (const auto& p : PortionsToEvict) { - p.GetPortionInfo().FillBlobRangesByStorage(blobRanges, index); - } - for (auto&& i : blobRanges) { - auto action = BlobsAction.GetReading(i.first); - for (auto&& b : i.second) { - action->AddRange(b); - } - } - engine.GetActualizationController()->StartActualization(RWAddress); + self.GetIndexAs().GetActualizationController()->StartActualization(RWAddress); } void TTTLColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TChangesFinishContext& /*context*/) { @@ -38,7 +26,7 @@ void TTTLColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TChan if (IsAborted()) { THashMap> restoreIndexAddresses; for (auto&& i : PortionsToEvict) { - AFL_VERIFY(restoreIndexAddresses[i.GetPortionInfo().GetPortionInfo().GetPathId()].emplace(i.GetPortionInfo().GetPortionInfo().GetPortionId()).second); + AFL_VERIFY(restoreIndexAddresses[i.GetPortionInfo()->GetPathId()].emplace(i.GetPortionInfo()->GetPortionId()).second); } for (auto&& i : GetPortionsToRemove()) { AFL_VERIFY(restoreIndexAddresses[i.first.GetPathId()].emplace(i.first.GetPortionId()).second); @@ -49,13 +37,14 @@ void TTTLColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TChan std::optional TTTLColumnEngineChanges::UpdateEvictedPortion( TPortionForEviction& info, NBlobOperations::NRead::TCompositeReadBlobs& srcBlobs, TConstructionContext& context) const { - const TPortionInfo& portionInfo = info.GetPortionInfo().GetPortionInfo(); + const TPortionInfo& portionInfo = *info.GetPortionInfo(); auto& evictFeatures = info.GetFeatures(); auto blobSchema = portionInfo.GetSchema(context.SchemaVersions); Y_ABORT_UNLESS(portionInfo.GetMeta().GetTierName() != evictFeatures.GetTargetTierName() || blobSchema->GetVersion() < evictFeatures.GetTargetScheme()->GetVersion()); - auto portionWithBlobs = TReadPortionInfoWithBlobs::RestorePortion(info.GetPortionInfo(), srcBlobs, blobSchema->GetIndexInfo()); + auto portionWithBlobs = TReadPortionInfoWithBlobs::RestorePortion( + GetPortionDataAccessor(info.GetPortionInfo()->GetPortionId()), srcBlobs, blobSchema->GetIndexInfo()); std::optional result = TReadPortionInfoWithBlobs::SyncPortion(std::move(portionWithBlobs), blobSchema, evictFeatures.GetTargetScheme(), evictFeatures.GetTargetTierName(), SaverContext.GetStoragesManager(), context.Counters.SplitterCounters); @@ -68,7 +57,7 @@ NKikimr::TConclusionStatus TTTLColumnEngineChanges::DoConstructBlobs(TConstructi for (auto&& info : PortionsToEvict) { if (auto pwb = UpdateEvictedPortion(info, Blobs, context)) { - AddPortionToRemove(info.GetPortionInfo().GetPortionInfoPtr()); + AddPortionToRemove(info.GetPortionInfo(), false); AppendedPortions.emplace_back(std::move(*pwb)); } } diff --git a/ydb/core/tx/columnshard/engines/changes/ttl.h b/ydb/core/tx/columnshard/engines/changes/ttl.h index 106c5f12f15f..3d027b6aa0a7 100644 --- a/ydb/core/tx/columnshard/engines/changes/ttl.h +++ b/ydb/core/tx/columnshard/engines/changes/ttl.h @@ -13,10 +13,10 @@ class TTTLColumnEngineChanges: public TChangesWithAppend { class TPortionForEviction { private: - TPortionDataAccessor PortionInfo; + TPortionInfo::TConstPtr PortionInfo; TPortionEvictionFeatures Features; public: - TPortionForEviction(const TPortionDataAccessor& portion, TPortionEvictionFeatures&& features) + TPortionForEviction(const TPortionInfo::TConstPtr& portion, TPortionEvictionFeatures&& features) : PortionInfo(portion) , Features(std::move(features)) { }; @@ -29,7 +29,7 @@ class TTTLColumnEngineChanges: public TChangesWithAppend { return Features; } - const TPortionDataAccessor& GetPortionInfo() const { + const TPortionInfo::TConstPtr& GetPortionInfo() const { return PortionInfo; } }; @@ -49,16 +49,30 @@ class TTTLColumnEngineChanges: public TChangesWithAppend { auto predictor = BuildMemoryPredictor(); ui64 result = 0; for (auto& p : PortionsToEvict) { - result = predictor->AddPortion(p.GetPortionInfo().GetPortionInfoPtr()); + result = predictor->AddPortion(p.GetPortionInfo()); } return result; } virtual std::shared_ptr DoBuildDataLockImpl() const override { const auto pred = [](const TPortionForEviction& p) { - return p.GetPortionInfo().GetPortionInfo().GetAddress(); + return p.GetPortionInfo()->GetAddress(); }; return std::make_shared(TypeString() + "::" + RWAddress.DebugString() + "::" + GetTaskIdentifier(), PortionsToEvict, pred); } + virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& context) override { + TBase::OnDataAccessorsInitialized(context); + THashMap> blobRanges; + for (const auto& p : PortionsToEvict) { + GetPortionDataAccessor(p.GetPortionInfo()->GetPortionId()).FillBlobRangesByStorage(blobRanges, *context.GetVersionedIndex()); + } + for (auto&& i : blobRanges) { + auto action = BlobsAction.GetReading(i.first); + for (auto&& b : i.second) { + action->AddRange(b); + } + } + } + public: class TMemoryPredictorSimplePolicy: public IMemoryPredictor { private: @@ -88,9 +102,10 @@ class TTTLColumnEngineChanges: public TChangesWithAppend { ui32 GetPortionsToEvictCount() const { return PortionsToEvict.size(); } - void AddPortionToEvict(const TPortionDataAccessor& info, TPortionEvictionFeatures&& features) { - AFL_VERIFY(!info.GetPortionInfo().HasRemoveSnapshot()); + void AddPortionToEvict(const TPortionInfo::TConstPtr& info, TPortionEvictionFeatures&& features) { + AFL_VERIFY(!info->HasRemoveSnapshot()); PortionsToEvict.emplace_back(info, std::move(features)); + PortionsToAccess->AddPortion(info); } static TString StaticTypeName() { diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 46c39964da59..40eab1b93585 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -21,7 +21,8 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, portionCopy.SetRemoveSnapshot(context.Snapshot); }; context.EngineLogs.GetGranuleVerified(i->GetPathId()) - .ModifyPortionOnExecute(context.DBWrapper, i, pred, schemaPtr->GetIndexInfo().GetPKFirstColumnId()); + .ModifyPortionOnExecute( + context.DBWrapper, GetPortionDataAccessor(i->GetPortionId()), pred, schemaPtr->GetIndexInfo().GetPKFirstColumnId()); } const auto predRemoveDroppedTable = [self](const TWritePortionInfoWithBlobsResult& item) { @@ -45,7 +46,8 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, portionCopy.MutableMeta().ResetCompactionLevel(TargetCompactionLevel.value_or(0)); }; context.EngineLogs.GetGranuleVerified(i->GetPathId()) - .ModifyPortionOnExecute(context.DBWrapper, i, pred, schemaPtr->GetIndexInfo().GetPKFirstColumnId()); + .ModifyPortionOnExecute( + context.DBWrapper, GetPortionDataAccessor(i->GetPortionId()), pred, schemaPtr->GetIndexInfo().GetPKFirstColumnId()); } } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.h b/ydb/core/tx/columnshard/engines/changes/with_appended.h index 7b60fccec855..4a9ac3d1bc04 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.h +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.h @@ -15,6 +15,10 @@ class TChangesWithAppend: public TColumnEngineChanges { protected: std::optional TargetCompactionLevel; TSaverContext SaverContext; + virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& /*context*/) override { + + } + virtual void DoCompile(TFinalizationContext& context) override; virtual void DoOnAfterCompile() override; virtual void DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) override; @@ -56,6 +60,7 @@ class TChangesWithAppend: public TColumnEngineChanges { for (auto&& i : portions) { AFL_VERIFY(i); AFL_VERIFY(PortionsToMove.emplace(i->GetAddress(), i).second)("portion_id", i->GetPortionId()); + PortionsToAccess->AddPortion(i); } } @@ -75,9 +80,12 @@ class TChangesWithAppend: public TColumnEngineChanges { TargetCompactionLevel = level; } - void AddPortionToRemove(const TPortionInfo::TConstPtr& info) { + void AddPortionToRemove(const TPortionInfo::TConstPtr& info, const bool addIntoDataAccessRequest = true) { AFL_VERIFY(!info->HasRemoveSnapshot()); AFL_VERIFY(PortionsToRemove.emplace(info->GetAddress(), info).second); + if (addIntoDataAccessRequest) { + PortionsToAccess->AddPortion(info); + } } std::vector AppendedPortions; diff --git a/ydb/core/tx/columnshard/engines/column_engine.cpp b/ydb/core/tx/columnshard/engines/column_engine.cpp index a211abcdca3c..51c2bc6cf2f2 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine.cpp @@ -1,9 +1,12 @@ #include "column_engine.h" + #include "portions/portion_info.h" -#include -#include +#include #include +#include + +#include namespace NKikimr::NOlap { @@ -17,14 +20,22 @@ ui64 IColumnEngine::GetMetadataLimit() { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("total", MemoryTotal); return MemoryTotal * 0.3; } else if (AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().HasAbsoluteValue()) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("value", AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetAbsoluteValue()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)( + "value", AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetAbsoluteValue()); return AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetAbsoluteValue(); } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("total", MemoryTotal)("kff", AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetTotalRatio()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("total", MemoryTotal)( + "kff", AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetTotalRatio()); return MemoryTotal * AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetTotalRatio(); } } +void IColumnEngine::FetchDataAccessors(const std::shared_ptr& request) const { + AFL_VERIFY(!!request); + AFL_VERIFY(!request->IsEmpty()); + DoFetchDataAccessors(request); +} + TSelectInfo::TStats TSelectInfo::Stats() const { TStats out; out.Portions = PortionsOrderedPK.size(); @@ -49,4 +60,4 @@ void TSelectInfo::DebugStream(IOutputStream& out) { } } -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 1c9ca5bd8936..20a423687df1 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -9,6 +9,7 @@ #include #include +#include namespace NKikimr::NColumnShard { class TTiersManager; @@ -17,6 +18,7 @@ class TTtl; namespace NKikimr::NOlap { class TInsertColumnEngineChanges; +class TDataAccessorsRequest; class TCompactColumnEngineChanges; class TColumnEngineChanges; class TTTLColumnEngineChanges; @@ -73,7 +75,8 @@ class TColumnEngineStats { i64 RawBytes = 0; TString DebugString() const { - return TStringBuilder() << "portions=" << Portions << ";blobs=" << Blobs << ";rows=" << Rows << ";bytes=" << Bytes << ";raw_bytes=" << RawBytes << ";"; + return TStringBuilder() << "portions=" << Portions << ";blobs=" << Blobs << ";rows=" << Rows << ";bytes=" << Bytes + << ";raw_bytes=" << RawBytes << ";"; } TPortionsStats operator+(const TPortionsStats& item) const { @@ -238,6 +241,7 @@ class TColumnEngineStats { class IColumnEngine { protected: virtual void DoRegisterTable(const ui64 pathId) = 0; + virtual void DoFetchDataAccessors(const std::shared_ptr& request) const = 0; public: class TSchemaInitializationData { @@ -276,6 +280,8 @@ class IColumnEngine { } }; + void FetchDataAccessors(const std::shared_ptr& request) const; + static ui64 GetMetadataLimit(); virtual ~IColumnEngine() = default; @@ -286,7 +292,7 @@ class IColumnEngine { virtual bool HasDataInPathId(const ui64 pathId) const = 0; virtual bool ErasePathId(const ui64 pathId) = 0; - virtual bool Load(IDbWrapper& db) = 0; + virtual std::shared_ptr BuildLoader(const std::shared_ptr& dsGroupSelector) = 0; void RegisterTable(const ui64 pathId) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "RegisterTable")("path_id", pathId); return DoRegisterTable(pathId); @@ -296,12 +302,13 @@ class IColumnEngine { ui64 pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter, const bool withUncommitted) const = 0; virtual std::shared_ptr StartInsert(std::vector&& dataToIndex) noexcept = 0; virtual std::shared_ptr StartCompaction(const std::shared_ptr& dataLocksManager) noexcept = 0; - virtual ui64 GetCompactionPriority( - const std::shared_ptr& dataLocksManager, const std::set& pathIds, const std::optional waitingPriority) noexcept = 0; + virtual ui64 GetCompactionPriority(const std::shared_ptr& dataLocksManager, const std::set& pathIds, + const std::optional waitingPriority) noexcept = 0; virtual std::shared_ptr StartCleanupPortions(const TSnapshot& snapshot, - const THashSet& pathsToDrop, const std::shared_ptr& dataLocksManager) noexcept = 0; + const THashSet& pathsToDrop, const std::shared_ptr& dataLocksManager) noexcept = 0; virtual std::shared_ptr StartCleanupTables(const THashSet& pathsToDrop) noexcept = 0; - virtual std::vector> StartTtl(const THashMap& pathEviction, const std::shared_ptr& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0; + virtual std::vector> StartTtl(const THashMap& pathEviction, + const std::shared_ptr& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0; virtual bool ApplyChangesOnTxCreate(std::shared_ptr changes, const TSnapshot& snapshot) noexcept = 0; virtual bool ApplyChangesOnExecute(IDbWrapper& db, std::shared_ptr changes, const TSnapshot& snapshot) noexcept = 0; virtual void RegisterSchemaVersion(const TSnapshot& snapshot, TIndexInfo&& info) = 0; @@ -316,7 +323,8 @@ class IColumnEngine { virtual TSnapshot LastUpdate() const { return TSnapshot::Zero(); } - virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; + virtual void OnTieringModified( + const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index facf3d88eb03..300c35a7871e 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -7,6 +7,7 @@ #include "changes/general_compaction.h" #include "changes/indexation.h" #include "changes/ttl.h" +#include "loading/stages.h" #include "portions/constructor.h" #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include @@ -26,9 +28,10 @@ namespace NKikimr::NOlap { -TColumnEngineForLogs::TColumnEngineForLogs( - ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const TSchemaInitializationData& schema) - : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) +TColumnEngineForLogs::TColumnEngineForLogs(const ui64 tabletId, + const std::shared_ptr& dataAccessorsManager, + const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const TSchemaInitializationData& schema) + : GranulesStorage(std::make_shared(SignalCounters, dataAccessorsManager, storagesManager)) , StoragesManager(storagesManager) , TabletId(tabletId) , LastPortion(0) @@ -37,9 +40,10 @@ TColumnEngineForLogs::TColumnEngineForLogs( RegisterSchemaVersion(snapshot, schema); } -TColumnEngineForLogs::TColumnEngineForLogs( - ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema) - : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) +TColumnEngineForLogs::TColumnEngineForLogs(const ui64 tabletId, + const std::shared_ptr& dataAccessorsManager, + const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema) + : GranulesStorage(std::make_shared(SignalCounters, dataAccessorsManager, storagesManager)) , StoragesManager(storagesManager) , TabletId(tabletId) , LastPortion(0) @@ -174,7 +178,8 @@ void TColumnEngineForLogs::RegisterOldSchemaVersion(const TSnapshot& snapshot, c return; } - ISnapshotSchema::TPtr secondLast = VersionedIndex.GetLastSchemaBeforeOrEqualSnapshotOptional(VersionedIndex.GetLastSchema()->GetVersion() - 1); + ISnapshotSchema::TPtr secondLast = + VersionedIndex.GetLastSchemaBeforeOrEqualSnapshotOptional(VersionedIndex.GetLastSchema()->GetVersion() - 1); AFL_VERIFY(!secondLast || secondLast->GetVersion() <= version)("reason", "incorrect schema registration order"); @@ -182,8 +187,8 @@ void TColumnEngineForLogs::RegisterOldSchemaVersion(const TSnapshot& snapshot, c if (schema.GetDiff()) { AFL_VERIFY(prevSchema)("reason", "no base schema to apply diff for"); - indexInfoOptional = NOlap::TIndexInfo::BuildFromProto( - *schema.GetDiff(), prevSchema->GetIndexInfo(), StoragesManager, SchemaObjectsCache); + indexInfoOptional = + NOlap::TIndexInfo::BuildFromProto(*schema.GetDiff(), prevSchema->GetIndexInfo(), StoragesManager, SchemaObjectsCache); } else { indexInfoOptional = NOlap::TIndexInfo::BuildFromProto(schema.GetSchemaVerified(), StoragesManager, SchemaObjectsCache); } @@ -192,25 +197,26 @@ void TColumnEngineForLogs::RegisterOldSchemaVersion(const TSnapshot& snapshot, c VersionedIndex.AddIndex(snapshot, std::move(*indexInfoOptional)); } -bool TColumnEngineForLogs::Load(IDbWrapper& db) { - Y_ABORT_UNLESS(!Loaded); - Loaded = true; - THashMap granuleToPathIdDecoder; - { - TMemoryProfileGuard g("TTxInit/LoadShardingInfo"); - if (!VersionedIndex.LoadShardingInfo(db)) { - return false; +std::shared_ptr TColumnEngineForLogs::BuildLoader(const std::shared_ptr& dsGroupSelector) { + auto result = std::make_shared("column_engines"); + result->AddChildren(std::make_shared("counters", this, dsGroupSelector)); + result->AddChildren(std::make_shared("sharding_info", this, dsGroupSelector)); + if (GranulesStorage->GetTables().size()) { + auto granules = std::make_shared("granules"); + for (auto&& i : GranulesStorage->GetTables()) { + granules->AddChildren(i.second->BuildLoader(dsGroupSelector, VersionedIndex)); } + result->AddChildren(granules); } + result->AddChildren(std::make_shared("finish", this)); + return result; +} +bool TColumnEngineForLogs::FinishLoading() { { - auto guard = GranulesStorage->GetStats()->StartPackModification(); - if (!LoadColumns(db)) { - return false; - } - TMemoryProfileGuard g("TTxInit/LoadCounters"); - if (!LoadCounters(db)) { - return false; + TMemoryProfileGuard g("TTxInit/LoadColumns/After"); + for (auto&& i : GranulesStorage->GetTables()) { + i.second->OnAfterPortionsLoad(); } } @@ -228,85 +234,6 @@ bool TColumnEngineForLogs::Load(IDbWrapper& db) { return true; } -bool TColumnEngineForLogs::LoadColumns(IDbWrapper& db) { - TPortionConstructors constructors; - { - NColumnShard::TLoadTimeSignals::TLoadTimer timer = SignalCounters.PortionsLoadingTimeCounters.StartGuard(); - TMemoryProfileGuard g("TTxInit/LoadColumns/Portions"); - if (!db.LoadPortions([&](TPortionInfoConstructor&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { - const TIndexInfo& indexInfo = portion.GetSchema(VersionedIndex)->GetIndexInfo(); - AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo, db.GetDsGroupSelectorVerified())); - AFL_VERIFY(constructors.AddConstructorVerified(std::move(portion))); - })) { - timer.AddLoadingFail(); - return false; - } - } - - { - NColumnShard::TLoadTimeSignals::TLoadTimer timer = SignalCounters.ColumnsLoadingTimeCounters.StartGuard(); - TMemoryProfileGuard g("TTxInit/LoadColumns/Records"); - TPortionInfo::TSchemaCursor schema(VersionedIndex); - if (!db.LoadColumns([&](const TColumnChunkLoadContextV1& loadContext) { - auto* constructor = constructors.GetConstructorVerified(loadContext.GetPathId(), loadContext.GetPortionId()); - constructor->LoadRecord(loadContext); - })) { - timer.AddLoadingFail(); - return false; - } - } - - { - NColumnShard::TLoadTimeSignals::TLoadTimeSignals::TLoadTimer timer = SignalCounters.IndexesLoadingTimeCounters.StartGuard(); - TMemoryProfileGuard g("TTxInit/LoadIndexes/Indexes"); - if (!db.LoadIndexes([&](const ui64 pathId, const ui64 portionId, const TIndexChunkLoadContext& loadContext) { - auto* constructor = constructors.GetConstructorVerified(pathId, portionId); - constructor->LoadIndex(loadContext); - })) { - timer.AddLoadingFail(); - return false; - }; - } - - { - TMemoryProfileGuard g("TTxInit/LoadColumns/Constructors"); - for (auto&& [granuleId, pathConstructors] : constructors) { - auto g = GetGranulePtrVerified(granuleId); - for (auto&& [portionId, constructor] : pathConstructors) { - g->UpsertPortionOnLoad(constructor.Build(false).MutablePortionInfoPtr()); - } - } - } - { - TMemoryProfileGuard g("TTxInit/LoadColumns/After"); - for (auto&& i : GranulesStorage->GetTables()) { - i.second->OnAfterPortionsLoad(); - } - } - return true; -} - -bool TColumnEngineForLogs::LoadCounters(IDbWrapper& db) { - auto callback = [&](ui32 id, ui64 value) { - switch (id) { - case LAST_PORTION: - LastPortion = value; - break; - case LAST_GRANULE: - LastGranule = value; - break; - case LAST_PLAN_STEP: - LastSnapshot = TSnapshot(value, LastSnapshot.GetTxId()); - break; - case LAST_TX_ID: - LastSnapshot = TSnapshot(LastSnapshot.GetPlanStep(), value); - break; - } - }; - - return db.LoadCounters(callback); -} - std::shared_ptr TColumnEngineForLogs::StartInsert(std::vector&& dataToIndex) noexcept { Y_ABORT_UNLESS(dataToIndex.size()); @@ -382,7 +309,7 @@ std::shared_ptr TColumnEngineForLogs::Start const TSnapshot& snapshot, const THashSet& pathsToDrop, const std::shared_ptr& dataLocksManager) noexcept { AFL_VERIFY(dataLocksManager); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartCleanup")("portions_count", CleanupPortions.size()); - auto changes = std::make_shared(StoragesManager); + std::shared_ptr changes = std::make_shared(StoragesManager); // Add all portions from dropped paths ui64 txSize = 0; @@ -404,13 +331,13 @@ std::shared_ptr TColumnEngineForLogs::Start ++skipLocked; continue; } - if (txSize + info->GetTxVolume() < txSizeLimit || changes->PortionsToDrop.empty()) { + if (txSize + info->GetTxVolume() < txSizeLimit || changes->GetPortionsToDrop().empty()) { txSize += info->GetTxVolume(); } else { limitExceeded = true; break; } - changes->PortionsToDrop.push_back(TPortionDataAccessor(info)); + changes->AddPortionToDrop(info); ++portionsFromDrop; } } @@ -429,13 +356,13 @@ std::shared_ptr TColumnEngineForLogs::Start continue; } AFL_VERIFY(it->second[i]->CheckForCleanup(snapshot))("p_snapshot", it->second[i]->GetRemoveSnapshotOptional())("snapshot", snapshot); - if (txSize + it->second[i]->GetTxVolume() < txSizeLimit || changes->PortionsToDrop.empty()) { + if (txSize + it->second[i]->GetTxVolume() < txSizeLimit || changes->GetPortionsToDrop().empty()) { txSize += it->second[i]->GetTxVolume(); } else { limitExceeded = true; break; } - changes->PortionsToDrop.push_back(TPortionDataAccessor(it->second[i])); + changes->AddPortionToDrop(it->second[i]); if (i + 1 < it->second.size()) { it->second[i] = std::move(it->second.back()); } @@ -451,9 +378,9 @@ std::shared_ptr TColumnEngineForLogs::Start } } AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartCleanup")("portions_count", CleanupPortions.size())( - "portions_prepared", changes->PortionsToDrop.size())("drop", portionsFromDrop)("skip", skipLocked); + "portions_prepared", changes->GetPortionsToDrop().size())("drop", portionsFromDrop)("skip", skipLocked); - if (changes->PortionsToDrop.empty()) { + if (changes->GetPortionsToDrop().empty()) { return nullptr; } @@ -525,6 +452,9 @@ void TColumnEngineForLogs::AppendPortion(const TPortionInfo::TPtr& portionInfo) AFL_VERIFY(!granule->GetPortionOptional(portionInfo->GetPortionId())); UpdatePortionStats(*portionInfo, EStatsUpdateType::ADD); granule->AppendPortion(portionInfo); + if (portionInfo->HasRemoveSnapshot()) { + AddCleanupPortion(portionInfo); + } } bool TColumnEngineForLogs::ErasePortion(const TPortionInfo& portionInfo, bool updateStats) { @@ -626,4 +556,46 @@ void TColumnEngineForLogs::DoRegisterTable(const ui64 pathId) { } } +bool TColumnEngineForLogs::TestingLoad(IDbWrapper& db) { + { + TMemoryProfileGuard g("TTxInit/LoadShardingInfo"); + if (!VersionedIndex.LoadShardingInfo(db)) { + return false; + } + } + + { + auto guard = GranulesStorage->GetStats()->StartPackModification(); + for (auto&& [_, i] : GranulesStorage->GetTables()) { + i->TestingLoad(db, VersionedIndex); + } + if (!LoadCounters(db)) { + return false; + } + FinishLoading(); + } + return true; +} + +bool TColumnEngineForLogs::LoadCounters(IDbWrapper& db) { + auto callback = [&](ui32 id, ui64 value) { + switch (id) { + case LAST_PORTION: + LastPortion = value; + break; + case LAST_GRANULE: + LastGranule = value; + break; + case LAST_PLAN_STEP: + LastSnapshot = TSnapshot(value, LastSnapshot.GetTxId()); + break; + case LAST_TX_ID: + LastSnapshot = TSnapshot(LastSnapshot.GetPlanStep(), value); + break; + } + }; + + return db.LoadCounters(callback); +} + } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 8274dea90855..8dcc14e0ab82 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace NKikimr::NArrow { struct TSortDescription; @@ -30,6 +31,10 @@ class TCleanupTablesColumnEngineChanges; namespace NDataSharing { class TDestinationSession; } +namespace NEngineLoading { +class TEngineShardingInfoReader; +class TEngineCountersReader; +} struct TReadMetadata; @@ -46,6 +51,8 @@ class TColumnEngineForLogs: public IColumnEngine { friend class TCleanupPortionsColumnEngineChanges; friend class TCleanupTablesColumnEngineChanges; friend class NDataSharing::TDestinationSession; + friend class NEngineLoading::TEngineShardingInfoReader; + friend class NEngineLoading::TEngineCountersReader; private: bool ActualizationStarted = false; @@ -82,10 +89,10 @@ class TColumnEngineForLogs: public IColumnEngine { ADD, }; - TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, - const TSchemaInitializationData& schema); - TColumnEngineForLogs( - ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema); + TColumnEngineForLogs(const ui64 tabletId, const std::shared_ptr& dataAccessorsManager, + const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const TSchemaInitializationData& schema); + TColumnEngineForLogs(const ui64 tabletId, const std::shared_ptr& dataAccessorsManager, + const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema); virtual void OnTieringModified( const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) override; @@ -105,9 +112,16 @@ class TColumnEngineForLogs: public IColumnEngine { } virtual void DoRegisterTable(const ui64 pathId) override; + void DoFetchDataAccessors(const std::shared_ptr& request) const override { + GranulesStorage->FetchDataAccessors(request); + } + + bool TestingLoadColumns(IDbWrapper& db); + bool LoadCounters(IDbWrapper& db); public: - bool Load(IDbWrapper& db) override; + virtual std::shared_ptr BuildLoader(const std::shared_ptr& dsGroupSelector) override; + bool FinishLoading(); virtual bool IsOverloadedByMetadata(const ui64 limit) const override { return limit < TGranulesStat::GetSumMetadataMemoryPortionsSize(); @@ -191,6 +205,8 @@ class TColumnEngineForLogs: public IColumnEngine { VersionedIndex.AddShardingInfo(shardingInfo); } + bool TestingLoad(IDbWrapper& db); + template void ModifyPortionOnComplete(const TPortionInfo::TConstPtr& portion, const TModifier& modifier) { auto exPortion = portion->MakeCopy(); @@ -214,10 +230,6 @@ class TColumnEngineForLogs: public IColumnEngine { bool Loaded = false; private: - bool LoadColumns(IDbWrapper& db); - bool LoadShardingInfo(IDbWrapper& db); - bool LoadCounters(IDbWrapper& db); - bool ErasePortion(const TPortionInfo& portionInfo, bool updateStats = true); void UpdatePortionStats( const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT, const TPortionInfo* exPortionInfo = nullptr); diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 684fe1e75b87..38c718e8dfa7 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -118,66 +118,84 @@ void TDbWrapper::EraseColumn(const NOlap::TPortionInfo& portion, const TColumnRe } } -bool TDbWrapper::LoadColumns(const std::function& callback) { +bool TDbWrapper::LoadColumns(const std::optional pathId, const std::function& callback) { NIceDb::TNiceDb db(Database); using IndexColumnsV1 = NColumnShard::Schema::IndexColumnsV1; - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } + const auto pred = [&](auto& rowset) { + if (!rowset.IsReady()) { + return false; + } - while (!rowset.EndOfSet()) { - NOlap::TColumnChunkLoadContextV1 chunkLoadContext(rowset); - callback(chunkLoadContext); + while (!rowset.EndOfSet()) { + NOlap::TColumnChunkLoadContextV1 chunkLoadContext(rowset); + callback(chunkLoadContext); - if (!rowset.Next()) { - return false; + if (!rowset.Next()) { + return false; + } } + return true; + }; + if (pathId) { + auto rowset = db.Table().Prefix(*pathId).Select(); + return pred(rowset); + } else { + auto rowset = db.Table().Select(); + return pred(rowset); } - return true; } -bool TDbWrapper::LoadPortions( +bool TDbWrapper::LoadPortions(const std::optional pathId, const std::function& callback) { NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - NOlap::TPortionInfoConstructor portion(rowset.GetValue(), rowset.GetValue()); - portion.SetSchemaVersion(rowset.GetValue()); - if (rowset.HaveValue() && rowset.GetValue()) { - portion.SetShardingVersion(rowset.GetValue()); - } - portion.SetRemoveSnapshot(rowset.GetValue(), rowset.GetValue()); - if (rowset.GetValue()) { - portion.SetMinSnapshotDeprecated( - TSnapshot(rowset.GetValue(), rowset.GetValue())); - } - - if (rowset.GetValueOrDefault(0)) { - portion.SetInsertWriteId((TInsertWriteId)rowset.GetValue()); - } - if (rowset.GetValueOrDefault(0)) { - AFL_VERIFY(rowset.GetValueOrDefault(0)); - portion.SetCommitSnapshot(TSnapshot(rowset.GetValue(), rowset.GetValue())); - } else { - AFL_VERIFY(!rowset.GetValueOrDefault(0)); + const auto pred = [&](auto& rowset) { + if (!rowset.IsReady()) { + return false; } - NKikimrTxColumnShard::TIndexPortionMeta metaProto; - const TString metadata = rowset.template GetValue(); - AFL_VERIFY(metaProto.ParseFromArray(metadata.data(), metadata.size()))("event", "cannot parse metadata as protobuf"); - callback(std::move(portion), metaProto); - - if (!rowset.Next()) { - return false; + while (!rowset.EndOfSet()) { + NOlap::TPortionInfoConstructor portion( + rowset.template GetValue(), rowset.template GetValue()); + portion.SetSchemaVersion(rowset.template GetValue()); + if (rowset.template HaveValue() && rowset.template GetValue()) { + portion.SetShardingVersion(rowset.template GetValue()); + } + portion.SetRemoveSnapshot(rowset.template GetValue(), rowset.template GetValue()); + if (rowset.template GetValue()) { + portion.SetMinSnapshotDeprecated(TSnapshot( + rowset.template GetValue(), rowset.template GetValue())); + } + + if (rowset.template GetValueOrDefault(0)) { + portion.SetInsertWriteId((TInsertWriteId)rowset.template GetValue()); + } + if (rowset.template GetValueOrDefault(0)) { + AFL_VERIFY(rowset.template GetValueOrDefault(0)); + portion.SetCommitSnapshot( + TSnapshot(rowset.template GetValue(), rowset.template GetValue())); + } else { + AFL_VERIFY(!rowset.template GetValueOrDefault(0)); + } + + NKikimrTxColumnShard::TIndexPortionMeta metaProto; + const TString metadata = rowset.template GetValue(); + AFL_VERIFY(metaProto.ParseFromArray(metadata.data(), metadata.size()))("event", "cannot parse metadata as protobuf"); + callback(std::move(portion), metaProto); + + if (!rowset.Next()) { + return false; + } } + return true; + }; + if (pathId) { + auto rowset = db.Table().Prefix(*pathId).Select(); + return pred(rowset); + } else { + auto rowset = db.Table().Select(); + return pred(rowset); } - return true; } void TDbWrapper::WriteIndex(const TPortionInfo& portion, const TIndexChunk& row) { @@ -206,23 +224,32 @@ void TDbWrapper::EraseIndex(const TPortionInfo& portion, const TIndexChunk& row) db.Table().Key(portion.GetPathId(), portion.GetPortionId(), row.GetIndexId(), 0).Delete(); } -bool TDbWrapper::LoadIndexes(const std::function& callback) { +bool TDbWrapper::LoadIndexes(const std::optional pathId, + const std::function& callback) { NIceDb::TNiceDb db(Database); using IndexIndexes = NColumnShard::Schema::IndexIndexes; - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } + const auto pred = [&](auto& rowset) { + if (!rowset.IsReady()) { + return false; + } - while (!rowset.EndOfSet()) { - NOlap::TIndexChunkLoadContext chunkLoadContext(rowset, DsGroupSelector); - callback(rowset.GetValue(), rowset.GetValue(), chunkLoadContext); + while (!rowset.EndOfSet()) { + NOlap::TIndexChunkLoadContext chunkLoadContext(rowset, DsGroupSelector); + callback(rowset.template GetValue(), rowset.template GetValue(), chunkLoadContext); - if (!rowset.Next()) { - return false; + if (!rowset.Next()) { + return false; + } } + return true; + }; + if (pathId) { + auto rowset = db.Table().Prefix(*pathId).Select(); + return pred(rowset); + } else { + auto rowset = db.Table().Select(); + return pred(rowset); } - return true; } void TDbWrapper::WriteCounter(ui32 counterId, ui64 value) { diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index 507fb1629c10..62566d7aead0 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -48,15 +48,17 @@ class IDbWrapper { virtual void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) = 0; virtual void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) = 0; - virtual bool LoadColumns(const std::function& callback) = 0; + virtual bool LoadColumns(const std::optional pathId, const std::function& callback) = 0; virtual void WritePortion(const NOlap::TPortionInfo& portion) = 0; virtual void ErasePortion(const NOlap::TPortionInfo& portion) = 0; - virtual bool LoadPortions(const std::function& callback) = 0; + virtual bool LoadPortions(const std::optional pathId, + const std::function& callback) = 0; virtual void WriteIndex(const TPortionInfo& portion, const TIndexChunk& row) = 0; virtual void EraseIndex(const TPortionInfo& portion, const TIndexChunk& row) = 0; - virtual bool LoadIndexes(const std::function& callback) = 0; + virtual bool LoadIndexes(const std::optional pathId, + const std::function& callback) = 0; virtual void WriteCounter(ui32 counterId, ui64 value) = 0; virtual bool LoadCounters(const std::function& callback) = 0; @@ -81,15 +83,16 @@ class TDbWrapper : public IDbWrapper { void WritePortion(const NOlap::TPortionInfo& portion) override; void ErasePortion(const NOlap::TPortionInfo& portion) override; - bool LoadPortions(const std::function& callback) override; + bool LoadPortions(const std::optional pathId, const std::function& callback) override; void WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) override; void EraseColumn(const NOlap::TPortionInfo& portion, const TColumnRecord& row) override; - bool LoadColumns(const std::function& callback) override; + bool LoadColumns(const std::optional pathId, const std::function& callback) override; virtual void WriteIndex(const TPortionInfo& portion, const TIndexChunk& row) override; virtual void EraseIndex(const TPortionInfo& portion, const TIndexChunk& row) override; - virtual bool LoadIndexes(const std::function& callback) override; + virtual bool LoadIndexes(const std::optional pathId, + const std::function& callback) override; void WriteCounter(ui32 counterId, ui64 value) override; bool LoadCounters(const std::function& callback) override; diff --git a/ydb/core/tx/columnshard/engines/loading/stages.cpp b/ydb/core/tx/columnshard/engines/loading/stages.cpp new file mode 100644 index 000000000000..5a12216f67ed --- /dev/null +++ b/ydb/core/tx/columnshard/engines/loading/stages.cpp @@ -0,0 +1,35 @@ +#include "stages.h" + +#include +#include +#include +#include + +namespace NKikimr::NOlap::NEngineLoading { + +bool TEngineShardingInfoReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + TDbWrapper db(txc.DB, &*DsGroupSelector); + return Self->VersionedIndex.LoadShardingInfo(db); +} + +bool TEngineShardingInfoReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return NColumnShard::Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TEngineLoadingFinish::DoExecute(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) { + Self->FinishLoading(); + return true; +} + +bool TEngineCountersReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + TDbWrapper db(txc.DB, &*DsGroupSelector); + return Self->LoadCounters(db); +} + +bool TEngineCountersReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return NColumnShard::Schema::Precharge(db, txc.DB.GetScheme()); +} + +} // namespace NKikimr::NOlap::NEngineLoading diff --git a/ydb/core/tx/columnshard/engines/loading/stages.h b/ydb/core/tx/columnshard/engines/loading/stages.h new file mode 100644 index 000000000000..fb23e5a693eb --- /dev/null +++ b/ydb/core/tx/columnshard/engines/loading/stages.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include + +namespace NKikimr::NOlap { +class TColumnEngineForLogs; +} + +namespace NKikimr::NOlap::NEngineLoading { + +class IEngineTxReader: public ITxReader { +private: + using TBase = ITxReader; + +protected: + const std::shared_ptr DsGroupSelector; + TColumnEngineForLogs* Self = nullptr; + +public: + IEngineTxReader(const TString& name, TColumnEngineForLogs* self, const std::shared_ptr& dsGroupSelector) + : TBase(name) + , DsGroupSelector(dsGroupSelector) + , Self(self) { + } +}; + +class TEngineCountersReader: public IEngineTxReader { +private: + using TBase = IEngineTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TEngineShardingInfoReader: public IEngineTxReader { +private: + using TBase = IEngineTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TEngineLoadingFinish: public ITxReader { +private: + using TBase = ITxReader; + TColumnEngineForLogs* Self = nullptr; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + +public: + TEngineLoadingFinish(const TString& name, TColumnEngineForLogs* self) + : TBase(name) + , Self(self) { + } +}; + +} // namespace NKikimr::NOlap::NEngineLoading diff --git a/ydb/core/tx/columnshard/engines/loading/ya.make b/ydb/core/tx/columnshard/engines/loading/ya.make new file mode 100644 index 000000000000..c50c4958e9d0 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/loading/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + stages.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/common + ydb/core/tx/columnshard/tx_reader +) + +END() diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.h b/ydb/core/tx/columnshard/engines/portions/constructor.h index 6534003c319a..fe3adb76011c 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor.h @@ -40,6 +40,14 @@ class TPortionInfoConstructor { TPortionInfoConstructor(TPortionInfoConstructor&&) noexcept = default; TPortionInfoConstructor& operator=(TPortionInfoConstructor&&) noexcept = default; + void ClearRecords() { + Records.clear(); + } + + void ClearIndexes() { + Indexes.clear(); + } + class TTestCopier { public: static TPortionInfoConstructor Copy(const TPortionInfoConstructor& source) { @@ -380,4 +388,47 @@ class TPortionConstructors { } }; +class TInGranuleConstructors { +private: + THashMap Constructors; + +public: + THashMap::iterator begin() { + return Constructors.begin(); + } + + THashMap::iterator end() { + return Constructors.end(); + } + + void ClearPortions() { + Constructors.clear(); + } + + void ClearColumns() { + for (auto&& i : Constructors) { + i.second.ClearRecords(); + } + } + + void ClearIndexes() { + for (auto&& i : Constructors) { + i.second.ClearIndexes(); + } + } + + TPortionInfoConstructor* GetConstructorVerified(const ui64 portionId) { + auto itPortionId = Constructors.find(portionId); + AFL_VERIFY(itPortionId != Constructors.end()); + return &itPortionId->second; + } + + TPortionInfoConstructor* AddConstructorVerified(TPortionInfoConstructor&& constructor) { + const ui64 portionId = constructor.GetPortionIdVerified(); + auto info = Constructors.emplace(portionId, std::move(constructor)); + AFL_VERIFY(info.second); + return &info.first->second; + } +}; + } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.h b/ydb/core/tx/columnshard/engines/portions/data_accessor.h index 836c6cc34a7f..48c1fcb9a6d9 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.h @@ -37,6 +37,10 @@ class TPortionDataAccessor { void FullValidation() const; public: + TPortionDataAccessor SwitchPortionInfo(TPortionInfo&& newPortion) const { + return TPortionDataAccessor(std::make_shared(std::move(newPortion))); + } + template static void AggregateIndexChunksData( const TAggregator& aggr, const std::vector& chunks, const std::set* columnIds, const bool validation) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 021df9458265..ae065401dd5f 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -103,6 +103,10 @@ class TPortionInfo { TPortionInfo(TPortionInfo&&) = default; TPortionInfo& operator=(TPortionInfo&&) = default; + ui32 PredictMetadataMemorySize(const ui32 columnsCount) const { + return (GetRecordsCount() / 10000 + 1) * sizeof(TColumnRecord) * columnsCount; + } + void SaveMetaToDatabase(IDbWrapper& db) const; TPortionInfo MakeCopy() const { diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h index 3b1d545094ac..e3ec8567b862 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,7 @@ class TComputeShardingPolicy { class TReadContext { private: YDB_READONLY_DEF(std::shared_ptr, StoragesManager); + YDB_READONLY_DEF(std::shared_ptr, DataAccessorsManager); const NColumnShard::TConcreteScanCounters Counters; TReadMetadataBase::TConstPtr ReadMetadata; NResourceBroker::NSubscribe::TTaskContext ResourcesTaskContext; @@ -99,10 +101,13 @@ class TReadContext { return ResourcesTaskContext; } - TReadContext(const std::shared_ptr& storagesManager, const NColumnShard::TConcreteScanCounters& counters, - const TReadMetadataBase::TConstPtr& readMetadata, const TActorId& scanActorId, const TActorId& resourceSubscribeActorId, - const TActorId& readCoordinatorActorId, const TComputeShardingPolicy& computeShardingPolicy, const ui64 scanId) + TReadContext(const std::shared_ptr& storagesManager, + const std::shared_ptr& dataAccessorsManager, + const NColumnShard::TConcreteScanCounters& counters, const TReadMetadataBase::TConstPtr& readMetadata, const TActorId& scanActorId, + const TActorId& resourceSubscribeActorId, const TActorId& readCoordinatorActorId, const TComputeShardingPolicy& computeShardingPolicy, + const ui64 scanId) : StoragesManager(storagesManager) + , DataAccessorsManager(dataAccessorsManager) , Counters(counters) , ReadMetadata(readMetadata) , ResourcesTaskContext("CS::SCAN_READ", counters.ResourcesSubscriberCounters) diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h index ac608b5ad670..ed6667e61bb1 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h @@ -107,6 +107,7 @@ struct TReadMetadataBase { } ISnapshotSchema::TPtr GetResultSchema() const { + AFL_VERIFY(ResultIndexSchema); return ResultIndexSchema; } @@ -121,6 +122,7 @@ struct TReadMetadataBase { } const TIndexInfo& GetIndexInfo(const std::optional& version = {}) const { + AFL_VERIFY(ResultIndexSchema); if (version && version < RequestSnapshot) { return GetIndexVersions().GetSchema(*version)->GetIndexInfo(); } @@ -150,6 +152,7 @@ struct TReadMetadataBase { } std::set GetProcessingColumnIds() const { + AFL_VERIFY(ResultIndexSchema); std::set result; for (auto&& i : GetProgram().GetProcessingColumns()) { result.emplace(ResultIndexSchema->GetIndexInfo().GetColumnIdVerified(i)); @@ -184,6 +187,7 @@ struct TReadMetadataBase { } std::shared_ptr GetReplaceKey() const { + AFL_VERIFY(ResultIndexSchema); return ResultIndexSchema->GetIndexInfo().GetReplaceKey(); } diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index 907eee97ca25..aa9f206b7ae7 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -1,8 +1,10 @@ #include "actor.h" + +#include #include #include + #include -#include namespace NKikimr::NOlap::NReader { constexpr i64 DEFAULT_READ_AHEAD_BYTES = (i64)2 * 1024 * 1024 * 1024; @@ -14,6 +16,7 @@ class TInFlightGuard: NNonCopyable::TNonCopyable { private: static inline TAtomicCounter InFlightGlobal = 0; i64 InFlightGuarded = 0; + public: ~TInFlightGuard() { Return(InFlightGuarded); @@ -35,7 +38,7 @@ class TInFlightGuard: NNonCopyable::TNonCopyable { } }; -} +} // namespace void TColumnShardScan::PassAway() { Send(ResourceSubscribeActorId, new TEvents::TEvPoisonPill); @@ -43,11 +46,14 @@ void TColumnShardScan::PassAway() { IActor::PassAway(); } -TColumnShardScan::TColumnShardScan(const TActorId& columnShardActorId, const TActorId& scanComputeActorId, const std::shared_ptr& storagesManager, - const TComputeShardingPolicy& computeShardingPolicy, ui32 scanId, ui64 txId, ui32 scanGen, ui64 requestCookie, - ui64 tabletId, TDuration timeout, const TReadMetadataBase::TConstPtr& readMetadataRange, - NKikimrDataEvents::EDataFormat dataFormat, const NColumnShard::TScanCounters& scanCountersPool) +TColumnShardScan::TColumnShardScan(const TActorId& columnShardActorId, const TActorId& scanComputeActorId, + const std::shared_ptr& storagesManager, + const std::shared_ptr& dataAccessorsManager, + const TComputeShardingPolicy& computeShardingPolicy, ui32 scanId, ui64 txId, ui32 scanGen, ui64 requestCookie, ui64 tabletId, + TDuration timeout, const TReadMetadataBase::TConstPtr& readMetadataRange, NKikimrDataEvents::EDataFormat dataFormat, + const NColumnShard::TScanCounters& scanCountersPool) : StoragesManager(storagesManager) + , DataAccessorsManager(dataAccessorsManager) , ColumnShardActorId(columnShardActorId) , ScanComputeActorId(scanComputeActorId) , BlobCacheActorId(NBlobCache::MakeBlobCacheServiceId()) @@ -60,16 +66,15 @@ TColumnShardScan::TColumnShardScan(const TActorId& columnShardActorId, const TAc , ReadMetadataRange(readMetadataRange) , Timeout(timeout ? timeout + SCAN_HARD_TIMEOUT_GAP : SCAN_HARD_TIMEOUT) , ScanCountersPool(scanCountersPool) - , Stats(NTracing::TTraceClient::GetLocalClient("SHARD", ::ToString(TabletId)/*, "SCAN_TXID:" + ::ToString(TxId)*/)) + , Stats(NTracing::TTraceClient::GetLocalClient("SHARD", ::ToString(TabletId) /*, "SCAN_TXID:" + ::ToString(TxId)*/)) , ComputeShardingPolicy(computeShardingPolicy) { AFL_VERIFY(ReadMetadataRange); KeyYqlSchema = ReadMetadataRange->GetKeyYqlSchema(); } void TColumnShardScan::Bootstrap(const TActorContext& ctx) { - TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_SCAN) - ("SelfId", SelfId())("TabletId", TabletId)("ScanId", ScanId)("TxId", TxId)("ScanGen", ScanGen) - ); + TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_SCAN) ("SelfId", SelfId())( + "TabletId", TabletId)("ScanId", ScanId)("TxId", TxId)("ScanGen", ScanGen)); auto g = Stats->MakeGuard("bootstrap"); ScanActorId = ctx.SelfID; @@ -77,7 +82,7 @@ void TColumnShardScan::Bootstrap(const TActorContext& ctx) { ResourceSubscribeActorId = ctx.Register(new NResourceBroker::NSubscribe::TActor(TabletId, SelfId())); ReadCoordinatorActorId = ctx.Register(new NBlobOperations::NRead::TReadCoordinatorActor(TabletId, SelfId())); - std::shared_ptr context = std::make_shared(StoragesManager, ScanCountersPool, + std::shared_ptr context = std::make_shared(StoragesManager, DataAccessorsManager, ScanCountersPool, ReadMetadataRange, SelfId(), ResourceSubscribeActorId, ReadCoordinatorActorId, ComputeShardingPolicy, ScanId); ScanIterator = ReadMetadataRange->StartScan(context); auto startResult = ScanIterator->Start(); @@ -90,7 +95,8 @@ void TColumnShardScan::Bootstrap(const TActorContext& ctx) { ScheduleWakeup(GetDeadline()); // propagate self actor id // TODO: FlagSubscribeOnSession ? - Send(ScanComputeActorId, new NKqp::TEvKqpCompute::TEvScanInitActor(ScanId, ctx.SelfID, ScanGen, TabletId), IEventHandle::FlagTrackDelivery); + Send(ScanComputeActorId, new NKqp::TEvKqpCompute::TEvScanInitActor(ScanId, ctx.SelfID, ScanGen, TabletId), + IEventHandle::FlagTrackDelivery); Become(&TColumnShardScan::StateScan); ContinueProcessing(); @@ -98,7 +104,6 @@ void TColumnShardScan::Bootstrap(const TActorContext& ctx) { } void TColumnShardScan::HandleScan(NColumnShard::TEvPrivate::TEvTaskProcessedResult::TPtr& ev) { - --InFlightReads; auto g = Stats->MakeGuard("task_result"); auto result = ev->Get()->ExtractResult(); if (result.IsFail()) { @@ -143,9 +148,8 @@ void TColumnShardScan::HandleScan(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev) noe auto prio = msg.GetStatusCode() == NYql::NDqProto::StatusIds::SUCCESS ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN; LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_COLUMNSHARD_SCAN, "Scan " << ScanActorId << " got AbortExecution" - << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId - << " code: " << NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode()) - << " reason: " << reason); + << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId + << " code: " << NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode()) << " reason: " << reason); AbortReason = std::move(reason); Finish(NColumnShard::TScanCounters::EStatusFinish::ExternalAbort); @@ -163,10 +167,8 @@ void TColumnShardScan::HandleScan(TEvents::TEvUndelivered::TPtr& ev) { } LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_COLUMNSHARD_SCAN, - "Scan " << ScanActorId << " undelivered event: " << eventType - << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId - << " reason: " << ev->Get()->Reason - << " description: " << AbortReason); + "Scan " << ScanActorId << " undelivered event: " << eventType << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen + << " tablet: " << TabletId << " reason: " << ev->Get()->Reason << " description: " << AbortReason); Finish(NColumnShard::TScanCounters::EStatusFinish::UndeliveredEvent); } @@ -174,7 +176,7 @@ void TColumnShardScan::HandleScan(TEvents::TEvUndelivered::TPtr& ev) { void TColumnShardScan::HandleScan(TEvents::TEvWakeup::TPtr& /*ev*/) { LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_COLUMNSHARD_SCAN, "Scan " << ScanActorId << " guard execution timeout" - << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId); + << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId); if (TMonotonic::Now() >= GetDeadline()) { Finish(NColumnShard::TScanCounters::EStatusFinish::Deadline); @@ -235,23 +237,28 @@ bool TColumnShardScan::ProduceResults() noexcept { { MakeResult(0); if (shardedBatch.IsSharded()) { - AFL_INFO(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "compute_sharding_success")("count", shardedBatch.GetSplittedByShards().size())("info", ComputeShardingPolicy.DebugString()); + AFL_INFO(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "compute_sharding_success")( + "count", shardedBatch.GetSplittedByShards().size())("info", ComputeShardingPolicy.DebugString()); Result->SplittedBatches = shardedBatch.GetSplittedByShards(); } else { if (ComputeShardingPolicy.IsEnabled()) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "compute_sharding_problems")("info", ComputeShardingPolicy.DebugString()); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "compute_sharding_problems")( + "info", ComputeShardingPolicy.DebugString()); } } TMemoryProfileGuard mGuard("SCAN_PROFILE::RESULT::TO_KQP", IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY)); Result->ArrowBatch = shardedBatch.GetRecordBatch(); Rows += batch->num_rows(); Bytes += NArrow::GetTableDataSize(Result->ArrowBatch); - - ACFL_DEBUG("stage", "data_format")("batch_size", NArrow::GetTableDataSize(Result->ArrowBatch))("num_rows", numRows)("batch_columns", JoinSeq(",", batch->schema()->field_names())); + + ACFL_DEBUG("stage", "data_format")("batch_size", NArrow::GetTableDataSize(Result->ArrowBatch))("num_rows", numRows)( + "batch_columns", JoinSeq(",", batch->schema()->field_names())); } if (CurrentLastReadKey) { - NArrow::NMerger::TSortableBatchPosition pNew(result.GetLastReadKey(), 0, result.GetLastReadKey()->schema()->field_names(), {}, ReadMetadataRange->IsDescSorted()); - NArrow::NMerger::TSortableBatchPosition pOld(CurrentLastReadKey, 0, CurrentLastReadKey->schema()->field_names(), {}, ReadMetadataRange->IsDescSorted()); + NArrow::NMerger::TSortableBatchPosition pNew( + result.GetLastReadKey(), 0, result.GetLastReadKey()->schema()->field_names(), {}, ReadMetadataRange->IsDescSorted()); + NArrow::NMerger::TSortableBatchPosition pOld( + CurrentLastReadKey, 0, CurrentLastReadKey->schema()->field_names(), {}, ReadMetadataRange->IsDescSorted()); AFL_VERIFY(pOld < pNew)("old", pOld.DebugJson().GetStringRobust())("new", pNew.DebugJson().GetStringRobust()); } CurrentLastReadKey = result.GetLastReadKey(); @@ -296,8 +303,8 @@ void TColumnShardScan::ContinueProcessing() { } } } - AFL_VERIFY(!ScanIterator || !ChunksLimiter.HasMore() || InFlightReads || ScanCountersPool.InWaiting())("scan_actor_id", ScanActorId)("tx_id", TxId)("scan_id", ScanId)("gen", ScanGen)("tablet", TabletId) - ("debug", ScanIterator->DebugString()); + AFL_VERIFY(!ScanIterator || !ChunksLimiter.HasMore() || ScanCountersPool.InWaiting())("scan_actor_id", ScanActorId)("tx_id", TxId)("scan_id", ScanId)( + "gen", ScanGen)("tablet", TabletId)("debug", ScanIterator->DebugString()); } void TColumnShardScan::MakeResult(size_t reserveRows /*= 0*/) { @@ -358,20 +365,19 @@ bool TColumnShardScan::SendResult(bool pageFault, bool lastBatch) { PageFaults = 0; LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_COLUMNSHARD_SCAN, - "Scan " << ScanActorId << " send ScanData to " << ScanComputeActorId - << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId - << " bytes: " << Bytes << " rows: " << Rows << " page faults: " << Result->PageFaults - << " finished: " << Result->Finished << " pageFault: " << Result->PageFault - << " arrow schema:\n" << (Result->ArrowBatch ? Result->ArrowBatch->schema()->ToString() : "")); + "Scan " << ScanActorId << " send ScanData to " << ScanComputeActorId << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen + << " tablet: " << TabletId << " bytes: " << Bytes << " rows: " << Rows << " page faults: " << Result->PageFaults + << " finished: " << Result->Finished << " pageFault: " << Result->PageFault << " arrow schema:\n" + << (Result->ArrowBatch ? Result->ArrowBatch->schema()->ToString() : "")); Finished = Result->Finished; if (Finished) { - ALS_INFO(NKikimrServices::TX_COLUMNSHARD_SCAN) << - "Scanner finished " << ScanActorId << " and sent to " << ScanComputeActorId - << " packs: " << PacksSum << " txId: " << TxId << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId - << " bytes: " << Bytes << "/" << BytesSum << " rows: " << Rows << "/" << RowsSum << " page faults: " << Result->PageFaults - << " finished: " << Result->Finished << " pageFault: " << Result->PageFault - << " stats:" << Stats->ToJson() << ";iterator:" << (ScanIterator ? ScanIterator->DebugString(false) : "NO"); + ALS_INFO(NKikimrServices::TX_COLUMNSHARD_SCAN) + << "Scanner finished " << ScanActorId << " and sent to " << ScanComputeActorId << " packs: " << PacksSum << " txId: " << TxId + << " scanId: " << ScanId << " gen: " << ScanGen << " tablet: " << TabletId << " bytes: " << Bytes << "/" << BytesSum + << " rows: " << Rows << "/" << RowsSum << " page faults: " << Result->PageFaults << " finished: " << Result->Finished + << " pageFault: " << Result->PageFault << " stats:" << Stats->ToJson() + << ";iterator:" << (ScanIterator ? ScanIterator->DebugString(false) : "NO"); Result->StatsOnFinished = std::make_shared(ScanIterator->GetStats()); } else { Y_ABORT_UNLESS(ChunksLimiter.Take(Bytes)); @@ -383,7 +389,7 @@ bool TColumnShardScan::SendResult(bool pageFault, bool lastBatch) { AckReceivedInstant.reset(); LastResultInstant = TMonotonic::Now(); - Send(ScanComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery); // TODO: FlagSubscribeOnSession ? + Send(ScanComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery); // TODO: FlagSubscribeOnSession ? ReportStats(); @@ -403,8 +409,7 @@ void TColumnShardScan::SendScanError(const TString& reason) { } void TColumnShardScan::Finish(const NColumnShard::TScanCounters::EStatusFinish status) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_COLUMNSHARD_SCAN, - "Scan " << ScanActorId << " finished for tablet " << TabletId); + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_COLUMNSHARD_SCAN, "Scan " << ScanActorId << " finished for tablet " << TabletId); Send(ColumnShardActorId, new NColumnShard::TEvPrivate::TEvReadFinished(RequestCookie, TxId)); AFL_VERIFY(StartInstant); @@ -432,4 +437,4 @@ TMonotonic TColumnShardScan::GetDeadline() const { } return *StartInstant + Timeout; } -} \ No newline at end of file +} // namespace NKikimr::NOlap::NReader diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.h b/ydb/core/tx/columnshard/engines/reader/actor/actor.h index ebf199826421..454f0d0d795f 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.h +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.h @@ -21,6 +21,7 @@ class TColumnShardScan: public TActorBootstrapped, NArrow::IRo TActorId ResourceSubscribeActorId; TActorId ReadCoordinatorActorId; const std::shared_ptr StoragesManager; + const std::shared_ptr DataAccessorsManager; std::optional StartInstant; public: @@ -32,9 +33,11 @@ class TColumnShardScan: public TActorBootstrapped, NArrow::IRo virtual void PassAway() override; TColumnShardScan(const TActorId& columnShardActorId, const TActorId& scanComputeActorId, - const std::shared_ptr& storagesManager, const TComputeShardingPolicy& computeShardingPolicy, ui32 scanId, ui64 txId, - ui32 scanGen, ui64 requestCookie, ui64 tabletId, TDuration timeout, const TReadMetadataBase::TConstPtr& readMetadataRange, - NKikimrDataEvents::EDataFormat dataFormat, const NColumnShard::TScanCounters& scanCountersPool); + const std::shared_ptr& storagesManager, + const std::shared_ptr& dataAccessorsManager, + const TComputeShardingPolicy& computeShardingPolicy, ui32 scanId, ui64 txId, ui32 scanGen, ui64 requestCookie, ui64 tabletId, + TDuration timeout, const TReadMetadataBase::TConstPtr& readMetadataRange, NKikimrDataEvents::EDataFormat dataFormat, + const NColumnShard::TScanCounters& scanCountersPool); void Bootstrap(const TActorContext& ctx); @@ -134,7 +137,6 @@ class TColumnShardScan: public TActorBootstrapped, NArrow::IRo TChunksLimiter ChunksLimiter; THolder Result; std::shared_ptr CurrentLastReadKey; - i64 InFlightReads = 0; bool Finished = false; std::optional LastResultInstant; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/columns_set.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/columns_set.h index 98e77f4971e9..23514aff1dce 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/columns_set.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/columns_set.h @@ -8,6 +8,12 @@ namespace NKikimr::NOlap::NReader::NPlain { +enum class EMemType { + Blob, + Raw, + RawSequential +}; + enum class EStageFeaturesIndexes { Filter = 0, Fetching = 1, diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp index ed6db34c5e59..839e658d54ab 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp @@ -12,26 +12,19 @@ std::unique_ptr TSpecialReadContext::Build ui64 TSpecialReadContext::GetMemoryForSources(const THashMap>& sources) { ui64 result = 0; - bool hasSequentialReadSources = false; for (auto&& i : sources) { auto fetchingPlan = GetColumnsFetchingPlan(i.second); AFL_VERIFY(i.second->GetIntervalsCount()); const ui64 sourceMemory = std::max(1, i.second->GetResourceGuardsMemory() / i.second->GetIntervalsCount()); - if (!i.second->IsSourceInMemory()) { - hasSequentialReadSources = true; - } result += sourceMemory; } AFL_VERIFY(result); - if (hasSequentialReadSources) { - result += ReadSequentiallyBufferSize; - } + result += ReadSequentiallyBufferSize; return result; } std::shared_ptr TSpecialReadContext::GetColumnsFetchingPlan(const std::shared_ptr& source) { - const bool needSnapshots = !source->GetExclusiveIntervalOnly() || ReadMetadata->GetRequestSnapshot() < source->GetRecordSnapshotMax() || - !source->IsSourceInMemory(); + const bool needSnapshots = !source->GetExclusiveIntervalOnly() || ReadMetadata->GetRequestSnapshot() < source->GetRecordSnapshotMax(); const bool partialUsageByPK = [&]() { switch (source->GetUsageClass()) { case TPKRangeFilter::EUsageClass::PartialUsage: @@ -43,7 +36,7 @@ std::shared_ptr TSpecialReadContext::GetColumnsFetchingPlan(con } }(); const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false); - const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly() && source->IsSourceInMemory(); + const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly(); const bool hasDeletions = source->GetHasDeletions(); bool needShardingFilter = false; if (!!ReadMetadata->GetRequestShardingInfo()) { @@ -83,33 +76,41 @@ class TColumnsAccumulator { , GuaranteeNotOptional(guaranteeNotOptional) { } - bool AddFetchingStep(TFetchingScript& script, const TColumnsSetIds& columns, const EStageFeaturesIndexes& stage) { - auto actualColumns = (TColumnsSetIds)columns - FetchingReadyColumns; + TColumnsSetIds GetNotFetchedAlready(const TColumnsSetIds& columns) const { + return columns - FetchingReadyColumns; + } + + bool AddFetchingStep(TFetchingScript& script, const TColumnsSetIds& columns, const EStageFeaturesIndexes stage) { + auto actualColumns = GetNotFetchedAlready(columns); FetchingReadyColumns = FetchingReadyColumns + (TColumnsSetIds)columns; if (!actualColumns.IsEmpty()) { - script.AddStep(std::make_shared(actualColumns, stage)); + script.Allocation(columns.GetColumnIds(), stage, EMemType::Blob); script.AddStep(std::make_shared(actualColumns)); return true; } return false; } - bool AddAssembleStep(TFetchingScript& script, const TColumnsSetIds& columns, const TString& purposeId, const bool optional) { - auto actualColumns = (TColumnsSetIds)columns - AssemblerReadyColumns; + bool AddAssembleStep(TFetchingScript& script, const TColumnsSetIds& columns, const TString& purposeId, const EStageFeaturesIndexes stage, + const bool sequential) { + auto actualColumns = columns - AssemblerReadyColumns; AssemblerReadyColumns = AssemblerReadyColumns + columns; if (!actualColumns.IsEmpty()) { auto actualSet = std::make_shared(actualColumns.GetColumnIds(), FullSchema); - if (optional) { - const auto notOptionalColumnIds = GuaranteeNotOptional->Intersect(*actualSet); - if (notOptionalColumnIds.size()) { - std::shared_ptr cross = actualSet->BuildSamePtr(notOptionalColumnIds); - script.AddStep(std::make_shared(cross, purposeId)); + if (sequential) { + const auto notSequentialColumnIds = GuaranteeNotOptional->Intersect(*actualSet); + if (notSequentialColumnIds.size()) { + script.Allocation(notSequentialColumnIds, stage, EMemType::Raw); + std::shared_ptr cross = actualSet->BuildSamePtr(notSequentialColumnIds); + script.AddStep(cross, purposeId); *actualSet = *actualSet - *cross; } if (!actualSet->IsEmpty()) { - script.AddStep(std::make_shared(actualSet, purposeId)); + script.Allocation(notSequentialColumnIds, stage, EMemType::RawSequential); + script.AddStep(actualSet, purposeId); } } else { - script.AddStep(std::make_shared(actualSet, purposeId)); + script.Allocation(actualColumns.GetColumnIds(), stage, EMemType::Raw); + script.AddStep(actualSet, purposeId); } return true; } @@ -121,12 +122,18 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c const bool partialUsageByPredicateExt, const bool useIndexes, const bool needFilterSharding, const bool needFilterDeletion) const { std::shared_ptr result = std::make_shared(*this); const bool partialUsageByPredicate = partialUsageByPredicateExt && PredicateColumns->GetColumnsCount(); + + TColumnsAccumulator acc(MergeColumns, ReadMetadata->GetResultSchema()); + result->AddStep(std::make_shared()); + if (exclusiveSource) { + result->AddStep(acc.GetNotFetchedAlready(*FFColumns)); + } + if (!!IndexChecker && useIndexes && exclusiveSource) { result->AddStep(std::make_shared(std::make_shared(IndexChecker->GetIndexIds()))); result->AddStep(std::make_shared(IndexChecker)); } bool hasFilterSharding = false; - TColumnsAccumulator acc(MergeColumns, ReadMetadata->GetResultSchema()); if (needFilterSharding && !ShardingColumns->IsEmpty()) { hasFilterSharding = true; TColumnsSetIds columnsFetch = *ShardingColumns; @@ -134,7 +141,7 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c columnsFetch = columnsFetch + *PKColumns + *SpecColumns; } acc.AddFetchingStep(*result, columnsFetch, EStageFeaturesIndexes::Filter); - acc.AddAssembleStep(*result, columnsFetch, "SPEC_SHARDING", false); + acc.AddAssembleStep(*result, columnsFetch, "SPEC_SHARDING", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared()); } if (!EFColumns->GetColumnsCount() && !partialUsageByPredicate) { @@ -156,19 +163,19 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c if (columnsFetch.GetColumnsCount() || hasFilterSharding || needFilterDeletion) { acc.AddFetchingStep(*result, columnsFetch, EStageFeaturesIndexes::Fetching); if (needSnapshots) { - acc.AddAssembleStep(*result, *SpecColumns, "SPEC", false); + acc.AddAssembleStep(*result, *SpecColumns, "SPEC", EStageFeaturesIndexes::Fetching, false); } if (!exclusiveSource) { - acc.AddAssembleStep(*result, *MergeColumns, "LAST_PK", false); + acc.AddAssembleStep(*result, *MergeColumns, "LAST_PK", EStageFeaturesIndexes::Fetching, false); } if (needSnapshots) { result->AddStep(std::make_shared()); } if (needFilterDeletion) { - acc.AddAssembleStep(*result, *DeletionColumns, "SPEC_DELETION", false); + acc.AddAssembleStep(*result, *DeletionColumns, "SPEC_DELETION", EStageFeaturesIndexes::Fetching, false); result->AddStep(std::make_shared()); } - acc.AddAssembleStep(*result, columnsFetch, "LAST", true); + acc.AddAssembleStep(*result, columnsFetch, "LAST", EStageFeaturesIndexes::Fetching, true); } else { return nullptr; } @@ -189,15 +196,15 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c acc.AddFetchingStep(*result, columnsFetch, EStageFeaturesIndexes::Filter); if (needFilterDeletion) { - acc.AddAssembleStep(*result, *DeletionColumns, "SPEC_DELETION", false); + acc.AddAssembleStep(*result, *DeletionColumns, "SPEC_DELETION", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared()); } if (partialUsageByPredicate) { - acc.AddAssembleStep(*result, *PredicateColumns, "PREDICATE", false); + acc.AddAssembleStep(*result, *PredicateColumns, "PREDICATE", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared()); } if (needSnapshots || FFColumns->Cross(*SpecColumns)) { - acc.AddAssembleStep(*result, *SpecColumns, "SPEC", false); + acc.AddAssembleStep(*result, *SpecColumns, "SPEC", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared()); } for (auto&& i : ReadMetadata->GetProgram().GetSteps()) { @@ -205,7 +212,7 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c break; } TColumnsSet stepColumnIds(i->GetFilterOriginalColumnIds(), ReadMetadata->GetResultSchema()); - acc.AddAssembleStep(*result, stepColumnIds, "EF", true); + acc.AddAssembleStep(*result, stepColumnIds, "EF", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared(i)); if (!i->IsFilterOnly()) { break; @@ -215,7 +222,7 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c result->AddStep(std::make_shared(GetReadMetadata()->Limit, GetReadMetadata()->IsDescSorted())); } acc.AddFetchingStep(*result, *FFColumns, EStageFeaturesIndexes::Fetching); - acc.AddAssembleStep(*result, *FFColumns, "LAST", true); + acc.AddAssembleStep(*result, *FFColumns, "LAST", EStageFeaturesIndexes::Fetching, true); } else { result->SetBranchName("merge"); TColumnsSet columnsFetch = *MergeColumns + *EFColumns; @@ -225,13 +232,13 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c AFL_VERIFY(columnsFetch.GetColumnsCount()); acc.AddFetchingStep(*result, columnsFetch, EStageFeaturesIndexes::Filter); - acc.AddAssembleStep(*result, *SpecColumns, "SPEC", false); - acc.AddAssembleStep(*result, *PKColumns, "PK", false); + acc.AddAssembleStep(*result, *SpecColumns, "SPEC", EStageFeaturesIndexes::Filter, false); + acc.AddAssembleStep(*result, *PKColumns, "PK", EStageFeaturesIndexes::Filter, false); if (needSnapshots) { result->AddStep(std::make_shared()); } if (needFilterDeletion) { - acc.AddAssembleStep(*result, *DeletionColumns, "SPEC_DELETION", false); + acc.AddAssembleStep(*result, *DeletionColumns, "SPEC_DELETION", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared()); } if (partialUsageByPredicate) { @@ -242,14 +249,14 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c break; } TColumnsSet stepColumnIds(i->GetFilterOriginalColumnIds(), ReadMetadata->GetResultSchema()); - acc.AddAssembleStep(*result, stepColumnIds, "EF", true); + acc.AddAssembleStep(*result, stepColumnIds, "EF", EStageFeaturesIndexes::Filter, false); result->AddStep(std::make_shared(i)); if (!i->IsFilterOnly()) { break; } } acc.AddFetchingStep(*result, *FFColumns, EStageFeaturesIndexes::Fetching); - acc.AddAssembleStep(*result, *FFColumns, "LAST", true); + acc.AddAssembleStep(*result, *FFColumns, "LAST", EStageFeaturesIndexes::Fetching, true); } return result; } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h index 4633859c4651..1a6a78cf53c2 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h @@ -23,11 +23,35 @@ class TFetchedData { YDB_READONLY_DEF(std::shared_ptr, Filter); YDB_READONLY(bool, UseFilter, false); + std::optional PortionAccessor; + bool DataAdded = false; + public: TFetchedData(const bool useFilter) : UseFilter(useFilter) { } + void SetUseFilter(const bool value) { + if (UseFilter == value) { + return; + } + AFL_VERIFY(!DataAdded); + } + + bool HasPortionAccessor() const { + return !!PortionAccessor; + } + + void SetPortionAccessor(TPortionDataAccessor&& accessor) { + AFL_VERIFY(!PortionAccessor); + PortionAccessor = std::move(accessor); + } + + const TPortionDataAccessor& GetPortionAccessor() const { + AFL_VERIFY(!!PortionAccessor); + return *PortionAccessor; + } + ui32 GetFilteredCount(const ui32 recordsCount, const ui32 defLimit) const { if (!Filter) { return std::min(defLimit, recordsCount); @@ -76,6 +100,7 @@ class TFetchedData { } void AddFilter(const std::shared_ptr& filter) { + DataAdded = true; if (!filter) { return; } @@ -120,6 +145,7 @@ class TFetchedData { } void AddBatch(const std::shared_ptr& table) { + DataAdded = true; AFL_VERIFY(table); if (UseFilter) { AddBatch(table->BuildTableVerified()); @@ -134,6 +160,7 @@ class TFetchedData { } void AddBatch(const std::shared_ptr& table) { + DataAdded = true; auto tableLocal = table; if (Filter && UseFilter) { AFL_VERIFY(Filter->Apply(tableLocal)); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp index 2d0ec349aa6a..39400a9a25f5 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp @@ -38,20 +38,6 @@ TConclusion TColumnBlobsFetchingStep::DoExecuteInplace( return !source->StartFetchingColumns(source, step, Columns); } -ui64 TColumnBlobsFetchingStep::DoPredictRawBytes(const std::shared_ptr& source) const { - ui64 result = source->GetColumnRawBytes(Columns.GetColumnIds()); - if (source->GetContext()->GetReadMetadata()->Limit && source->GetExclusiveIntervalOnly()) { - result = std::max(result * 1.0 * source->GetContext()->GetReadMetadata()->Limit / source->GetRecordsCount(), - source->GetColumnBlobBytes(Columns.GetColumnIds())); - } - if (!result) { - return Columns.GetColumnIds().size() * source->GetRecordsCount() * - sizeof(ui32); // null for all records for all columns in future will be - } else { - return result; - } -} - ui64 TColumnBlobsFetchingStep::GetProcessingDataSize(const std::shared_ptr& source) const { return source->GetColumnBlobBytes(Columns.GetColumnIds()); } @@ -61,10 +47,6 @@ TConclusion TIndexBlobsFetchingStep::DoExecuteInplace( return !source->StartFetchingIndexes(source, step, Indexes); } -ui64 TIndexBlobsFetchingStep::DoPredictRawBytes(const std::shared_ptr& source) const { - return source->GetIndexRawBytes(Indexes->GetIndexIdsSet()); -} - TConclusion TAssemblerStep::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { source->AssembleColumns(Columns); return true; @@ -76,41 +58,25 @@ ui64 TAssemblerStep::GetProcessingDataSize(const std::shared_ptr& s TConclusion TOptionalAssemblerStep::DoExecuteInplace( const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { - source->AssembleColumns(Columns); + source->AssembleColumns(Columns, !source->GetExclusiveIntervalOnly() || !source->IsSourceInMemory()); return true; } -bool TOptionalAssemblerStep::DoInitSourceSeqColumnIds(const std::shared_ptr& source) const { - for (auto&& i : Columns->GetColumnIds()) { - if (source->AddSequentialEntityIds(i)) { - return true; - } - } - return false; -} - ui64 TOptionalAssemblerStep::GetProcessingDataSize(const std::shared_ptr& source) const { - return source->GetColumnRawBytes(Columns->GetColumnIds()); + return source->GetColumnsVolume(Columns->GetColumnIds(), EMemType::RawSequential); } TConclusion TFilterProgramStep::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { AFL_VERIFY(source); AFL_VERIFY(Step); - std::shared_ptr table; - if (source->IsSourceInMemory(Step->GetFilterOriginalColumnIds())) { - auto filter = Step->BuildFilter(source->GetStageData().GetTable()); - if (!filter.ok()) { - return TConclusionStatus::Fail(filter.status().message()); - } - source->MutableStageData().AddFilter(*filter); + auto filter = Step->BuildFilter(source->GetStageData().GetTable()); + if (!filter.ok()) { + return TConclusionStatus::Fail(filter.status().message()); } + source->MutableStageData().AddFilter(*filter); return true; } -ui64 TFilterProgramStep::DoPredictRawBytes(const std::shared_ptr& source) const { - return NArrow::TColumnFilter::GetPredictedMemorySize(source->GetRecordsCount()); -} - TConclusion TPredicateFilter::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { auto filter = source->GetContext()->GetReadMetadata()->GetPKRangesFilter().BuildFilter(source->GetStageData().GetTable()->BuildTableVerified()); @@ -156,6 +122,7 @@ TConclusion TShardingFilter::DoExecuteInplace(const std::shared_ptr TBuildFakeSpec::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { + source->SetSourceInMemory(true); std::vector> columns; for (auto&& f : IIndexInfo::ArrowSchemaSnapshot()->fields()) { columns.emplace_back(NArrow::TThreadSimpleArraysCache::GetConst(f->type(), NArrow::DefaultScalar(f->type()), Count)); @@ -226,22 +193,28 @@ TAllocateMemoryStep::TFetchingStepAllocation::TFetchingStepAllocation( TConclusion TAllocateMemoryStep::DoExecuteInplace( const std::shared_ptr& source, const TFetchingScriptCursor& step) const { - auto allocation = std::make_shared(source, GetProcessingDataSize(source), step); + ui64 size = 0; + for (auto&& i : Packs) { + ui32 sizeLocal = source->GetColumnsVolume(i.GetColumns().GetColumnIds(), i.GetMemType()); + if (source->GetStageData().GetUseFilter() && source->GetContext()->GetReadMetadata()->Limit && i.GetMemType() != EMemType::Blob) { + const ui32 filtered = + source->GetStageData().GetFilteredCount(source->GetRecordsCount(), source->GetContext()->GetReadMetadata()->Limit); + if (filtered < source->GetRecordsCount()) { + sizeLocal = sizeLocal * 1.0 * filtered / source->GetRecordsCount(); + } + } + size += sizeLocal; + } + + + auto allocation = std::make_shared(source, size, step); NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation(source->GetContext()->GetProcessMemoryControlId(), source->GetContext()->GetCommonContext()->GetScanId(), source->GetFirstIntervalId(), { allocation }, (ui32)StageIndex); return false; } -ui64 TAllocateMemoryStep::GetProcessingDataSize(const std::shared_ptr& source) const { - ui64 size = source->GetColumnRawBytes(Columns.GetColumnIds()); - - if (source->GetStageData().GetUseFilter() && source->GetContext()->GetReadMetadata()->Limit) { - const ui32 filtered = source->GetStageData().GetFilteredCount(source->GetRecordsCount(), source->GetContext()->GetReadMetadata()->Limit); - if (filtered < source->GetRecordsCount()) { - size = std::max(size * 1.0 * filtered / source->GetRecordsCount(), source->GetColumnBlobBytes(Columns.GetColumnIds())); - } - } - return size; +ui64 TAllocateMemoryStep::GetProcessingDataSize(const std::shared_ptr& /*source*/) const { + return 0; } TString TFetchingScript::DebugString() const { @@ -268,10 +241,53 @@ TFetchingScript::TFetchingScript(const TSpecialReadContext& context) : Limit(context.GetReadMetadata()->Limit) { } +void TFetchingScript::Allocation(const std::set& entityIds, const EStageFeaturesIndexes stage, const EMemType mType) { + if (Steps.size() == 0) { + AddStep(entityIds, mType, stage); + } else { + std::optional addIndex; + for (i32 i = Steps.size() - 1; i >= 0; --i) { + if (auto allocation = std::dynamic_pointer_cast(Steps[i])) { + if (allocation->GetStage() == stage) { + allocation->AddAllocation(entityIds, mType); + return; + } else { + addIndex = i + 1; + } + break; + } else if (std::dynamic_pointer_cast(Steps[i])) { + continue; + } else if (std::dynamic_pointer_cast(Steps[i])) { + continue; + } else { + addIndex = i + 1; + break; + } + } + AFL_VERIFY(addIndex); + InsertStep(*addIndex, entityIds, mType, stage); + } +} + NKikimr::TConclusion TFilterCutLimit::DoExecuteInplace( const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { source->MutableStageData().CutFilter(source->GetRecordsCount(), Limit, Reverse); return true; } +TConclusion TPortionAccessorFetchingStep::DoExecuteInplace( + const std::shared_ptr& source, const TFetchingScriptCursor& step) const { + return !source->StartFetchingAccessor(source, step); +} + +TConclusion TDetectInMem::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { + AFL_VERIFY(source->GetExclusiveIntervalOnly()); + if (source->GetColumnRawBytes(Columns.GetColumnIds()) > 1e+8) { + source->SetSourceInMemory(false); + } else { + source->SetSourceInMemory(true); + } + return true; +} + } // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h index 133aa4db3669..1fc9f8bce543 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h @@ -33,12 +33,6 @@ class IFetchingStep { void AddDataSize(const ui64 size) { SumSize += size; } - virtual ui64 DoPredictRawBytes(const std::shared_ptr& /*source*/) const { - return 0; - } - virtual bool DoInitSourceSeqColumnIds(const std::shared_ptr& /*source*/) const { - return false; - } virtual ~IFetchingStep() = default; @@ -73,6 +67,8 @@ class TFetchingScript { public: TFetchingScript(const TSpecialReadContext& context); + void Allocation(const std::set& entityIds, const EStageFeaturesIndexes stage, const EMemType mType); + void AddStepDataSize(const ui32 index, const ui64 size) { GetStep(index)->AddDataSize(size); } @@ -95,11 +91,18 @@ class TFetchingScript { return Steps[index]; } - ui64 PredictRawBytes(const std::shared_ptr& source) const { - ui64 result = 0; - for (auto&& current : Steps) { - result += current->DoPredictRawBytes(source); - } + template + std::shared_ptr AddStep(Args... args) { + auto result = std::make_shared(args...); + Steps.emplace_back(result); + return result; + } + + template + std::shared_ptr InsertStep(const ui32 index, Args... args) { + AFL_VERIFY(index <= Steps.size())("index", index)("size", Steps.size()); + auto result = std::make_shared(args...); + Steps.insert(Steps.begin() + index, result); return result; } @@ -108,15 +111,6 @@ class TFetchingScript { Steps.emplace_back(step); } - bool InitSourceSeqColumnIds(const std::shared_ptr& source) const { - for (auto it = Steps.rbegin(); it != Steps.rend(); ++it) { - if ((*it)->DoInitSourceSeqColumnIds(source)) { - return true; - } - } - return false; - } - bool IsFinished(const ui32 currentStepIdx) const { AFL_VERIFY(currentStepIdx <= Steps.size()); return currentStepIdx == Steps.size(); @@ -192,9 +186,6 @@ class TBuildFakeSpec: public IFetchingStep { protected: virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; - virtual ui64 DoPredictRawBytes(const std::shared_ptr& /*source*/) const override { - return TIndexInfo::GetSpecialColumnsRecordSize() * Count; - } public: TBuildFakeSpec(const ui32 count) @@ -222,7 +213,19 @@ class TApplyIndexStep: public IFetchingStep { class TAllocateMemoryStep: public IFetchingStep { private: using TBase = IFetchingStep; - TColumnsSetIds Columns; + class TColumnsPack { + private: + YDB_READONLY_DEF(TColumnsSetIds, Columns); + YDB_READONLY(EMemType, MemType, EMemType::Blob); + + public: + TColumnsPack(const TColumnsSetIds& columns, const EMemType memType) + : Columns(columns) + , MemType(memType) { + } + }; + std::vector Packs; + THashMap> Control; const EStageFeaturesIndexes StageIndex; protected: @@ -238,21 +241,49 @@ class TAllocateMemoryStep: public IFetchingStep { public: TFetchingStepAllocation(const std::shared_ptr& source, const ui64 mem, const TFetchingScriptCursor& step); }; - virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; virtual ui64 GetProcessingDataSize(const std::shared_ptr& source) const override; - virtual ui64 DoPredictRawBytes(const std::shared_ptr& /*source*/) const override { - return 0; - } virtual TString DoDebugString() const override { - return TStringBuilder() << "columns=" << Columns.DebugString() << ";stage=" << StageIndex << ";"; + return TStringBuilder() << "stage=" << StageIndex << ";"; } public: - TAllocateMemoryStep(const TColumnsSetIds& columns, const EStageFeaturesIndexes stageIndex) + void AddAllocation(const TColumnsSetIds& ids, const EMemType memType) { + if (!ids.GetColumnsCount()) { + return; + } + for (auto&& i : ids.GetColumnIds()) { + AFL_VERIFY(Control[i].emplace(memType).second); + } + Packs.emplace_back(ids, memType); + } + EStageFeaturesIndexes GetStage() const { + return StageIndex; + } + + TAllocateMemoryStep(const TColumnsSetIds& columns, const EMemType memType, const EStageFeaturesIndexes stageIndex) : TBase("ALLOCATE_MEMORY::" + ::ToString(stageIndex)) - , Columns(columns) , StageIndex(stageIndex) { + AddAllocation(columns, memType); + } +}; + +class TDetectInMemStep: public IFetchingStep { +private: + using TBase = IFetchingStep; + const TColumnsSetIds Columns; + +protected: + virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; + virtual TString DoDebugString() const override { + return TStringBuilder() << "columns=" << Columns.DebugString() << ";"; + } + +public: + virtual ui64 GetProcessingDataSize(const std::shared_ptr& source) const override; + TDetectInMemStep(const TColumnsSetIds& columns) + : TBase("FETCHING_COLUMNS") + , Columns(columns) { AFL_VERIFY(Columns.GetColumnsCount()); } }; @@ -264,7 +295,6 @@ class TColumnBlobsFetchingStep: public IFetchingStep { protected: virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; - virtual ui64 DoPredictRawBytes(const std::shared_ptr& source) const override; virtual TString DoDebugString() const override { return TStringBuilder() << "columns=" << Columns.DebugString() << ";"; } @@ -278,6 +308,22 @@ class TColumnBlobsFetchingStep: public IFetchingStep { } }; +class TPortionAccessorFetchingStep: public IFetchingStep { +private: + using TBase = IFetchingStep; + +protected: + virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; + virtual TString DoDebugString() const override { + return TStringBuilder(); + } + +public: + TPortionAccessorFetchingStep() + : TBase("FETCHING_ACCESSOR") { + } +}; + class TIndexBlobsFetchingStep: public IFetchingStep { private: using TBase = IFetchingStep; @@ -285,7 +331,6 @@ class TIndexBlobsFetchingStep: public IFetchingStep { protected: virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; - virtual ui64 DoPredictRawBytes(const std::shared_ptr& source) const override; virtual TString DoDebugString() const override { return TStringBuilder() << "indexes=" << Indexes->DebugString() << ";"; } @@ -326,9 +371,6 @@ class TOptionalAssemblerStep: public IFetchingStep { return TStringBuilder() << "columns=" << Columns->DebugString() << ";"; } -protected: - virtual bool DoInitSourceSeqColumnIds(const std::shared_ptr& source) const override; - public: virtual ui64 GetProcessingDataSize(const std::shared_ptr& source) const override; @@ -346,9 +388,6 @@ class TFilterProgramStep: public IFetchingStep { using TBase = IFetchingStep; std::shared_ptr Step; -protected: - virtual ui64 DoPredictRawBytes(const std::shared_ptr& source) const override; - public: virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; TFilterProgramStep(const std::shared_ptr& step) @@ -363,18 +402,12 @@ class TFilterCutLimit: public IFetchingStep { const ui32 Limit; const bool Reverse; -protected: - virtual ui64 DoPredictRawBytes(const std::shared_ptr& /*source*/) const override { - return 0; - } - public: virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; TFilterCutLimit(const ui32 limit, const bool reverse) : TBase("LIMIT") , Limit(limit) - , Reverse(reverse) - { + , Reverse(reverse) { } }; @@ -400,6 +433,20 @@ class TSnapshotFilter: public IFetchingStep { } }; +class TDetectInMem: public IFetchingStep { +private: + using TBase = IFetchingStep; + TColumnsSetIds Columns; + +public: + virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; + TDetectInMem(const TColumnsSetIds& columns) + : TBase("DETECT_IN_MEM") + , Columns(columns) + { + } +}; + class TDeletionFilter: public IFetchingStep { private: using TBase = IFetchingStep; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp index 87de386beda9..78dc27ac74ec 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp @@ -67,23 +67,21 @@ void TScanHead::OnIntervalResult(std::shared_ptrGetCommonContext()->GetReadMetadata()->HasGuaranteeExclusivePK(); +// const bool guaranteeExclusivePK = Context->GetCommonContext()->GetReadMetadata()->HasGuaranteeExclusivePK(); TScanContext context; for (auto itPoint = BorderPoints.begin(); itPoint != BorderPoints.end(); ++itPoint) { auto& point = itPoint->second; context.OnStartPoint(point); if (context.GetIsSpecialPoint()) { - auto detectorResult = DetectSourcesFeatureInContextIntervalScan(context.GetCurrentSources(), guaranteeExclusivePK); for (auto&& i : context.GetCurrentSources()) { i.second->IncIntervalsCount(); } - if (!detectorResult) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "scanner_initializer_aborted")( - "reason", detectorResult.GetErrorMessage()); - Abort(); - return detectorResult; - } } +// const bool isExclusive = context.GetCurrentSources().size() == 1; + for (auto&& i : context.GetCurrentSources()) { + i.second->SetExclusiveIntervalOnly(false);//(isExclusive && i.second->GetExclusiveIntervalOnly()) || guaranteeExclusivePK); + } + for (auto&& i : point.GetFinishSources()) { i->InitFetchingPlan(Context->GetColumnsFetchingPlan(i)); } @@ -95,14 +93,6 @@ TConclusionStatus TScanHead::Start() { for (auto&& i : context.GetCurrentSources()) { i.second->IncIntervalsCount(); } - auto detectorResult = - DetectSourcesFeatureInContextIntervalScan(context.GetCurrentSources(), guaranteeExclusivePK || context.GetIsExclusiveInterval()); - if (!detectorResult) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "scanner_initializer_aborted")( - "reason", detectorResult.GetErrorMessage()); - Abort(); - return detectorResult; - } } } return TConclusionStatus::Success(); @@ -129,121 +119,6 @@ TScanHead::TScanHead(std::deque>&& sources, const s } } -class TSourcesStorageForMemoryOptimization { -private: - class TSourceInfo { - private: - YDB_READONLY(ui64, Memory, 0); - YDB_READONLY_DEF(std::shared_ptr, Source); - YDB_READONLY_DEF(std::shared_ptr, FetchingInfo); - - public: - TSourceInfo(const std::shared_ptr& source, const std::shared_ptr& fetchingInfo) - : Source(source) - , FetchingInfo(fetchingInfo) { - Memory = FetchingInfo->PredictRawBytes(Source); - } - - NJson::TJsonValue DebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue("source", Source->DebugJsonForMemory()); - result.InsertValue("memory", Memory); - // result.InsertValue("FetchingInfo", FetchingInfo->DebugJsonForMemory()); - return result; - } - - bool ReduceMemory() { - const bool result = FetchingInfo->InitSourceSeqColumnIds(Source); - if (result) { - Memory = FetchingInfo->PredictRawBytes(Source); - } - return result; - } - - bool operator<(const TSourceInfo& item) const { - return Memory < item.Memory; - } - - }; - - std::vector Sources; - YDB_READONLY(ui64, MemorySum, 0); - -public: - TString DebugString() const { - NJson::TJsonValue resultJson; - auto& memorySourcesArr = resultJson.InsertValue("sources_by_memory", NJson::JSON_ARRAY); - resultJson.InsertValue("sources_by_memory_count", Sources.size()); - for (auto&& it: Sources) { - auto& sourceMap = memorySourcesArr.AppendValue(NJson::JSON_MAP); - auto& sourcesArr = sourceMap.InsertValue("sources", NJson::JSON_ARRAY); - sourcesArr.AppendValue(it.DebugJson()); - } - return resultJson.GetStringRobust(); - } - - void AddSource(const std::shared_ptr& source, const std::shared_ptr& fetching) { - Sources.emplace_back(TSourceInfo(source, fetching)); - MemorySum += Sources.back().GetMemory(); - } - - bool Optimize(const ui64 memoryLimit) { - if (MemorySum <= memoryLimit) { - return true; - } - std::sort(Sources.begin(), Sources.end()); - while (true) { - std::vector nextSources; - while (memoryLimit < MemorySum && Sources.size()) { - const ui64 currentMemory = Sources.back().GetMemory(); - if (Sources.back().ReduceMemory()) { - AFL_VERIFY(currentMemory <= MemorySum); - MemorySum -= currentMemory; - MemorySum += Sources.back().GetMemory(); - nextSources.emplace_back(std::move(Sources.back())); - } - Sources.pop_back(); - } - if (nextSources.empty() || MemorySum <= memoryLimit) { - break; - } - std::sort(nextSources.begin(), nextSources.end()); - std::swap(nextSources, Sources); - } - return MemorySum <= memoryLimit; - } -}; - -TConclusionStatus TScanHead::DetectSourcesFeatureInContextIntervalScan( - const THashMap>& intervalSources, const bool isExclusiveInterval) const { - TSourcesStorageForMemoryOptimization optimizer; - for (auto&& i : intervalSources) { - if (!isExclusiveInterval) { - i.second->SetExclusiveIntervalOnly(false); - } - auto fetchingPlan = Context->GetColumnsFetchingPlan(i.second); - optimizer.AddSource(i.second, fetchingPlan); - } - const ui64 startMemory = optimizer.GetMemorySum(); - if (!optimizer.Optimize(Context->ReduceMemoryIntervalLimit) && Context->RejectMemoryIntervalLimit < optimizer.GetMemorySum()) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "next_internal_broken")("reason", "a lot of memory need")("start", startMemory)( - "reduce_limit", Context->ReduceMemoryIntervalLimit)("reject_limit", Context->RejectMemoryIntervalLimit)( - "need", optimizer.GetMemorySum())("path_id", Context->GetReadMetadata()->GetPathId())( - "details", IS_LOG_PRIORITY_ENABLED(NActors::NLog::PRI_DEBUG, NKikimrServices::TX_COLUMNSHARD_SCAN) ? optimizer.DebugString() - : "NEED_DEBUG_LEVEL"); - Context->GetCommonContext()->GetCounters().OnOptimizedIntervalMemoryFailed(optimizer.GetMemorySum()); - return TConclusionStatus::Fail("We need a lot of memory in time for interval scanner: " + ::ToString(optimizer.GetMemorySum()) + - " path_id: " + Context->GetReadMetadata()->GetPathId() + ". We need wait compaction processing. Sorry."); - } else if (optimizer.GetMemorySum() < startMemory) { - AFL_INFO(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "memory_reduce_active")("reason", "need reduce memory")("start", startMemory)( - "reduce_limit", Context->ReduceMemoryIntervalLimit)("reject_limit", Context->RejectMemoryIntervalLimit)( - "need", optimizer.GetMemorySum())("path_id", Context->GetReadMetadata()->GetPathId()); - Context->GetCommonContext()->GetCounters().OnOptimizedIntervalMemoryReduced(startMemory - optimizer.GetMemorySum()); - } - Context->GetCommonContext()->GetCounters().OnOptimizedIntervalMemoryRequired(optimizer.GetMemorySum()); - return TConclusionStatus::Success(); -} - TConclusion TScanHead::BuildNextInterval() { if (Context->IsAborted()) { return false; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index 6f73d97664db..bd24a04b2493 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -30,7 +30,7 @@ void IDataSource::RegisterInterval(TFetchingInterval& interval, const std::share if (AtomicCas(&SourceStartedFlag, 1, 0)) { SetFirstIntervalId(interval.GetIntervalId()); AFL_VERIFY(FetchingPlan); - StageData = std::make_unique(GetExclusiveIntervalOnly() && IsSourceInMemory()); + StageData = std::make_unique(GetExclusiveIntervalOnly()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("InitFetchingPlan", FetchingPlan->DebugString())("source_idx", SourceIdx); NActors::TLogContextGuard logGuard(NActors::TLogContextBuilder::Build()("source", SourceIdx)("method", "InitFetchingPlan")); if (Context->IsAborted()) { @@ -59,18 +59,18 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob ui32 fetchedChunks = 0; ui32 nullChunks = 0; for (auto&& i : columnIds) { - auto columnChunks = Portion.GetColumnChunksPointers(i); + auto columnChunks = GetStageData().GetPortionAccessor().GetColumnChunksPointers(i); if (columnChunks.empty()) { continue; } - auto itFilter = cFilter.GetIterator(false, Portion.GetPortionInfo().GetRecordsCount()); + auto itFilter = cFilter.GetIterator(false, Portion->GetRecordsCount()); bool itFinished = false; for (auto&& c : columnChunks) { AFL_VERIFY(!itFinished); if (!itFilter.IsBatchForSkip(c->GetMeta().GetRecordsCount())) { - auto reading = blobsAction.GetReading(Portion.GetPortionInfo().GetColumnStorageId(c->GetColumnId(), Schema->GetIndexInfo())); + auto reading = blobsAction.GetReading(Portion->GetColumnStorageId(c->GetColumnId(), Schema->GetIndexInfo())); reading->SetIsBackgroundProcess(false); - reading->AddRange(Portion.GetPortionInfo().RestoreBlobRange(c->BlobRange)); + reading->AddRange(Portion->RestoreBlobRange(c->BlobRange)); ++fetchedChunks; } else { defaultBlocks.emplace(c->GetAddress(), TPortionDataAccessor::TAssembleBlobInfo(c->GetMeta().GetRecordsCount(), @@ -79,7 +79,7 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob } itFinished = !itFilter.Next(c->GetMeta().GetRecordsCount()); } - AFL_VERIFY(itFinished)("filter", itFilter.DebugString())("count", Portion.GetPortionInfo().GetRecordsCount()); + AFL_VERIFY(itFinished)("filter", itFilter.DebugString())("count", Portion->GetRecordsCount()); } AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks)( "reading_actions", blobsAction.GetStorageIds())("columns", columnIds.size()); @@ -119,15 +119,15 @@ bool TPortionDataSource::DoStartFetchingIndexes( TBlobsAction action(GetContext()->GetCommonContext()->GetStoragesManager(), NBlobOperations::EConsumer::SCAN); { std::set indexIds; - for (auto&& i : Portion.GetIndexes()) { + for (auto&& i : GetStageData().GetPortionAccessor().GetIndexes()) { if (!indexes->GetIndexIdsSet().contains(i.GetIndexId())) { continue; } indexIds.emplace(i.GetIndexId()); if (auto bRange = i.GetBlobRangeOptional()) { - auto readAction = action.GetReading(Portion.GetPortionInfo().GetIndexStorageId(i.GetIndexId(), Schema->GetIndexInfo())); + auto readAction = action.GetReading(Portion->GetIndexStorageId(i.GetIndexId(), Schema->GetIndexInfo())); readAction->SetIsBackgroundProcess(false); - readAction->AddRange(Portion.GetPortionInfo().RestoreBlobRange(*bRange)); + readAction->AddRange(Portion->RestoreBlobRange(*bRange)); } } if (indexes->GetIndexIdsSet().size() != indexIds.size()) { @@ -152,7 +152,7 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in THashMap> indexBlobs; std::set indexIds = indexChecker->GetIndexIds(); // NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString()); - std::vector pages = Portion.BuildPages(); + std::vector pages = GetStageData().GetPortionAccessor().BuildPages(); NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter(); for (auto&& p : pages) { for (auto&& i : p.GetIndexes()) { @@ -178,7 +178,7 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in constructor.Add(false, p.GetRecordsCount()); } } - AFL_VERIFY(constructor.Size() == Portion.GetPortionInfo().GetRecordsCount()); + AFL_VERIFY(constructor.Size() == Portion->GetRecordsCount()); if (constructor.IsTotalDenyFilter()) { StageData->AddFilter(NArrow::TColumnFilter::BuildDenyFilter()); } else if (constructor.IsTotalAllowFilter()) { @@ -188,25 +188,61 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in } } -void TPortionDataSource::DoAssembleColumns(const std::shared_ptr& columns) { - auto blobSchema = GetContext()->GetReadMetadata()->GetLoadSchemaVerified(Portion.GetPortionInfo()); +void TPortionDataSource::DoAssembleColumns(const std::shared_ptr& columns, const bool sequential) { + auto blobSchema = GetContext()->GetReadMetadata()->GetLoadSchemaVerified(*Portion); std::optional ss; - if (Portion.GetPortionInfo().HasInsertWriteId()) { - if (Portion.GetPortionInfo().HasCommitSnapshot()) { - ss = Portion.GetPortionInfo().GetCommitSnapshotVerified(); - } else if (GetContext()->GetReadMetadata()->IsMyUncommitted(Portion.GetPortionInfo().GetInsertWriteIdVerified())) { + if (Portion->HasInsertWriteId()) { + if (Portion->HasCommitSnapshot()) { + ss = Portion->GetCommitSnapshotVerified(); + } else if (GetContext()->GetReadMetadata()->IsMyUncommitted(Portion->GetInsertWriteIdVerified())) { ss = GetContext()->GetReadMetadata()->GetRequestSnapshot(); } } - auto batch = Portion.PrepareForAssemble(*blobSchema, columns->GetFilteredSchemaVerified(), MutableStageData().MutableBlobs(), ss) - .AssembleToGeneralContainer(SequentialEntityIds) + auto batch = GetStageData() + .GetPortionAccessor() + .PrepareForAssemble(*blobSchema, columns->GetFilteredSchemaVerified(), MutableStageData().MutableBlobs(), ss) + .AssembleToGeneralContainer(sequential ? columns->GetColumnIds() : std::set()) .DetachResult(); MutableStageData().AddBatch(batch); } +namespace { +class TPortionAccessorFetchingSubscriber: public IDataAccessorRequestsSubscriber { +private: + TFetchingScriptCursor Step; + std::shared_ptr Source; + virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { + AFL_VERIFY(!result.HasErrors()); + AFL_VERIFY(result.GetPortions().size() == 1); + Source->MutableStageData().SetPortionAccessor(std::move(result.ExtractPortionsVector().front())); + AFL_VERIFY(Step.Next()); + auto task = std::make_shared(Source, std::move(Step), Source->GetContext()->GetCommonContext()->GetScanActorId()); + NConveyor::TScanServiceOperator::SendTaskToExecute(task); + } + +public: + TPortionAccessorFetchingSubscriber(const TFetchingScriptCursor& step, const std::shared_ptr& source) + : Step(step) + , Source(source) { + } +}; + +} // namespace + +bool TPortionDataSource::DoStartFetchingAccessor(const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step) { + AFL_VERIFY(!StageData->HasPortionAccessor()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", step.GetName())("fetching_info", step.DebugString()); + + std::shared_ptr request = std::make_shared(); + request->AddPortion(Portion); + request->RegisterSubscriber(std::make_shared(step, sourcePtr)); + GetContext()->GetCommonContext()->GetDataAccessorsManager()->AskData(request); + return true; +} + bool TCommittedDataSource::DoStartFetchingColumns( const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const TColumnsSetIds& /*columns*/) { if (ReadStarted) { @@ -227,7 +263,7 @@ bool TCommittedDataSource::DoStartFetchingColumns( return true; } -void TCommittedDataSource::DoAssembleColumns(const std::shared_ptr& columns) { +void TCommittedDataSource::DoAssembleColumns(const std::shared_ptr& columns, const bool /*sequential*/) { TMemoryProfileGuard mGuard("SCAN_PROFILE::ASSEMBLER::COMMITTED", IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY)); const ISnapshotSchema::TPtr batchSchema = GetContext()->GetReadMetadata()->GetIndexVersions().GetSchemaVerified(GetCommitted().GetSchemaVersion()); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h index 81abdd73b4d1..545bd4b8f8ec 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +52,7 @@ class IDataSource { YDB_READONLY(TPKRangeFilter::EUsageClass, UsageClass, TPKRangeFilter::EUsageClass::PartialUsage); protected: - bool IsSourceInMemoryFlag = true; + std::optional IsSourceInMemoryFlag; THashMap Intervals; std::unique_ptr StageData; @@ -64,16 +65,22 @@ class IDataSource { const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const TColumnsSetIds& columns) = 0; virtual bool DoStartFetchingIndexes( const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const std::shared_ptr& indexes) = 0; - virtual void DoAssembleColumns(const std::shared_ptr& columns) = 0; + virtual void DoAssembleColumns(const std::shared_ptr& columns, const bool sequential) = 0; virtual void DoAbort() = 0; virtual void DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexMeta) = 0; - virtual bool DoAddSequentialEntityIds(const ui32 entityId) = 0; virtual NJson::TJsonValue DoDebugJsonForMemory() const { return NJson::JSON_MAP; } virtual bool DoAddTxConflict() = 0; + virtual bool DoStartFetchingAccessor(const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step) = 0; public: + bool StartFetchingAccessor(const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step) { + return DoStartFetchingAccessor(sourcePtr, step); + } + + virtual ui64 PredictAccessorMemoryBytes() const = 0; + bool AddTxConflict() { if (!Context->GetCommonContext()->HasLock()) { return false; @@ -98,7 +105,19 @@ class IDataSource { } bool IsSourceInMemory() const { - return IsSourceInMemoryFlag; + if (!ExclusiveIntervalOnly) { + return false; + } + AFL_VERIFY(IsSourceInMemoryFlag); + return *IsSourceInMemoryFlag; + } + void SetSourceInMemory(const bool value) { + AFL_VERIFY(!IsSourceInMemoryFlag); + IsSourceInMemoryFlag = value; + AFL_VERIFY(StageData); + if (!value) { + StageData->SetUseFilter(value); + } } void SetFirstIntervalId(const ui64 value) { AFL_VERIFY(!FirstIntervalId); @@ -108,14 +127,6 @@ class IDataSource { AFL_VERIFY(!!FirstIntervalId); return *FirstIntervalId; } - virtual bool IsSourceInMemory(const std::set& fieldIds) const = 0; - bool AddSequentialEntityIds(const ui32 entityId) { - if (DoAddSequentialEntityIds(entityId)) { - IsSourceInMemoryFlag = false; - return true; - } - return false; - } virtual THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobsOriginal) const = 0; virtual ui64 GetPathId() const = 0; @@ -144,11 +155,11 @@ class IDataSource { return DoApplyIndex(indexMeta); } - void AssembleColumns(const std::shared_ptr& columns) { + void AssembleColumns(const std::shared_ptr& columns, const bool sequential = false) { if (columns->IsEmpty()) { return; } - DoAssembleColumns(columns); + DoAssembleColumns(columns, sequential); } bool StartFetchingColumns(const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const TColumnsSetIds& columns) { @@ -169,6 +180,8 @@ class IDataSource { ++IntervalsCount; } + virtual ui64 GetColumnsVolume(const std::set& columnIds, const EMemType type) const = 0; + virtual ui64 GetColumnRawBytes(const std::set& columnIds) const = 0; virtual ui64 GetIndexRawBytes(const std::set& indexIds) const = 0; virtual ui64 GetColumnBlobBytes(const std::set& columnsIds) const = 0; @@ -263,10 +276,8 @@ class IDataSource { class TPortionDataSource: public IDataSource { private: using TBase = IDataSource; - std::set SequentialEntityIds; - TPortionDataAccessor Portion; + const TPortionInfo::TConstPtr Portion; std::shared_ptr Schema; - mutable THashMap FingerprintedData; void NeedFetchColumns(const std::set& columnIds, TBlobsAction& blobsAction, THashMap& nullBlocks, const std::shared_ptr& filter); @@ -276,121 +287,98 @@ class TPortionDataSource: public IDataSource { const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const TColumnsSetIds& columns) override; virtual bool DoStartFetchingIndexes( const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const std::shared_ptr& indexes) override; - virtual void DoAssembleColumns(const std::shared_ptr& columns) override; + virtual void DoAssembleColumns(const std::shared_ptr& columns, const bool sequential) override; virtual NJson::TJsonValue DoDebugJson() const override { NJson::TJsonValue result = NJson::JSON_MAP; result.InsertValue("type", "portion"); - result.InsertValue("info", Portion.GetPortionInfo().DebugString()); - result.InsertValue("commit", Portion.GetPortionInfo().GetCommitSnapshotOptional().value_or(TSnapshot::Zero()).DebugString()); - result.InsertValue("insert", (ui64)Portion.GetPortionInfo().GetInsertWriteIdOptional().value_or(TInsertWriteId(0))); + result.InsertValue("info", Portion->DebugString()); + result.InsertValue("commit", Portion->GetCommitSnapshotOptional().value_or(TSnapshot::Zero()).DebugString()); + result.InsertValue("insert", (ui64)Portion->GetInsertWriteIdOptional().value_or(TInsertWriteId(0))); return result; } virtual NJson::TJsonValue DoDebugJsonForMemory() const override { NJson::TJsonValue result = TBase::DoDebugJsonForMemory(); - auto columns = Portion.GetColumnIds(); - for (auto&& i : SequentialEntityIds) { - AFL_VERIFY(columns.erase(i)); - } - // result.InsertValue("sequential_columns", JoinSeq(",", SequentialEntityIds)); - if (SequentialEntityIds.size()) { - result.InsertValue("min_memory_seq", Portion.GetMinMemoryForReadColumns(SequentialEntityIds)); - result.InsertValue("min_memory_seq_blobs", Portion.GetColumnBlobBytes(SequentialEntityIds)); - result.InsertValue("in_mem", Portion.GetColumnRawBytes(columns, false)); + if (GetStageData().HasPortionAccessor()) { + auto columns = GetStageData().GetPortionAccessor().GetColumnIds(); + // result.InsertValue("sequential_columns", JoinSeq(",", SequentialEntityIds)); + result.InsertValue("in_mem", GetStageData().GetPortionAccessor().GetColumnRawBytes(columns, false)); + result.InsertValue("columns_in_mem", JoinSeq(",", columns)); } - result.InsertValue("columns_in_mem", JoinSeq(",", columns)); - result.InsertValue("portion_id", Portion.GetPortionInfo().GetPortionId()); - result.InsertValue("raw", Portion.GetPortionInfo().GetTotalRawBytes()); - result.InsertValue("blob", Portion.GetPortionInfo().GetTotalBlobBytes()); - result.InsertValue("read_memory", GetColumnRawBytes(Portion.GetColumnIds())); + result.InsertValue("portion_id", Portion->GetPortionId()); + result.InsertValue("raw", Portion->GetTotalRawBytes()); + result.InsertValue("blob", Portion->GetTotalBlobBytes()); + result.InsertValue("read_memory", GetColumnRawBytes(GetStageData().GetPortionAccessor().GetColumnIds())); return result; } virtual void DoAbort() override; virtual ui64 GetPathId() const override { - return Portion.GetPortionInfo().GetPathId(); + return Portion->GetPathId(); } - virtual bool DoAddSequentialEntityIds(const ui32 entityId) override { - FingerprintedData.clear(); - return SequentialEntityIds.emplace(entityId).second; + + virtual bool DoStartFetchingAccessor(const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step) override; + virtual ui64 PredictAccessorMemoryBytes() const override { + return Portion->PredictMetadataMemorySize(Schema->GetColumnsCount()); } public: virtual bool DoAddTxConflict() override { - if (Portion.GetPortionInfo().HasCommitSnapshot() || !Portion.GetPortionInfo().HasInsertWriteId()) { + if (Portion->HasCommitSnapshot() || !Portion->HasInsertWriteId()) { GetContext()->GetReadMetadata()->SetBrokenWithCommitted(); return true; - } else if (!GetContext()->GetReadMetadata()->IsMyUncommitted(Portion.GetPortionInfo().GetInsertWriteIdVerified())) { - GetContext()->GetReadMetadata()->SetConflictedWriteId(Portion.GetPortionInfo().GetInsertWriteIdVerified()); + } else if (!GetContext()->GetReadMetadata()->IsMyUncommitted(Portion->GetInsertWriteIdVerified())) { + GetContext()->GetReadMetadata()->SetConflictedWriteId(Portion->GetInsertWriteIdVerified()); return true; } return false; } virtual bool HasIndexes(const std::set& indexIds) const override { - return Portion.HasIndexes(indexIds); + return Schema->GetIndexInfo().HasIndexes(indexIds); } virtual THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobsOriginal) const override { - return Portion.DecodeBlobAddresses(std::move(blobsOriginal), Schema->GetIndexInfo()); - } - - virtual bool IsSourceInMemory(const std::set& fieldIds) const override { - for (auto&& i : SequentialEntityIds) { - if (fieldIds.contains(i)) { - return false; - } + return GetStageData().GetPortionAccessor().DecodeBlobAddresses(std::move(blobsOriginal), Schema->GetIndexInfo()); + } + + virtual ui64 GetColumnsVolume(const std::set& columnIds, const EMemType type) const override { + AFL_VERIFY(columnIds.size()); + switch (type) { + case EMemType::Raw: + return GetStageData().GetPortionAccessor().GetColumnRawBytes(columnIds, false); + case EMemType::Blob: + return GetStageData().GetPortionAccessor().GetColumnBlobBytes(columnIds, false); + case EMemType::RawSequential: + return GetStageData().GetPortionAccessor().GetMinMemoryForReadColumns(columnIds); } - return true; } virtual ui64 GetColumnRawBytes(const std::set& columnsIds) const override { AFL_VERIFY(columnsIds.size()); - const ui64 fp = CombineHashes(*columnsIds.begin(), *columnsIds.rbegin()); - auto it = FingerprintedData.find(fp); - if (it != FingerprintedData.end()) { - return it->second; - } - ui64 result = 0; - if (SequentialEntityIds.size()) { - std::set selectedSeq; - std::set selectedInMem; - for (auto&& i : columnsIds) { - if (SequentialEntityIds.contains(i)) { - selectedSeq.emplace(i); - } else { - selectedInMem.emplace(i); - } - } - result = Portion.GetMinMemoryForReadColumns(selectedSeq) + - Portion.GetColumnBlobBytes(selectedSeq, false) + - Portion.GetColumnRawBytes(selectedInMem, false); - } else { - result = Portion.GetColumnRawBytes(columnsIds, false); - } - FingerprintedData.emplace(fp, result); - return result; + return GetStageData().GetPortionAccessor().GetColumnRawBytes(columnsIds, false); } virtual ui64 GetColumnBlobBytes(const std::set& columnsIds) const override { - return Portion.GetColumnBlobBytes(columnsIds, false); + return GetStageData().GetPortionAccessor().GetColumnBlobBytes(columnsIds, false); } virtual ui64 GetIndexRawBytes(const std::set& indexIds) const override { - return Portion.GetIndexRawBytes(indexIds, false); + return Portion->GetTotalRawBytes(); + return GetStageData().GetPortionAccessor().GetIndexRawBytes(indexIds, false); } const TPortionInfo& GetPortionInfo() const { - return Portion.GetPortionInfo(); + return *Portion; } const TPortionInfo::TConstPtr& GetPortionInfoPtr() const { - return Portion.GetPortionInfoPtr(); + return Portion; } TPortionDataSource(const ui32 sourceIdx, const std::shared_ptr& portion, const std::shared_ptr& context) : TBase(sourceIdx, context, portion->IndexKeyStart(), portion->IndexKeyEnd(), portion->RecordSnapshotMin(TSnapshot::Zero()), - portion->RecordSnapshotMax(TSnapshot::Zero()), - portion->GetRecordsCount(), portion->GetShardingVersionOptional(), portion->GetMeta().GetDeletionsCount()) + portion->RecordSnapshotMax(TSnapshot::Zero()), portion->GetRecordsCount(), portion->GetShardingVersionOptional(), + portion->GetMeta().GetDeletionsCount()) , Portion(portion) , Schema(GetContext()->GetReadMetadata()->GetLoadSchemaVerified(*portion)) { } @@ -415,7 +403,7 @@ class TCommittedDataSource: public IDataSource { return; } - virtual void DoAssembleColumns(const std::shared_ptr& columns) override; + virtual void DoAssembleColumns(const std::shared_ptr& columns, const bool sequential) override; virtual NJson::TJsonValue DoDebugJson() const override { NJson::TJsonValue result = NJson::JSON_MAP; result.InsertValue("type", "commit"); @@ -425,9 +413,6 @@ class TCommittedDataSource: public IDataSource { virtual ui64 GetPathId() const override { return 0; } - virtual bool DoAddSequentialEntityIds(const ui32 /*entityId*/) override { - return false; - } virtual bool DoAddTxConflict() override { if (CommittedBlob.IsCommitted()) { @@ -451,10 +436,6 @@ class TCommittedDataSource: public IDataSource { return result; } - virtual bool IsSourceInMemory(const std::set& /*fieldIds*/) const override { - return true; - } - virtual bool HasIndexes(const std::set& /*indexIds*/) const override { return false; } @@ -467,6 +448,25 @@ class TCommittedDataSource: public IDataSource { return CommittedBlob.GetBlobRange().Size; } + virtual bool DoStartFetchingAccessor(const std::shared_ptr& /*sourcePtr*/, const TFetchingScriptCursor& /*step*/) override { + return false; + } + virtual ui64 PredictAccessorMemoryBytes() const override { + return 0; + } + + virtual ui64 GetColumnsVolume(const std::set& columnIds, const EMemType type) const override { + AFL_VERIFY(columnIds.size()); + switch (type) { + case EMemType::Raw: + return GetColumnRawBytes(columnIds); + case EMemType::Blob: + return GetColumnBlobBytes(columnIds); + case EMemType::RawSequential: + return GetColumnRawBytes(columnIds); + } + } + virtual ui64 GetIndexRawBytes(const std::set& /*columnIds*/) const override { AFL_VERIFY(false); return 0; diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.cpp index c47dd37eacb6..85f12b65ba78 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.cpp @@ -1,5 +1,24 @@ #include "iterator.h" +#include namespace NKikimr::NOlap::NReader::NSysView::NAbstract { +TStatsIteratorBase::TStatsIteratorBase(const std::shared_ptr& context, const NTable::TScheme::TTableSchema& statsSchema) + : StatsSchema(statsSchema) + , Context(context) + , ReadMetadata(context->GetReadMetadataPtrVerifiedAs()) + , KeySchema(MakeArrowSchema(StatsSchema.Columns, StatsSchema.KeyColumns)) + , ResultSchema(MakeArrowSchema(StatsSchema.Columns, ReadMetadata->ResultColumnIds)) + , IndexGranules(ReadMetadata->IndexGranules) { + if (ResultSchema->num_fields() == 0) { + ResultSchema = KeySchema; + } + std::vector allColumnIds; + for (const auto& c : StatsSchema.Columns) { + allColumnIds.push_back(c.second.Id); + } + std::sort(allColumnIds.begin(), allColumnIds.end()); + DataSchema = MakeArrowSchema(StatsSchema.Columns, allColumnIds); } + +} // namespace NKikimr::NOlap::NReader::NSysView::NAbstract diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h index 25644835dc9d..72ffbdda7754 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/iterator.h @@ -14,13 +14,20 @@ class TStatsIteratorBase: public TScanIteratorBase { protected: virtual bool AppendStats(const std::vector>& builders, TGranuleMetaView& granule) const = 0; virtual ui32 PredictRecordsCount(const TGranuleMetaView& granule) const = 0; + std::shared_ptr Context; TReadStatsMetadata::TConstPtr ReadMetadata; const bool Reverse = false; std::shared_ptr KeySchema; std::shared_ptr ResultSchema; std::deque IndexGranules; + mutable THashMap FetchedAccessors; + public: + virtual bool IsReadyForBatch() const { + return true; + } + virtual TConclusionStatus Start() override { return TConclusionStatus::Success(); } @@ -31,6 +38,9 @@ class TStatsIteratorBase: public TScanIteratorBase { virtual TConclusion> GetBatch() override { while (!Finished()) { + if (!IsReadyForBatch()) { + return std::shared_ptr(); + } auto batchOpt = ExtractStatsBatch(); if (!batchOpt) { AFL_VERIFY(Finished()); @@ -88,23 +98,7 @@ class TStatsIteratorBase: public TScanIteratorBase { } - TStatsIteratorBase(const NAbstract::TReadStatsMetadata::TConstPtr& readMetadata, const NTable::TScheme::TTableSchema& statsSchema) - : StatsSchema(statsSchema) - , ReadMetadata(readMetadata) - , KeySchema(MakeArrowSchema(StatsSchema.Columns, StatsSchema.KeyColumns)) - , ResultSchema(MakeArrowSchema(StatsSchema.Columns, ReadMetadata->ResultColumnIds)) - , IndexGranules(ReadMetadata->IndexGranules) - { - if (ResultSchema->num_fields() == 0) { - ResultSchema = KeySchema; - } - std::vector allColumnIds; - for (const auto& c : StatsSchema.Columns) { - allColumnIds.push_back(c.second.Id); - } - std::sort(allColumnIds.begin(), allColumnIds.end()); - DataSchema = MakeArrowSchema(StatsSchema.Columns, allColumnIds); - } + TStatsIteratorBase(const std::shared_ptr& context, const NTable::TScheme::TTableSchema& statsSchema); }; template @@ -143,8 +137,8 @@ class TStatsIterator : public TStatsIteratorBase { } }; - TStatsIterator(const NAbstract::TReadStatsMetadata::TConstPtr& readMetadata) - : TBase(readMetadata, StatsSchema) + TStatsIterator(const std::shared_ptr& context) + : TBase(context, StatsSchema) { } diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 9cef7b0a6230..f08ef66110c1 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -1,11 +1,13 @@ #include "chunks.h" + #include #include namespace NKikimr::NOlap::NReader::NSysView::NChunks { -void TStatsIterator::AppendStats(const std::vector>& builders, const TPortionInfo::TConstPtr& portionPtr) const { - const TPortionInfo& portion = *portionPtr; +void TStatsIterator::AppendStats( + const std::vector>& builders, const TPortionDataAccessor& portionPtr) const { + const TPortionInfo& portion = portionPtr.GetPortionInfo(); auto portionSchema = ReadMetadata->GetLoadSchemaVerified(portion); auto it = PortionType.find(portion.GetMeta().Produced); if (it == PortionType.end()) { @@ -22,7 +24,7 @@ void TStatsIterator::AppendStats(const std::vector records; - for (auto&& r : TPortionDataAccessor(portionPtr).GetRecords()) { + for (auto&& r : portionPtr.GetRecords()) { records.emplace_back(&r); } if (Reverse) { @@ -65,8 +67,10 @@ void TStatsIterator::AppendStats(const std::vectorGetBlobRange().GetBlobIdxVerified()); if (itBlobIdString == blobsIds.end()) { - itBlobIdString = blobsIds.emplace( - r->GetBlobRange().GetBlobIdxVerified(), portion.GetBlobId(r->GetBlobRange().GetBlobIdxVerified()).ToStringLegacy()).first; + itBlobIdString = blobsIds + .emplace(r->GetBlobRange().GetBlobIdxVerified(), + portion.GetBlobId(r->GetBlobRange().GetBlobIdxVerified()).ToStringLegacy()) + .first; } NArrow::Append( *builders[9], arrow::util::string_view(itBlobIdString->second.data(), itBlobIdString->second.size())); @@ -81,7 +85,7 @@ void TStatsIterator::AppendStats(const std::vector indexes; - for (auto&& r : TPortionDataAccessor(portionPtr).GetIndexes()) { + for (auto&& r : portionPtr.GetIndexes()) { indexes.emplace_back(&r); } if (Reverse) { @@ -117,25 +121,32 @@ void TStatsIterator::AppendStats(const std::vector TReadStatsMetadata::StartScan(const std::shared_ptr& readContext) const { - return std::make_unique(readContext->GetReadMetadataPtrVerifiedAs()); + return std::make_unique(readContext); } std::vector> TReadStatsMetadata::GetKeyYqlSchema() const { return GetColumns(TStatsIterator::StatsSchema, TStatsIterator::StatsSchema.KeyColumns); } -std::shared_ptr TConstructor::BuildMetadata(const NColumnShard::TColumnShard* self, const TReadDescription& read) const { +std::shared_ptr TConstructor::BuildMetadata( + const NColumnShard::TColumnShard* self, const TReadDescription& read) const { auto* index = self->GetIndexOptional(); return std::make_shared(index ? index->CopyVersionedIndexPtr() : nullptr, self->TabletID(), - IsReverse ? TReadMetadataBase::ESorting::DESC : TReadMetadataBase::ESorting::ASC, - read.GetProgram(), index ? index->GetVersionedIndex().GetLastSchema() : nullptr, read.GetSnapshot()); + IsReverse ? TReadMetadataBase::ESorting::DESC : TReadMetadataBase::ESorting::ASC, read.GetProgram(), + index ? index->GetVersionedIndex().GetLastSchema() : nullptr, read.GetSnapshot()); } bool TStatsIterator::AppendStats(const std::vector>& builders, NAbstract::TGranuleMetaView& granule) const { ui64 recordsCount = 0; - while (auto portion = granule.PopFrontPortion()) { - recordsCount += TPortionDataAccessor(portion).GetRecords().size() + TPortionDataAccessor(portion).GetIndexes().size(); - AppendStats(builders, portion); + while (granule.GetPortions().size()) { + auto it = FetchedAccessors.find(granule.GetPortions().front()->GetPortionId()); + if (it == FetchedAccessors.end()) { + break; + } + recordsCount += it->second.GetRecords().size() + it->second.GetIndexes().size(); + AppendStats(builders, it->second); + granule.PopFrontPortion(); + FetchedAccessors.erase(it); if (recordsCount > 10000) { break; } @@ -146,12 +157,45 @@ bool TStatsIterator::AppendStats(const std::vectorGetPortionId()); + if (it == FetchedAccessors.end()) { + break; + } + recordsCount += it->second.GetRecords().size() + it->second.GetIndexes().size(); if (recordsCount > 10000) { break; } } + AFL_VERIFY(recordsCount); return recordsCount; } +TConclusionStatus TStatsIterator::Start() { + ProcessGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(ReadMetadata->GetTxId(), {}); + ScopeGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildScopeGuard(ReadMetadata->GetTxId(), 1); + const ui32 columnsCount = ReadMetadata->GetKeyYqlSchema().size(); + for (auto&& i : IndexGranules) { + GroupGuards.emplace_back(NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildGroupGuard(ReadMetadata->GetTxId(), 1)); + for (auto&& p : i.GetPortions()) { + std::shared_ptr request = std::make_shared(); + request->AddPortion(p); + auto allocation = std::make_shared(request, p->PredictMetadataMemorySize(columnsCount), Context); + request->RegisterSubscriber(allocation); + + NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation( + ProcessGuard->GetProcessId(), ScopeGuard->GetScopeId(), GroupGuards.back()->GetGroupId(), { allocation }, std::nullopt); + } + } + return TConclusionStatus::Success(); } + +TStatsIterator::TFetchingAccessorAllocation::TFetchingAccessorAllocation( + const std::shared_ptr& request, const ui64 mem, const std::shared_ptr& context) + : TBase(mem) + , AccessorsManager(context->GetDataAccessorsManager()) + , Request(request) + , WaitingCountersGuard(context->GetCounters().GetFetcherAcessorsGuard()) + , OwnerId(context->GetScanActorId()) { +} + +} // namespace NKikimr::NOlap::NReader::NSysView::NChunks diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h index 548e61cff624..1022f8b3f76c 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h @@ -1,24 +1,29 @@ #pragma once +#include #include #include #include -#include +#include namespace NKikimr::NOlap::NReader::NSysView::NChunks { class TConstructor: public TStatScannerConstructor { private: using TBase = TStatScannerConstructor; + protected: - virtual std::shared_ptr BuildMetadata(const NColumnShard::TColumnShard* self, const TReadDescription& read) const override; + virtual std::shared_ptr BuildMetadata( + const NColumnShard::TColumnShard* self, const TReadDescription& read) const override; + public: using TBase::TBase; }; -class TReadStatsMetadata: public NAbstract::TReadStatsMetadata, std::enable_shared_from_this { +class TReadStatsMetadata: public NAbstract::TReadStatsMetadata { private: using TBase = NAbstract::TReadStatsMetadata; using TSysViewSchema = NKikimr::NSysView::Schema::PrimaryIndexStats; + public: using TBase::TBase; @@ -53,39 +58,124 @@ class TStatsIterator: public NAbstract::TStatsIterator ColumnNamesById; mutable THashMap PortionType; mutable THashMap> EntityStorageNames; + std::shared_ptr ProcessGuard; + std::shared_ptr ScopeGuard; + std::vector> GroupGuards; using TBase = NAbstract::TStatsIterator; - virtual bool AppendStats(const std::vector>& builders, NAbstract::TGranuleMetaView& granule) const override; + + virtual bool IsReadyForBatch() const override { + return IndexGranules.size() && IndexGranules.front().GetPortions().size() && + FetchedAccessors.contains(IndexGranules.front().GetPortions().front()->GetPortionId()); + } + + virtual bool AppendStats( + const std::vector>& builders, NAbstract::TGranuleMetaView& granule) const override; virtual ui32 PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const override; - void AppendStats(const std::vector>& builders, const TPortionInfo::TConstPtr& portion) const; + void AppendStats(const std::vector>& builders, const TPortionDataAccessor& portion) const; + + class TApplyResult: public IDataTasksProcessor::ITask { + private: + using TBase = IDataTasksProcessor::ITask; + YDB_READONLY_DEF(std::vector, Accessors); + NColumnShard::TCounterGuard WaitingCountersGuard; + public: + TString GetTaskClassIdentifier() const override { + return "TApplyResult"; + } + + TApplyResult(const std::vector& accessors, NColumnShard::TCounterGuard&& waitingCountersGuard) + : TBase(NActors::TActorId()) + , Accessors(accessors) + , WaitingCountersGuard(std::move(waitingCountersGuard)) + { + } + + virtual TConclusionStatus DoExecuteImpl() override { + AFL_VERIFY(false)("event", "not applicable"); + return TConclusionStatus::Success(); + } + virtual bool DoApply(IDataReader& /*indexedDataRead*/) const override { + AFL_VERIFY(false); + return false; + } + }; + + class TFetchingAccessorAllocation: public NGroupedMemoryManager::IAllocation, public IDataAccessorRequestsSubscriber { + private: + using TBase = NGroupedMemoryManager::IAllocation; + std::shared_ptr Guard; + std::shared_ptr AccessorsManager; + std::shared_ptr Request; + NColumnShard::TCounterGuard WaitingCountersGuard; + const NActors::TActorId OwnerId; + virtual bool DoOnAllocated(std::shared_ptr&& guard, + const std::shared_ptr& /*selfPtr*/) override { + Guard = std::move(guard); + AccessorsManager->AskData(std::move(Request)); + return true; + } + + virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { + if (result.HasErrors()) { + NActors::TActivationContext::AsActorContext().Send( + OwnerId, new NColumnShard::TEvPrivate::TEvTaskProcessedResult(TConclusionStatus::Fail("cannot fetch accessors"))); + } else { + AFL_VERIFY(result.GetPortions().size() == 1)("count", result.GetPortions().size()); + NActors::TActivationContext::AsActorContext().Send( + OwnerId, new NColumnShard::TEvPrivate::TEvTaskProcessedResult( + std::make_shared(result.GetPortions(), std::move(WaitingCountersGuard)))); + } + } + + public: + TFetchingAccessorAllocation(const std::shared_ptr& request, const ui64 mem, const std::shared_ptr& context); + }; + + virtual void Apply(const std::shared_ptr& task) override { + if (IndexGranules.empty()) { + return; + } + auto result = std::dynamic_pointer_cast(task); + AFL_VERIFY(result); + AFL_VERIFY(result->GetAccessors().size() == 1); + FetchedAccessors.emplace(result->GetAccessors().front().GetPortionInfo().GetPortionId(), result->GetAccessors().front()); + } + + virtual TConclusionStatus Start() override; + public: using TBase::TBase; }; class TStoreSysViewPolicy: public NAbstract::ISysViewPolicy { protected: - virtual std::unique_ptr DoCreateConstructor(const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override { + virtual std::unique_ptr DoCreateConstructor( + const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override { return std::make_unique(snapshot, itemsLimit, reverse); } virtual std::shared_ptr DoCreateMetadataFiller() const override { return std::make_shared(); } -public: - static const inline TFactory::TRegistrator Registrator = TFactory::TRegistrator(TString(::NKikimr::NSysView::StorePrimaryIndexStatsName)); +public: + static const inline TFactory::TRegistrator Registrator = + TFactory::TRegistrator(TString(::NKikimr::NSysView::StorePrimaryIndexStatsName)); }; class TTableSysViewPolicy: public NAbstract::ISysViewPolicy { protected: - virtual std::unique_ptr DoCreateConstructor(const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override { + virtual std::unique_ptr DoCreateConstructor( + const TSnapshot& snapshot, const ui64 itemsLimit, const bool reverse) const override { return std::make_unique(snapshot, itemsLimit, reverse); } virtual std::shared_ptr DoCreateMetadataFiller() const override { return std::make_shared(); } -public: - static const inline TFactory::TRegistrator Registrator = TFactory::TRegistrator(TString(::NKikimr::NSysView::TablePrimaryIndexStatsName)); +public: + static const inline TFactory::TRegistrator Registrator = + TFactory::TRegistrator(TString(::NKikimr::NSysView::TablePrimaryIndexStatsName)); }; -} +} // namespace NKikimr::NOlap::NReader::NSysView::NChunks diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp index 4ab8e7ad2bbc..f6b1424927b4 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp @@ -16,7 +16,7 @@ bool TStatsIterator::AppendStats(const std::vector TReadStatsMetadata::StartScan(const std::shared_ptr& readContext) const { - return std::make_unique(readContext->GetReadMetadataPtrVerifiedAs()); + return std::make_unique(readContext); } std::vector> TReadStatsMetadata::GetKeyYqlSchema() const { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp index dc88fd26abde..bbeb039f7910 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp @@ -23,7 +23,7 @@ bool TStatsIterator::AppendStats(const std::vector TReadStatsMetadata::StartScan(const std::shared_ptr& readContext) const { - return std::make_unique(readContext->GetReadMetadataPtrVerifiedAs()); + return std::make_unique(readContext); } std::vector> TReadStatsMetadata::GetKeyYqlSchema() const { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp index ea4fafa737cf..a1c4f5a50376 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp @@ -53,7 +53,7 @@ bool TStatsIterator::AppendStats(const std::vector TReadStatsMetadata::StartScan(const std::shared_ptr& readContext) const { - return std::make_unique(readContext->GetReadMetadataPtrVerifiedAs()); + return std::make_unique(readContext); } std::vector> TReadStatsMetadata::GetKeyYqlSchema() const { diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp index 55d28a5a61f4..17b7c8497fbd 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp @@ -80,10 +80,9 @@ void TTxInternalScan::Complete(const TActorContext& ctx) { readMetadataRange->OnBeforeStartReading(*Self); const ui64 requestCookie = Self->InFlightReadsTracker.AddInFlightRequest(readMetadataRange, index); - auto scanActor = ctx.Register(new TColumnShardScan(Self->SelfId(), scanComputeActor, Self->GetStoragesManager(), TComputeShardingPolicy(), - ScanId, LockId.value_or(0), ScanGen, requestCookie, Self->TabletID(), TDuration::Max(), readMetadataRange, - NKikimrDataEvents::FORMAT_ARROW, - Self->Counters.GetScanCounters())); + auto scanActor = ctx.Register(new TColumnShardScan(Self->SelfId(), scanComputeActor, Self->GetStoragesManager(), Self->DataAccessorsManager.GetObjectPtrVerified(), + TComputeShardingPolicy(), ScanId, LockId.value_or(0), ScanGen, requestCookie, Self->TabletID(), TDuration::Max(), readMetadataRange, + NKikimrDataEvents::FORMAT_ARROW, Self->Counters.GetScanCounters())); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "TTxInternalScan started")("actor_id", scanActor)("trace_detailed", detailedInfo); } diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp index 74f09deb0197..d3c1aa0f47ff 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp @@ -125,7 +125,7 @@ void TTxScan::Complete(const TActorContext& ctx) { TComputeShardingPolicy shardingPolicy; AFL_VERIFY(shardingPolicy.DeserializeFromProto(request.GetComputeShardingPolicy())); - auto scanActor = ctx.Register(new TColumnShardScan(Self->SelfId(), scanComputeActor, Self->GetStoragesManager(), shardingPolicy, scanId, + auto scanActor = ctx.Register(new TColumnShardScan(Self->SelfId(), scanComputeActor, Self->GetStoragesManager(), Self->DataAccessorsManager.GetObjectPtrVerified(), shardingPolicy, scanId, txId, scanGen, requestCookie, Self->TabletID(), timeout, readMetadataRange, dataFormat, Self->Counters.GetScanCounters())); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "TTxScan started")("actor_id", scanActor)("trace_detailed", detailedInfo); diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index b9004b743b5a..3a78cd08782e 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -288,6 +288,15 @@ struct TIndexInfo: public IIndexInfo { return Indexes.contains(indexId); } + bool HasIndexes(const std::set& indexIds) const { + for (auto&& i : indexIds) { + if (!Indexes.contains(i)) { + return false; + } + } + return true; + } + std::optional GetColumnIndexOptional(const ui32 id) const; ui32 GetColumnIndexVerified(const ui32 id) const { auto result = GetColumnIndexOptional(id); diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index 0601a9983dea..bbcb2f1ee35a 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -123,7 +123,7 @@ void TTieringActualizer::DoExtractTasks(TTieringProcessContext& tasksContext, co for (auto&& p : portions) { auto portion = externalContext.GetPortionVerified(p); if (!address.WriteIs(NBlobOperations::TGlobal::DefaultStorageId) && !address.WriteIs(NTiering::NCommon::DeleteTierName)) { - if (!portion->HasRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized) || portion->HasInsertWriteId()) { + if (!NYDBTest::TControllers::GetColumnShardController()->CheckPortionForEvict(portion)) { Counters.SkipEvictionForCompaction->Add(1); continue; } diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp index 5aa46a01de1f..fa841b1132e4 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp @@ -1,9 +1,11 @@ #include "granule.h" +#include "stages.h" #include "storage.h" #include #include #include +#include #include @@ -30,6 +32,7 @@ bool TGranuleMeta::ErasePortion(const ui64 portion) { } else { AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "portion_erased")("portion_info", it->second->DebugString())("pathId", PathId); } + DataAccessorsManager->RemovePortion(it->second); OnBeforeChangePortion(it->second); Portions.erase(it); OnAfterChangePortion(nullptr, nullptr); @@ -121,6 +124,7 @@ const NKikimr::NOlap::TGranuleAdditiveSummary& TGranuleMeta::GetAdditiveSummary( TGranuleMeta::TGranuleMeta( const ui64 pathId, const TGranulesStorage& owner, const NColumnShard::TGranuleDataCounters& counters, const TVersionedIndex& versionedIndex) : PathId(pathId) + , DataAccessorsManager(owner.GetDataAccessorsManager()) , Counters(counters) , PortionInfoGuard(owner.GetCounters().BuildPortionBlobsGuard()) , Stats(owner.GetStats()) @@ -184,10 +188,64 @@ void TGranuleMeta::CommitImmediateOnExecute( portion.MutablePortionInfo().SetCommitSnapshot(snapshot); TDbWrapper wrapper(txc.DB, nullptr); portion.SaveToDatabase(wrapper, 0, false); + DataAccessorsManager->AddPortion(portion); } void TGranuleMeta::CommitImmediateOnComplete(const std::shared_ptr portion, IColumnEngine& engine) { (static_cast(engine)).AppendPortion(portion); } +std::shared_ptr TGranuleMeta::BuildLoader( + const std::shared_ptr& dsGroupSelector, const TVersionedIndex& vIndex) { + auto result = std::make_shared("granule"); + auto portionsLoadContext = std::make_shared(); + result->AddChildren(std::make_shared("portions", &vIndex, this, dsGroupSelector, portionsLoadContext)); + result->AddChildren(std::make_shared("columns", &vIndex, this, dsGroupSelector, portionsLoadContext)); + result->AddChildren(std::make_shared("indexes", &vIndex, this, dsGroupSelector, portionsLoadContext)); + result->AddChildren(std::make_shared("finish", &vIndex, this, dsGroupSelector, portionsLoadContext)); + return result; +} + +bool TGranuleMeta::TestingLoad(IDbWrapper& db, const TVersionedIndex& versionedIndex) { + auto constructors = std::make_shared(); + { + if (!db.LoadPortions(PathId, [&](TPortionInfoConstructor&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + const TIndexInfo& indexInfo = portion.GetSchema(versionedIndex)->GetIndexInfo(); + AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo, db.GetDsGroupSelectorVerified())); + AFL_VERIFY(constructors->MutableConstructors().AddConstructorVerified(std::move(portion))); + })) { + return false; + } + } + + { + TPortionInfo::TSchemaCursor schema(versionedIndex); + if (!db.LoadColumns(PathId, [&](const TColumnChunkLoadContextV1& loadContext) { + auto* constructor = constructors->MutableConstructors().GetConstructorVerified(loadContext.GetPortionId()); + constructor->LoadRecord(loadContext); + })) { + return false; + } + } + + { + if (!db.LoadIndexes(PathId, [&](const ui64 /*pathId*/, const ui64 portionId, const TIndexChunkLoadContext& loadContext) { + auto* constructor = constructors->MutableConstructors().GetConstructorVerified(portionId); + constructor->LoadIndex(loadContext); + })) { + return false; + }; + } + FinishLoading(constructors); + return true; +} + +void TGranuleMeta::FinishLoading(const std::shared_ptr& context) { + AFL_VERIFY(!LoadingFinished); + LoadingFinished = true; + for (auto&& [portionId, constructor] : context->MutableConstructors()) { + UpsertPortionOnLoad(constructor.Build(false).MutablePortionInfoPtr()); + } +} + } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.h b/ydb/core/tx/columnshard/engines/storage/granule/granule.h index ab6a5406a325..cc4623c1a616 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -12,9 +14,14 @@ namespace NKikimr::NOlap { +namespace NLoading { +class TPortionsLoadContext; +} + class TGranulesStorage; class TGranulesStat; class TColumnChunkLoadContext; +class TVersionedIndex; class TDataClassSummary: public NColumnShard::TBaseGranuleDataClassSummary { private: @@ -118,6 +125,7 @@ class TGranuleMeta: TNonCopyable { mutable bool AllowInsertionFlag = false; const ui64 PathId; + std::shared_ptr DataAccessorsManager; const NColumnShard::TGranuleDataCounters Counters; NColumnShard::TEngineLogsCounters::TPortionsInfoGuard PortionInfoGuard; std::shared_ptr Stats; @@ -148,8 +156,20 @@ class TGranuleMeta: TNonCopyable { } return it->second; } + bool DataAccessorConstructed = false; + bool LoadingFinished = false; public: + std::shared_ptr BuildLoader(const std::shared_ptr& dsGroupSelector, const TVersionedIndex& vIndex); + void FinishLoading(const std::shared_ptr& context); + bool TestingLoad(IDbWrapper& db, const TVersionedIndex& versionedIndex); + + std::unique_ptr BuildDataAccessor() { + AFL_VERIFY(!DataAccessorConstructed); + DataAccessorConstructed = true; + return std::make_unique(PathId); + } + void RefreshTiering(const std::optional& tiering) { NActualizer::TAddExternalContext context(HasAppData() ? AppDataVerified().TimeProvider->Now() : TInstant::Now(), Portions); ActualizationIndex->RefreshTiering(tiering, context); @@ -157,12 +177,17 @@ class TGranuleMeta: TNonCopyable { template void ModifyPortionOnExecute( - IDbWrapper& wrapper, const TPortionInfo::TConstPtr& portion, const TModifier& modifier, const ui32 firstPKColumnId) const { - const auto innerPortion = GetInnerPortion(portion).DetachResult(); - AFL_VERIFY((ui64)innerPortion.get() == (ui64)portion.get()); + IDbWrapper& wrapper, const TPortionDataAccessor& portion, const TModifier& modifier, const ui32 firstPKColumnId) const { + const auto innerPortion = GetInnerPortion(portion.GetPortionInfoPtr()).DetachResult(); + AFL_VERIFY((ui64)innerPortion.get() == (ui64)&portion.GetPortionInfo()); auto copy = innerPortion->MakeCopy(); modifier(copy); - TPortionDataAccessor(std::make_shared(std::move(copy))).SaveToDatabase(wrapper, firstPKColumnId, false); + if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { + auto accessorCopy = portion.SwitchPortionInfo(std::move(copy)); + accessorCopy.SaveToDatabase(wrapper, firstPKColumnId, false); + } else { + copy.SaveMetaToDatabase(wrapper); + } } template @@ -178,6 +203,7 @@ class TGranuleMeta: TNonCopyable { AFL_VERIFY(!InsertedPortions.contains(portion.GetPortionInfo().GetInsertWriteIdVerified())); TDbWrapper wrapper(txc.DB, nullptr); portion.SaveToDatabase(wrapper, 0, false); + DataAccessorsManager->AddPortion(portion); } void InsertPortionOnComplete(const std::shared_ptr& portion) { @@ -190,7 +216,7 @@ class TGranuleMeta: TNonCopyable { AFL_VERIFY(it != InsertedPortions.end()); it->second->SetCommitSnapshot(snapshot); TDbWrapper wrapper(txc.DB, nullptr); - TPortionDataAccessor(it->second).SaveToDatabase(wrapper, 0, true); + it->second->SaveMetaToDatabase(wrapper); } void CommitPortionOnComplete(const TInsertWriteId insertWriteId, IColumnEngine& engine); @@ -198,12 +224,14 @@ class TGranuleMeta: TNonCopyable { void AbortPortionOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TInsertWriteId insertWriteId) const { auto it = InsertedPortions.find(insertWriteId); AFL_VERIFY(it != InsertedPortions.end()); + it->second->SetCommitSnapshot(TSnapshot(1, 1)); + it->second->SetRemoveSnapshot(TSnapshot(1, 2)); TDbWrapper wrapper(txc.DB, nullptr); - TPortionDataAccessor(it->second).RemoveFromDatabase(wrapper); + it->second->SaveMetaToDatabase(wrapper); } - void AbortPortionOnComplete(const TInsertWriteId insertWriteId) { - AFL_VERIFY(InsertedPortions.erase(insertWriteId)); + void AbortPortionOnComplete(const TInsertWriteId insertWriteId, IColumnEngine& engine) { + CommitPortionOnComplete(insertWriteId, engine); } void CommitImmediateOnExecute( diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h index b51cfc0dfd6d..d71f9ae79b7a 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h @@ -18,7 +18,7 @@ class TPortionInfoStat { public: TPortionInfoStat(const TPortionInfo::TConstPtr& portionInfo) : PortionInfo(portionInfo) - , MinRawBytes(TPortionDataAccessor(PortionInfo).GetMinMemoryForReadColumns({})) + , MinRawBytes(PortionInfo->GetTotalBlobBytes()) , BlobBytes(PortionInfo->GetTotalBlobBytes()) { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp b/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp new file mode 100644 index 000000000000..7cab9eec3642 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp @@ -0,0 +1,60 @@ +#include "granule.h" +#include "stages.h" + +#include +#include + +namespace NKikimr::NOlap::NLoading { + +bool TGranulePortionsReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + TDbWrapper db(txc.DB, &*DsGroupSelector); + const TVersionedIndex& index = *VersionedIndex; + Context->MutableConstructors().ClearPortions(); + return db.LoadPortions(Self->GetPathId(), [&](TPortionInfoConstructor&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + const TIndexInfo& indexInfo = portion.GetSchema(index)->GetIndexInfo(); + AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo, db.GetDsGroupSelectorVerified())); + AFL_VERIFY(Context->MutableConstructors().AddConstructorVerified(std::move(portion))); + }); + return true; +} + +bool TGranulePortionsReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return db.Table().Prefix(Self->GetPathId()).Select().IsReady(); +} + +bool TGranuleColumnsReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + TDbWrapper db(txc.DB, &*DsGroupSelector); + TPortionInfo::TSchemaCursor schema(*VersionedIndex); + Context->MutableConstructors().ClearColumns(); + return db.LoadColumns(Self->GetPathId(), [&](const TColumnChunkLoadContextV1& loadContext) { + auto* constructor = Context->MutableConstructors().GetConstructorVerified(loadContext.GetPortionId()); + constructor->LoadRecord(loadContext); + }); +} + +bool TGranuleColumnsReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return db.Table().Prefix(Self->GetPathId()).Select().IsReady(); +} + +bool TGranuleIndexesReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + TDbWrapper db(txc.DB, &*DsGroupSelector); + Context->MutableConstructors().ClearIndexes(); + return db.LoadIndexes(Self->GetPathId(), [&](const ui64 /*pathId*/, const ui64 portionId, const TIndexChunkLoadContext& loadContext) { + auto* constructor = Context->MutableConstructors().GetConstructorVerified(portionId); + constructor->LoadIndex(loadContext); + }); +} + +bool TGranuleIndexesReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return db.Table().Prefix(Self->GetPathId()).Select().IsReady(); +} + +bool TGranuleFinishLoading::DoExecute(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) { + Self->FinishLoading(Context); + return true; +} + +} // namespace NKikimr::NOlap::NLoading diff --git a/ydb/core/tx/columnshard/engines/storage/granule/stages.h b/ydb/core/tx/columnshard/engines/storage/granule/stages.h new file mode 100644 index 000000000000..d78717d7b7e8 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/granule/stages.h @@ -0,0 +1,90 @@ +#pragma once +#include +#include + +namespace NKikimr::NOlap { +class TGranuleMeta; +} + +namespace NKikimr::NOlap::NLoading { + +class TPortionsLoadContext { +private: + TInGranuleConstructors Constructors; + +public: + TInGranuleConstructors& MutableConstructors() { + return Constructors; + } + const TInGranuleConstructors& GetConstructors() const { + return Constructors; + } +}; + +class IGranuleTxReader: public ITxReader { +private: + using TBase = ITxReader; + +protected: + const std::shared_ptr DsGroupSelector; + TGranuleMeta* Self = nullptr; + const TVersionedIndex* VersionedIndex; + const std::shared_ptr Context; + +public: + IGranuleTxReader(const TString& name, const TVersionedIndex* versionedIndex, TGranuleMeta* self, const std::shared_ptr& dsGroupSelector, + const std::shared_ptr& context) + : TBase(name) + , DsGroupSelector(dsGroupSelector) + , Self(self) + , VersionedIndex(versionedIndex) + , Context(context) { + } +}; + +class TGranulePortionsReader: public IGranuleTxReader { +private: + using TBase = IGranuleTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TGranuleColumnsReader: public IGranuleTxReader { +private: + using TBase = IGranuleTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TGranuleIndexesReader: public IGranuleTxReader { +private: + using TBase = IGranuleTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TGranuleFinishLoading: public IGranuleTxReader { +private: + using TBase = IGranuleTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + +public: + using TBase::TBase; +}; + +} // namespace NKikimr::NColumnShard::NLoading diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.h b/ydb/core/tx/columnshard/engines/storage/granule/storage.h index 021ae1829d7c..47b1e463f4bd 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.h @@ -1,8 +1,11 @@ #pragma once #include "granule.h" -#include + #include #include +#include +#include +#include namespace NKikimr::NOlap { @@ -31,9 +34,7 @@ class TGranulesStat { public: TGranulesStat(const NColumnShard::TEngineLogsCounters& counters) - : Counters(counters) - { - + : Counters(counters) { } const NColumnShard::TEngineLogsCounters& GetCounters() const { @@ -43,6 +44,7 @@ class TGranulesStat { class TModificationGuard: TNonCopyable { private: TGranulesStat& Owner; + public: TModificationGuard(TGranulesStat& storage) : Owner(storage) { @@ -89,31 +91,43 @@ class TGranulesStat { const i64 value = SumMetadataMemoryPortionsSize.Add(portion.GetMetadataMemorySize()); Counters.OnIndexMetadataUsageBytes(value); } - }; class TGranulesStorage { private: const NColumnShard::TEngineLogsCounters Counters; + const std::shared_ptr DataAccessorsManager; std::shared_ptr StoragesManager; - THashMap> Tables; // pathId into Granule that equal to Table + THashMap> Tables; // pathId into Granule that equal to Table std::shared_ptr Stats; + public: - TGranulesStorage(const NColumnShard::TEngineLogsCounters counters, const std::shared_ptr& storagesManager) + const std::shared_ptr& GetDataAccessorsManager() const { + return DataAccessorsManager; + } + + TGranulesStorage(const NColumnShard::TEngineLogsCounters counters, + const std::shared_ptr& dataAccessorsManager, + const std::shared_ptr& storagesManager) : Counters(counters) + , DataAccessorsManager(dataAccessorsManager) , StoragesManager(storagesManager) - , Stats(std::make_shared(Counters)) - { + , Stats(std::make_shared(Counters)) { + } + void FetchDataAccessors(const std::shared_ptr& request) const { + DataAccessorsManager->AskData(request); } const std::shared_ptr& GetStats() const { return Stats; } - std::shared_ptr RegisterTable(const ui64 pathId, const NColumnShard::TGranuleDataCounters& counters, const TVersionedIndex& versionedIndex) { + std::shared_ptr RegisterTable( + const ui64 pathId, const NColumnShard::TGranuleDataCounters& counters, const TVersionedIndex& versionedIndex) { auto infoEmplace = Tables.emplace(pathId, std::make_shared(pathId, *this, counters, versionedIndex)); AFL_VERIFY(infoEmplace.second); + DataAccessorsManager->RegisterController(infoEmplace.first->second->BuildDataAccessor()); return infoEmplace.first->second; } @@ -125,6 +139,7 @@ class TGranulesStorage { if (!it->second->IsErasable()) { return false; } + DataAccessorsManager->UnregisterController(pathId); Tables.erase(it); return true; } @@ -171,6 +186,12 @@ class TGranulesStorage { return it->second; } + std::shared_ptr GetGranuleVerified(const ui64 pathId) const { + auto it = Tables.find(pathId); + AFL_VERIFY(it != Tables.end()); + return it->second; + } + const std::shared_ptr& GetStoragesManager() const { return StoragesManager; } @@ -185,4 +206,4 @@ class TGranulesStorage { std::shared_ptr* granuleResult = nullptr) const; }; -} // namespace NKikimr::NOlap +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/granule/ya.make b/ydb/core/tx/columnshard/engines/storage/granule/ya.make index 3326bfb5c64f..533afbec74c0 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/ya.make +++ b/ydb/core/tx/columnshard/engines/storage/granule/ya.make @@ -4,6 +4,7 @@ SRCS( granule.cpp storage.cpp portions_index.cpp + stages.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h index c9e81e47abf4..4d6559413445 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h @@ -894,11 +894,7 @@ class TPortionsBucket: public TMoveOnly { ("count", portions.size())("info", Others.DebugString())("event", "start_optimization")("stop_point", stopPoint ? stopPoint->DebugString() : "") ("main_portion", MainPortion ? MainPortion->GetPortionId() : 0); TSaverContext saverContext(storagesManager); - std::vector accessors; - for (auto&& i : portions) { - accessors.emplace_back(i); - } - auto result = std::make_shared(granule, accessors, saverContext); + auto result = std::make_shared(granule, portions, saverContext); if (MainPortion) { NArrow::NMerger::TSortableBatchPosition pos(MainPortion->IndexKeyStart().ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); result->AddCheckPoint(pos, false); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp index 81900b91c3e0..a5ffa1c95df7 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp @@ -34,11 +34,8 @@ std::shared_ptr TOptimizerPlanner::DoGetOptimizationTask( TSaverContext saverContext(StoragesManager); std::shared_ptr result; // if (level->GetLevelId() == 0) { - std::vector accessors; - for (auto&& i : data.GetRepackPortions(level->GetLevelId())) { - accessors.emplace_back(i); - } - result = std::make_shared(granule, accessors, saverContext); + result = + std::make_shared(granule, data.GetRepackPortions(level->GetLevelId()), saverContext); // } else { // result = std::make_shared( // granule, data.GetRepackPortions(level->GetLevelId()), saverContext); @@ -53,8 +50,8 @@ std::shared_ptr TOptimizerPlanner::DoGetOptimizationTask( AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("task_id", result->GetTaskIdentifier())("positions", positions.DebugString())( "level", level->GetLevelId())("target", data.GetTargetCompactionLevel())("data", data.DebugString()); result->SetCheckPoints(std::move(positions)); - for (auto&& i : result->SwitchedPortions) { - AFL_VERIFY(!locksManager->IsLocked(i.GetPortionInfo())); + for (auto&& i : result->GetSwitchedPortions()) { + AFL_VERIFY(!locksManager->IsLocked(i)); } return result; } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp index 39d4e103433f..e851c284bfc4 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp @@ -29,11 +29,7 @@ std::shared_ptr TPortionsBucket::BuildOptimizationTask(std AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("size", size)("next", Finish.DebugString())("count", context.GetPortions().size())( "event", "start_optimization"); TSaverContext saverContext(storagesManager); - std::vector accessors; - for (auto&& i : context.GetPortions()) { - accessors.emplace_back(i); - } - auto result = std::make_shared(granule, accessors, saverContext); + auto result = std::make_shared(granule, context.GetPortions(), saverContext); for (auto&& i : context.GetSplitRightOpenIntervalPoints()) { NArrow::NMerger::TSortableBatchPosition pos(i.ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); result->AddCheckPoint(pos, false); diff --git a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp index b8834cbfcad7..1847f05e6eff 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp @@ -46,7 +46,8 @@ class TTestInsertTableDB : public IDbWrapper { } virtual void ErasePortion(const NOlap::TPortionInfo& /*portion*/) override { } - virtual bool LoadPortions(const std::function& /*callback*/) override { + virtual bool LoadPortions(const std::optional /*reqPathId*/, + const std::function& /*callback*/) override { return true; } @@ -54,7 +55,7 @@ class TTestInsertTableDB : public IDbWrapper { } void EraseColumn(const TPortionInfo&, const TColumnRecord&) override { } - bool LoadColumns(const std::function&) override { + bool LoadColumns(const std::optional /*reqPathId*/, const std::function&) override { return true; } @@ -62,7 +63,8 @@ class TTestInsertTableDB : public IDbWrapper { } virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { } - virtual bool LoadIndexes(const std::function& /*callback*/) override { + virtual bool LoadIndexes(const std::optional /*reqPathId*/, + const std::function& /*callback*/) override { return true; } diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index a0a17f51cc09..a12ebe58e692 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -33,16 +33,17 @@ namespace { std::shared_ptr EmptyDataLocksManager = std::make_shared(); -class TTestDbWrapper : public IDbWrapper { +class TTestDbWrapper: public IDbWrapper { private: std::map> LoadContexts; + public: virtual const IBlobGroupSelector* GetDsGroupSelector() const override { return &Default(); } struct TIndex { - THashMap> Columns; // pathId -> portions + THashMap> Columns; // pathId -> portions THashMap Counters; }; @@ -75,8 +76,7 @@ class TTestDbWrapper : public IDbWrapper { Aborted.erase(data.GetInsertWriteId()); } - bool Load(TInsertTableAccessor& accessor, - const TInstant&) override { + bool Load(TInsertTableAccessor& accessor, const TInstant&) override { for (auto&& i : Inserted) { accessor.AddInserted(std::move(i.second), true); } @@ -84,7 +84,7 @@ class TTestDbWrapper : public IDbWrapper { accessor.AddAborted(std::move(i.second), true); } for (auto&& i : Committed) { - for (auto&& c: i.second) { + for (auto&& c : i.second) { auto copy = c; accessor.AddCommitted(std::move(copy), true); } @@ -103,9 +103,12 @@ class TTestDbWrapper : public IDbWrapper { virtual void ErasePortion(const NOlap::TPortionInfo& portion) override { AFL_VERIFY(Portions.erase(portion.GetPortionId())); } - virtual bool LoadPortions(const std::function& callback) override { + virtual bool LoadPortions(const std::optional pathId, + const std::function& callback) override { for (auto&& i : Portions) { - callback(NOlap::TPortionInfoConstructor(i.second, false, false, false), i.second.GetMeta().SerializeToProto()); + if (!pathId || *pathId == i.second.GetPathId()) { + callback(NOlap::TPortionInfoConstructor(i.second, false, false, false), i.second.GetMeta().SerializeToProto()); + } } return true; } @@ -117,8 +120,7 @@ class TTestDbWrapper : public IDbWrapper { } auto& data = Indices[0].Columns[portion.GetPathId()]; - NOlap::TColumnChunkLoadContextV1 loadContext( - portion.GetPathId(), portion.GetPortionId(), row.GetAddress(), row.BlobRange, rowProto); + NOlap::TColumnChunkLoadContextV1 loadContext(portion.GetPathId(), portion.GetPortionId(), row.GetAddress(), row.BlobRange, rowProto); auto itInsertInfo = LoadContexts[portion.GetAddress()].emplace(row.GetAddress(), loadContext); if (!itInsertInfo.second) { itInsertInfo.first->second = loadContext; @@ -166,9 +168,12 @@ class TTestDbWrapper : public IDbWrapper { portionLocal.MutableRecords().swap(filtered); } - bool LoadColumns(const std::function& callback) override { + bool LoadColumns(const std::optional reqPathId, const std::function& callback) override { auto& columns = Indices[0].Columns; for (auto& [pathId, portions] : columns) { + if (pathId && *reqPathId != pathId) { + continue; + } for (auto& [portionId, portionLocal] : portions) { auto copy = NOlap::TPortionInfoConstructor::TTestCopier::Copy(portionLocal); copy.MutableRecords().clear(); @@ -184,9 +189,14 @@ class TTestDbWrapper : public IDbWrapper { return true; } - virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {} - virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {} - virtual bool LoadIndexes(const std::function& /*callback*/) override { return true; } + virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { + } + virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { + } + virtual bool LoadIndexes(const std::optional /*reqPathId*/, + const std::function& /*callback*/) override { + return true; + } void WriteCounter(ui32 counterId, ui64 value) override { auto& counters = Indices[0].Counters; @@ -211,20 +221,16 @@ class TTestDbWrapper : public IDbWrapper { static const std::vector testColumns = { // PK - NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), - NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp)), + NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8)), NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8)), // - NArrow::NTest::TTestColumn("message", TTypeInfo(NTypeIds::Utf8) ) + NArrow::NTest::TTestColumn("message", TTypeInfo(NTypeIds::Utf8)) }; -static const std::vector testKey = { - NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), - NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ) -}; +static const std::vector testKey = { NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp)), + NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8)), NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8)) }; template class TBuilder { @@ -241,8 +247,7 @@ class TBuilder { }; TBuilder() - : Schema(NArrow::MakeArrowSchema(testColumns)) - { + : Schema(NArrow::MakeArrowSchema(testColumns)) { auto status = arrow::RecordBatchBuilder::Make(Schema, arrow::default_memory_pool(), &BatchBuilder); Y_ABORT_UNLESS(status.ok()); } @@ -283,7 +288,7 @@ TString MakeTestBlob(i64 start = 0, i64 end = 100, ui32 step = 1) { for (i64 ts = start; ts < end; ts += step) { TString str = ToString(ts); TString sortedStr = Sprintf("%05ld", (long)ts); - builder.AddRow({ts, sortedStr, str, str, str}); + builder.AddRow({ ts, sortedStr, str, str, str }); } auto batch = builder.Finish(); return NArrow::SerializeBatchNoCompression(batch); @@ -327,7 +332,7 @@ bool Insert(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, st for (auto&& i : changes->AppendedPortions) { blobsCount += i.GetBlobs().size(); } - UNIT_ASSERT_VALUES_EQUAL(blobsCount, 1); // add 2 columns: planStep, txId + UNIT_ASSERT_VALUES_EQUAL(blobsCount, 1); // add 2 columns: planStep, txId AddIdsToBlobs(changes->AppendedPortions, blobs, step); @@ -347,13 +352,38 @@ struct TExpected { ui32 NewGranules; }; +class TTestCompactionAccessorsSubscriber: public NOlap::IDataAccessorRequestsSubscriber { +private: + std::shared_ptr Changes; + const std::shared_ptr VersionedIndex; + + virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { + const TDataAccessorsInitializationContext context(VersionedIndex); + Changes->SetFetchedDataAccessors(std::move(result), context); + } + +public: + TTestCompactionAccessorsSubscriber( + const std::shared_ptr& changes, const std::shared_ptr& versionedIndex) + : Changes(changes) + , VersionedIndex(versionedIndex) { + } +}; + bool Compact(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, NBlobOperations::NRead::TCompositeReadBlobs&& blobs, ui32& step, - const TExpected& /*expected*/, THashMap* blobsPool = nullptr) { - std::shared_ptr changes = dynamic_pointer_cast(engine.StartCompaction(EmptyDataLocksManager)); + const TExpected& /*expected*/, THashMap* blobsPool = nullptr) { + std::shared_ptr changes = + dynamic_pointer_cast(engine.StartCompaction(EmptyDataLocksManager)); UNIT_ASSERT(changes); // UNIT_ASSERT_VALUES_EQUAL(changes->SwitchedPortions.size(), expected.SrcPortions); - changes->Blobs = std::move(blobs); changes->StartEmergency(); + { + auto request = changes->ExtractDataAccessorsRequest(); + request->RegisterSubscriber( + std::make_shared(changes, std::make_shared(engine.GetVersionedIndex()))); + engine.FetchDataAccessors(changes->ExtractDataAccessorsRequest()); + } + changes->Blobs = std::move(blobs); NOlap::TConstructionContext context(engine.GetVersionedIndex(), NColumnShard::TIndexationCounters("Compaction"), NOlap::TSnapshot(step, 1)); Y_ABORT_UNLESS(changes->ConstructBlobs(context).Ok()); @@ -370,7 +400,10 @@ bool Compact(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, N if (blobsPool) { for (auto&& i : changes->AppendedPortions) { for (auto&& r : i.GetPortionResult().GetRecords()) { - Y_ABORT_UNLESS(blobsPool->emplace(i.GetPortionResult().GetPortionInfo().RestoreBlobRange(r.BlobRange), i.GetBlobByRangeVerified(r.ColumnId, r.Chunk)).second); + Y_ABORT_UNLESS(blobsPool + ->emplace(i.GetPortionResult().GetPortionInfo().RestoreBlobRange(r.BlobRange), + i.GetBlobByRangeVerified(r.ColumnId, r.Chunk)) + .second); } } } @@ -385,8 +418,7 @@ bool Cleanup(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, u if (!expectedToDrop && !changes) { return true; } - UNIT_ASSERT_VALUES_EQUAL(changes->PortionsToDrop.size(), expectedToDrop); - + UNIT_ASSERT_VALUES_EQUAL(changes->GetPortionsToDrop().size(), expectedToDrop); changes->StartEmergency(); const bool result = engine.ApplyChangesOnTxCreate(changes, snap) && engine.ApplyChangesOnExecute(db, changes, snap); @@ -398,15 +430,19 @@ bool Cleanup(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, u return result; } -bool Ttl(TColumnEngineForLogs& engine, TTestDbWrapper& db, - const THashMap& pathEviction, ui32 expectedToDrop) { +bool Ttl(TColumnEngineForLogs& engine, TTestDbWrapper& db, const THashMap& pathEviction, ui32 expectedToDrop) { std::vector> vChanges = engine.StartTtl(pathEviction, EmptyDataLocksManager, 512 * 1024 * 1024); AFL_VERIFY(vChanges.size() == 1)("count", vChanges.size()); auto changes = vChanges.front(); UNIT_ASSERT_VALUES_EQUAL(changes->GetPortionsToRemove().size(), expectedToDrop); - changes->StartEmergency(); + { + auto request = changes->ExtractDataAccessorsRequest(); + request->RegisterSubscriber( + std::make_shared(changes, std::make_shared(engine.GetVersionedIndex()))); + engine.FetchDataAccessors(changes->ExtractDataAccessorsRequest()); + } const bool result = engine.ApplyChangesOnTxCreate(changes, TSnapshot(1, 1)) && engine.ApplyChangesOnExecute(db, changes, TSnapshot(1, 1)); NOlap::TWriteIndexContext contextExecute(nullptr, db, engine, TSnapshot(1, 1)); changes->WriteIndexOnExecute(nullptr, contextExecute); @@ -433,7 +469,7 @@ std::shared_ptr MakeStrPredicate(const std::string& key, NArrow::EOp return std::make_shared(op, arrow::RecordBatch::Make(std::make_shared(std::move(fields)), 1, { *res })); } -} // namespace +} // namespace std::shared_ptr InitializeStorageManager() { return NKikimr::NOlap::TTestStoragesManager::GetInstance(); @@ -442,13 +478,12 @@ std::shared_ptr InitializeStorageManager() { std::shared_ptr CommonStoragesManager = InitializeStorageManager(); Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { - void WriteLoadRead(const std::vector& ydbSchema, - const std::vector& key) { + void WriteLoadRead(const std::vector& ydbSchema, const std::vector& key) { TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); - std::vector paths = {1, 2}; + std::vector paths = { 1, 2 }; TString testBlob = MakeTestBlob(); @@ -459,16 +494,17 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] // load TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine( + 0, std::make_shared(), CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); for (auto&& i : paths) { engine.RegisterTable(i); } - engine.Load(db); + engine.TestingLoad(db); - std::vector dataToIndex = { - TCommittedData(TUserData::Build(paths[0], blobRanges[0], TLocalHelper::GetMetaProto(), 0, {}), TSnapshot(1, 2), 0, (TInsertWriteId)2), - TCommittedData(TUserData::Build(paths[0], blobRanges[1], TLocalHelper::GetMetaProto(), 0, {}), TSnapshot(2, 1), 0, (TInsertWriteId)1) - }; + std::vector dataToIndex = { TCommittedData( + TUserData::Build(paths[0], blobRanges[0], TLocalHelper::GetMetaProto(), 0, {}), TSnapshot(1, 2), 0, (TInsertWriteId)2), + TCommittedData( + TUserData::Build(paths[0], blobRanges[1], TLocalHelper::GetMetaProto(), 0, {}), TSnapshot(2, 1), 0, (TInsertWriteId)1) }; // write @@ -493,28 +529,28 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { columnIds.insert(indexInfo.GetColumnIdVerified(c.GetName())); } - { // select from snap before insert + { // select from snap before insert ui64 planStep = 1; ui64 txId = 0; auto selectInfo = engine.Select(paths[0], TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 0); } - { // select from snap between insert (greater txId) + { // select from snap between insert (greater txId) ui64 planStep = 1; ui64 txId = 2; auto selectInfo = engine.Select(paths[0], TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 0); } - { // select from snap after insert (greater planStep) + { // select from snap after insert (greater planStep) ui64 planStep = 2; ui64 txId = 1; auto selectInfo = engine.Select(paths[0], TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 1); } - { // select another pathId + { // select another pathId ui64 planStep = 2; ui64 txId = 1; auto selectInfo = engine.Select(paths[1], TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); @@ -527,18 +563,14 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexWriteLoadReadStrPK) { - std::vector key = { - NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ) - }; + std::vector key = { NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8)), NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp)) }; WriteLoadRead(testColumns, key); } - void ReadWithPredicates(const std::vector& ydbSchema, - const std::vector& key) { + void ReadWithPredicates(const std::vector& ydbSchema, const std::vector& key) { TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -547,9 +579,10 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui32 step = 1000; TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine( + 0, std::make_shared(), CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); engine.RegisterTable(pathId); - engine.Load(db); + engine.TestingLoad(db); // insert ui64 planStep = 1; @@ -576,8 +609,8 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // compact planStep = 2; -// bool ok = Compact(engine, db, TSnapshot(planStep, 1), std::move(blobs), step, {20, 4, 4}); -// UNIT_ASSERT(ok); + // bool ok = Compact(engine, db, TSnapshot(planStep, 1), std::move(blobs), step, {20, 4, 4}); + // UNIT_ASSERT(ok); // read @@ -588,7 +621,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { const TIndexInfo& indexInfo = engine.GetVersionedIndex().GetLastSchema()->GetIndexInfo(); THashSet oneColumnId = { indexInfo.GetColumnIdVerified(key[0].GetName()) }; - { // full scan + { // full scan ui64 txId = 1; auto selectInfo = engine.Select(pathId, TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 20); @@ -610,7 +643,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { { ui64 txId = 1; - std::shared_ptr lt10k = MakePredicate(8999, NArrow::EOperation::Less); // TODO: better border checks + std::shared_ptr lt10k = MakePredicate(8999, NArrow::EOperation::Less); // TODO: better border checks if (key[0].GetType() == TTypeInfo(NTypeIds::Utf8)) { lt10k = MakeStrPredicate("08999", NArrow::EOperation::Less); } @@ -626,12 +659,9 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexReadWithPredicatesStrPK) { - std::vector key = { - NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), - NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ) - }; + std::vector key = { NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8)), NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8)), + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp)) }; ReadWithPredicates(testColumns, key); } @@ -640,7 +670,8 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { TTestBasicRuntime runtime; TTestDbWrapper db; auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); - TIndexInfo tableInfo = NColumnShard::BuildTableInfo(testColumns, testKey);; + TIndexInfo tableInfo = NColumnShard::BuildTableInfo(testColumns, testKey); + ; ui64 pathId = 1; ui32 step = 1000; @@ -649,9 +680,10 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 planStep = 1; TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine( + 0, std::make_shared(), CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); engine.RegisterTable(pathId); - engine.Load(db); + engine.TestingLoad(db); ui64 numRows = 1000; ui64 rowPos = 0; @@ -673,16 +705,17 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { UNIT_ASSERT(ok); } - { // check it's overloaded after reload - TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo)); + { // check it's overloaded after reload + TColumnEngineForLogs tmpEngine( + 0, std::make_shared(), CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo)); tmpEngine.RegisterTable(pathId); - tmpEngine.Load(db); + tmpEngine.TestingLoad(db); } // compact planStep = 2; - bool ok = Compact(engine, db, TSnapshot(planStep, 1), std::move(blobsAll), step, {23, 5, 5}); + bool ok = Compact(engine, db, TSnapshot(planStep, 1), std::move(blobsAll), step, { 23, 5, 5 }); UNIT_ASSERT(ok); // success write after compaction @@ -704,10 +737,11 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { UNIT_ASSERT(ok); } - { // check it's not overloaded after reload - TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo)); + { // check it's not overloaded after reload + TColumnEngineForLogs tmpEngine( + 0, std::make_shared(), CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo)); tmpEngine.RegisterTable(pathId); - tmpEngine.Load(db); + tmpEngine.TestingLoad(db); } } @@ -725,16 +759,17 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 planStep = 1; TSnapshot indexSnapshot(1, 1); { - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine( + 0, std::make_shared(), CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); engine.RegisterTable(pathId); - engine.Load(db); + engine.TestingLoad(db); const ui64 numRows = 1000; const ui64 txCount = 20; const ui64 tsIncrement = 1; const auto blobTsRange = numRows * tsIncrement; - const auto gap = TDuration::Hours(1); //much longer than blobTsRange*txCount - auto blobStartTs = (TInstant::Now() - gap).MicroSeconds(); + const auto gap = TDuration::Hours(1); //much longer than blobTsRange*txCount + auto blobStartTs = (TInstant::Now() - gap).MicroSeconds(); for (ui64 txId = 1; txId <= txCount; ++txId) { TString testBlob = MakeTestBlob(blobStartTs, blobStartTs + blobTsRange, tsIncrement); auto blobRange = MakeBlobRange(++step, testBlob.size()); @@ -751,9 +786,9 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { bool ok = Insert(engine, db, ss, std::move(dataToIndex), blobs, step); UNIT_ASSERT(ok); blobStartTs += blobTsRange; - if (txId == txCount / 2) { - //Make a gap. - //NB After this gap, some rows may be in the future at the point of setting TTL + if (txId == txCount / 2) { + //Make a gap. + //NB After this gap, some rows may be in the future at the point of setting TTL blobStartTs += gap.MicroSeconds(); } } @@ -764,13 +799,13 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // bool ok = Compact(engine, db, TSnapshot(planStep, 1), std::move(blobs), step, {20, 4, 4}); // UNIT_ASSERT(ok); - // read + // read planStep = 3; const TIndexInfo& indexInfo = engine.GetVersionedIndex().GetLastSchema()->GetIndexInfo(); - THashSet oneColumnId = {indexInfo.GetColumnIdVerified(testColumns[0].GetName())}; + THashSet oneColumnId = { indexInfo.GetColumnIdVerified(testColumns[0].GetName()) }; - { // full scan + { // full scan ui64 txId = 1; auto selectInfo = engine.Select(pathId, TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 20); @@ -779,7 +814,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // Cleanup Cleanup(engine, db, TSnapshot(planStep, 1), 0); - { // full scan + { // full scan ui64 txId = 1; auto selectInfo = engine.Select(pathId, TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 20); @@ -791,11 +826,11 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { NOlap::TTiering tiering; AFL_VERIFY(tiering.Add(NOlap::TTierInfo::MakeTtl(gap, "timestamp"))); pathTtls.emplace(pathId, std::move(tiering)); - Ttl(engine, db, pathTtls, txCount / 2 ); + Ttl(engine, db, pathTtls, txCount / 2); // read + load + read - { // full scan + { // full scan ui64 txId = 1; auto selectInfo = engine.Select(pathId, TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 10); @@ -803,14 +838,15 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // load - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine( + 0, std::make_shared(), CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); engine.RegisterTable(pathId); - engine.Load(db); + engine.TestingLoad(db); const TIndexInfo& indexInfo = engine.GetVersionedIndex().GetLastSchema()->GetIndexInfo(); THashSet oneColumnId = { indexInfo.GetColumnIdVerified(testColumns[0].GetName()) }; - { // full scan + { // full scan ui64 txId = 1; auto selectInfo = engine.Select(pathId, TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false), false); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 10); @@ -819,4 +855,4 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } } -} +} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/engines/ya.make b/ydb/core/tx/columnshard/engines/ya.make index 9e5237c77bc0..2715fdd21631 100644 --- a/ydb/core/tx/columnshard/engines/ya.make +++ b/ydb/core/tx/columnshard/engines/ya.make @@ -29,6 +29,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/changes ydb/core/tx/columnshard/engines/portions ydb/core/tx/columnshard/engines/protos + ydb/core/tx/columnshard/engines/loading ydb/core/tx/program ydb/core/tx/columnshard/common diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp b/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp index b275e17f2fdb..94acb1bc8ccc 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp @@ -23,4 +23,9 @@ ui64 ICSController::GetGuaranteeIndexationStartBytesLimit() const { const ui64 defaultValue = NColumnShard::TSettings::GuaranteeIndexationStartBytesLimit; return DoGetGuaranteeIndexationStartBytesLimit(defaultValue); } + +bool ICSController::CheckPortionForEvict(const std::shared_ptr& portion) const { + return portion->HasRuntimeFeature(NOlap::TPortionInfo::ERuntimeFeature::Optimized) && !portion->HasInsertWriteId(); +} + } diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index c104590235d3..0f32f19b7307 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -146,6 +146,8 @@ class ICSController { const std::set& /*snapshotsToSave*/, const std::set& /*snapshotsToRemove*/) { } + virtual bool CheckPortionForEvict(const std::shared_ptr& portion) const; + TDuration GetPingCheckPeriod() const { const TDuration defaultValue = 0.6 * GetReadTimeoutClean(); return DoGetPingCheckPeriod(defaultValue); diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp index 1f3bd99ba9db..e048c11614ed 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp @@ -25,16 +25,18 @@ void TController::DoOnAfterGCAction(const ::NKikimr::NColumnShard::TColumnShard& } } -void TController::CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& shard, TCheckContext& context) const { +void TController::CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& shard, TCheckContext& /*context*/) const { if (!shard.HasIndex()) { return; } + /* const auto& index = shard.GetIndexAs(); std::vector> granules = index.GetTables({}, {}); THashMap> ids; for (auto&& i : granules) { + auto accessor = i->GetDataAccessorPtrVerifiedAs(); for (auto&& p : i->GetPortions()) { - NOlap::TPortionDataAccessor(p.second).FillBlobIdsByStorage(ids, index.GetVersionedIndex()); + accessor->BuildAccessor(p.second).FillBlobIdsByStorage(ids, index.GetVersionedIndex()); } } for (auto&& i : ids) { @@ -63,6 +65,7 @@ void TController::CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& s } } context.AddCategories(shard.TabletID(), std::move(shardBlobsCategories)); + */ } TController::TCheckContext TController::CheckInvariants() const { diff --git a/ydb/core/tx/columnshard/loading/stages.cpp b/ydb/core/tx/columnshard/loading/stages.cpp new file mode 100644 index 000000000000..ed9a5fd6f87c --- /dev/null +++ b/ydb/core/tx/columnshard/loading/stages.cpp @@ -0,0 +1,230 @@ +#include "stages.h" + +#include +#include +#include +#include + +namespace NKikimr::NColumnShard::NLoading { + +bool TInsertTableInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + TBlobGroupSelector dsGroupSelector(Self->Info()); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + auto localInsertTable = std::make_unique(); + for (auto&& i : Self->TablesManager.GetTables()) { + localInsertTable->RegisterPathInfo(i.first); + } + if (!localInsertTable->Load(db, dbTable, TAppData::TimeProvider->Now())) { + ACFL_ERROR("step", "TInsertTable::Load_Fails"); + return false; + } + Self->InsertTable.swap(localInsertTable); + return true; +} + +bool TInsertTableInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TTxControllerInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + auto localTxController = std::make_unique(*Self); + if (!localTxController->Load(txc)) { + return false; + } + Self->ProgressTxController.swap(localTxController); + return true; +} + +bool TTxControllerInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TOperationsManagerInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + auto localOperationsManager = std::make_unique(); + if (!localOperationsManager->Load(txc)) { + return false; + } + Self->OperationsManager.swap(localOperationsManager); + return true; +} + +bool TOperationsManagerInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TStoragesManagerInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + AFL_VERIFY(Self->StoragesManager); + return Self->StoragesManager->LoadIdempotency(txc.DB); +} + +bool TStoragesManagerInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TLongTxInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return false; + } + + while (!rowset.EndOfSet()) { + const TInsertWriteId writeId = (TInsertWriteId)rowset.GetValue(); + const ui32 writePartId = rowset.GetValue(); + NKikimrLongTxService::TLongTxId proto; + Y_ABORT_UNLESS(proto.ParseFromString(rowset.GetValue())); + const auto longTxId = NLongTxService::TLongTxId::FromProto(proto); + + std::optional granuleShardingVersion; + if (rowset.HaveValue() && + rowset.GetValue()) { + granuleShardingVersion = rowset.GetValue(); + } + + Self->LoadLongTxWrite(writeId, writePartId, longTxId, granuleShardingVersion); + + if (!rowset.Next()) { + return false; + } + } + return true; +} + +bool TLongTxInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TDBLocksInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + if (txc.DB.GetScheme().GetTableInfo(Schema::Locks::TableId)) { + TColumnShardLocksDb locksDb(*Self, txc); + if (!Self->SysLocks.Load(locksDb)) { + return false; + } + } + return true; +} + +bool TDBLocksInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TBackgroundSessionsInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + return Self->BackgroundSessionsManager->LoadIdempotency(txc); +} + +bool TSharingSessionsInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + auto local = std::make_shared(); + if (!local->Load(txc.DB, Self->TablesManager.GetPrimaryIndexAsOptional())) { + return false; + } + Self->SharingSessionsManager = local; + return true; +} + +bool TInFlightReadsInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + TInFlightReadsTracker local(Self->StoragesManager, Self->Counters.GetRequestsTracingCounters()); + if (!local.LoadFromDatabase(txc.DB)) { + return false; + } + Self->InFlightReadsTracker = std::move(local); + return true; +} + +bool TSpecialValuesInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::CurrentSchemeShardId, Self->CurrentSchemeShardId)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastSchemaSeqNoGeneration, Self->LastSchemaSeqNo.Generation)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastSchemaSeqNoRound, Self->LastSchemaSeqNo.Round)) { + return false; + } + if (!Schema::GetSpecialProtoValue(db, Schema::EValueIds::ProcessingParams, Self->ProcessingParams)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastPlannedStep, Self->LastPlannedStep)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastPlannedTxId, Self->LastPlannedTxId)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastExportNumber, Self->LastExportNo)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::OwnerPathId, Self->OwnerPathId)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::OwnerPath, Self->OwnerPath)) { + return false; + } + + { + ui64 lastCompletedStep = 0; + ui64 lastCompletedTx = 0; + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastCompletedStep, lastCompletedStep)) { + return false; + } + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::LastCompletedTxId, lastCompletedTx)) { + return false; + } + Self->LastCompletedTx = NOlap::TSnapshot(lastCompletedStep, lastCompletedTx); + } + + return true; +} + +bool TSpecialValuesInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return Schema::Precharge(db, txc.DB.GetScheme()); +} + +bool TTablesManagerInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + TTablesManager tablesManagerLocal(Self->StoragesManager, Self->DataAccessorsManager.GetObjectPtrVerified(), Self->TabletID()); + { + TMemoryProfileGuard g("TTxInit/TTablesManager"); + if (!tablesManagerLocal.InitFromDB(db)) { + return false; + } + } + Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, tablesManagerLocal.GetTables().size()); + Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_PRESETS, tablesManagerLocal.GetSchemaPresets().size()); + Self->Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_TTLS, tablesManagerLocal.GetTtl().PathsCount()); + + Self->TablesManager = std::move(tablesManagerLocal); + return true; +} + +std::shared_ptr TTablesManagerInitializer::BuildNextReaderAfterLoad() { + if (Self->TablesManager.HasPrimaryIndex()) { + return Self->TablesManager.MutablePrimaryIndex().BuildLoader(std::make_shared(Self->Info())); + } else { + return nullptr; + } +} + +bool TTablesManagerInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + NIceDb::TNiceDb db(txc.DB); + return (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()) & + (int)Schema::Precharge(db, txc.DB.GetScheme()); +} + +} // namespace NKikimr::NColumnShard::NLoading diff --git a/ydb/core/tx/columnshard/loading/stages.h b/ydb/core/tx/columnshard/loading/stages.h new file mode 100644 index 000000000000..7fced6b3eba4 --- /dev/null +++ b/ydb/core/tx/columnshard/loading/stages.h @@ -0,0 +1,141 @@ +#pragma once +#include + +namespace NKikimr::NColumnShard { +class TColumnShard; + +} + +namespace NKikimr::NColumnShard::NLoading { + +class ITxShardInitReader: public ITxReader { +private: + using TBase = ITxReader; + +protected: + TColumnShard* Self = nullptr; + +public: + ITxShardInitReader(const TString& name, TColumnShard* shard) + : TBase(name) + , Self(shard) { + } +}; + +class TInsertTableInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TTxControllerInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TOperationsManagerInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TStoragesManagerInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; +class TLongTxInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TDBLocksInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TBackgroundSessionsInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + +public: + using TBase::TBase; +}; + +class TSharingSessionsInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + +public: + using TBase::TBase; +}; + +class TInFlightReadsInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + +public: + using TBase::TBase; +}; + +class TSpecialValuesInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +class TTablesManagerInitializer: public ITxShardInitReader { +private: + using TBase = ITxShardInitReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + virtual std::shared_ptr BuildNextReaderAfterLoad() override; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; + +public: + using TBase::TBase; +}; + +} // namespace NKikimr::NColumnShard::NLoading diff --git a/ydb/core/tx/columnshard/loading/ya.make b/ydb/core/tx/columnshard/loading/ya.make new file mode 100644 index 000000000000..f6aa2b9e641e --- /dev/null +++ b/ydb/core/tx/columnshard/loading/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + stages.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/tx_reader + ydb/services/metadata/abstract + ydb/core/tx/columnshard/blobs_action/events + ydb/core/tx/columnshard/data_sharing +) + +END() diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp index fcd56dbb0515..df21f5066018 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp @@ -2,6 +2,7 @@ #include "normalizer.h" #include +#include #include #include #include @@ -139,7 +140,7 @@ TConclusion> TChunksNormalizer::DoInit( return tasks; } - TTablesManager tablesManager(controller.GetStoragesManager(), 0); + TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(), 0); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TChunksNormalizer")("error", "can't initialize tables manager"); return TConclusionStatus::Fail("Can't load index"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp index e57e4640a7a2..dc8788e04b81 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp @@ -1,6 +1,7 @@ #include "normalizer.h" #include +#include #include #include #include @@ -24,7 +25,7 @@ TConclusion> TPortionsNormalizerBase::DoInit( return TConclusionStatus::Fail("Not ready"); } - NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), 0); + NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(), 0); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TPortionsNormalizer")("error", "can't initialize tables manager"); return TConclusionStatus::Fail("Can't load index"); @@ -93,7 +94,7 @@ TConclusion> TPortionsNormalizerBase::DoInit( TConclusionStatus TPortionsNormalizerBase::InitPortions( const NColumnShard::TTablesManager& tablesManager, NIceDb::TNiceDb& db, THashMap& constructors) { TDbWrapper wrapper(db.GetDatabase(), nullptr); - if (!wrapper.LoadPortions([&](TPortionInfoConstructor&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + if (!wrapper.LoadPortions({}, [&](TPortionInfoConstructor&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { const TIndexInfo& indexInfo = portion.GetSchema(tablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex())->GetIndexInfo(); AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo, DsGroupSelector)); diff --git a/ydb/core/tx/columnshard/normalizer/portion/restore_portion_from_chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/restore_portion_from_chunks.cpp index f4fb377f3b7d..0aa97fc9916f 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/restore_portion_from_chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/restore_portion_from_chunks.cpp @@ -72,7 +72,7 @@ TConclusion> TNormalizer::DoInit( return TConclusionStatus::Fail("Not ready"); } - TTablesManager tablesManager(controller.GetStoragesManager(), 0); + TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(), 0); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TChunksNormalizer")("error", "can't initialize tables manager"); return TConclusionStatus::Fail("Can't load index"); diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 10b0b564631c..b90e27218134 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -145,7 +145,8 @@ void TWriteOperation::AbortOnComplete(TColumnShard& owner) const { Y_ABORT_UNLESS(Status == EOperationStatus::Prepared); if (WritePortions) { for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId).AbortPortionOnComplete(i); + owner.MutableIndexAs().MutableGranuleVerified(PathId).AbortPortionOnComplete( + i, owner.MutableIndexAs()); } } } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index c6601e5a2f9c..1047b10372fd 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -189,8 +189,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { "version", schemaInfo.GetSchema().GetVersion()); NOlap::IColumnEngine::TSchemaInitializationData schemaInitializationData(schemaInfo); if (!PrimaryIndex) { - PrimaryIndex = std::make_unique( - TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInitializationData); + PrimaryIndex = std::make_unique(TabletId, DataAccessorsManager, StoragesManager, + preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInitializationData); } else { PrimaryIndex->RegisterSchemaVersion(preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInitializationData); } @@ -203,15 +203,6 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { return true; } -bool TTablesManager::LoadIndex(NOlap::TDbWrapper& idxDB) { - if (PrimaryIndex) { - if (!PrimaryIndex->Load(idxDB)) { - return false; - } - } - return true; -} - bool TTablesManager::HasTable(const ui64 pathId, bool withDeleted) const { auto it = Tables.find(pathId); if (it == Tables.end()) { @@ -302,7 +293,7 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo); if (!PrimaryIndex) { PrimaryIndex = std::make_unique( - TabletId, StoragesManager, version, NOlap::IColumnEngine::TSchemaInitializationData(versionInfo)); + TabletId, DataAccessorsManager, StoragesManager, version, NOlap::IColumnEngine::TSchemaInitializationData(versionInfo)); for (auto&& i : Tables) { PrimaryIndex->RegisterTable(i.first); } @@ -320,8 +311,8 @@ std::unique_ptr TTablesManager::CreateAddShar } void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, - const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, const std::optional& schema, NIceDb::TNiceDb& db, - std::shared_ptr& manager) { + const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, const std::optional& schema, + NIceDb::TNiceDb& db, std::shared_ptr& manager) { auto it = Tables.find(pathId); AFL_VERIFY(it != Tables.end()); auto& table = it->second; @@ -354,11 +345,12 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& table.AddVersion(version); } -TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId) +TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, + const std::shared_ptr& dataAccessorsManager, const ui64 tabletId) : StoragesManager(storagesManager) + , DataAccessorsManager(dataAccessorsManager) , LoadTimeCounters(std::make_unique()) - , TabletId(tabletId) -{ + , TabletId(tabletId) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 567250bf35c4..1a55c68547d6 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -4,6 +4,7 @@ #include "columnshard_ttl.h" #include "blobs_action/abstract/storages_manager.h" +#include "data_accessor/manager.h" #include "engines/column_engine.h" #include @@ -153,13 +154,15 @@ class TTablesManager { TTtl Ttl; std::unique_ptr PrimaryIndex; std::shared_ptr StoragesManager; + std::shared_ptr DataAccessorsManager; std::unique_ptr LoadTimeCounters; ui64 TabletId = 0; public: friend class TTxInit; - TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId); + TTablesManager(const std::shared_ptr& storagesManager, + const std::shared_ptr& dataAccessorsManager, const ui64 tabletId); const std::unique_ptr& GetLoadTimeCounters() const { return LoadTimeCounters; @@ -242,7 +245,6 @@ class TTablesManager { } bool InitFromDB(NIceDb::TNiceDb& db); - bool LoadIndex(NOlap::TDbWrapper& db); const TTableInfo& GetTable(const ui64 pathId) const; ui64 GetMemoryUsage() const; diff --git a/ydb/core/tx/columnshard/test_helper/controllers.h b/ydb/core/tx/columnshard/test_helper/controllers.h index 68cd6a1dc4ed..a3e1eb1de47c 100644 --- a/ydb/core/tx/columnshard/test_helper/controllers.h +++ b/ydb/core/tx/columnshard/test_helper/controllers.h @@ -13,6 +13,8 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController { YDB_READONLY(TAtomicCounter, StatisticsUsageCount, 0); YDB_READONLY(TAtomicCounter, MaxValueUsageCount, 0); YDB_ACCESSOR_DEF(std::optional, SmallSizeDetector); + YDB_ACCESSOR(bool, SkipSpecialCheckForEvict, false); + protected: virtual void OnTieringModified(const std::shared_ptr& /*tiers*/) override; virtual void OnExportFinished() override { @@ -34,6 +36,15 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController { return TDuration::Zero(); } public: + virtual bool CheckPortionForEvict(const std::shared_ptr& portion) const override { + if (SkipSpecialCheckForEvict) { + return true; + } else { + return TBase::CheckPortionForEvict(portion); + } + } + + TWaitCompactionController() { SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); } diff --git a/ydb/core/tx/columnshard/tx_reader/abstract.cpp b/ydb/core/tx/columnshard/tx_reader/abstract.cpp new file mode 100644 index 000000000000..bbda1c5cee65 --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/abstract.cpp @@ -0,0 +1,38 @@ +#include "abstract.h" + +namespace NKikimr { + +bool ITxReader::Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) { + if (IsReady) { + if (!NextReaderAfterLoad) { + return true; + } else { + return NextReaderAfterLoad->Execute(txc, ctx); + } + } + IsStarted = true; + { + TMemoryProfileGuard g("ITxReader/" + StageName + "/Precharge"); + NColumnShard::TLoadTimeSignals::TLoadTimer timer = PrechargeCounters.StartGuard(); + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("load_stage_name", "PRECHARGE:" + StageName); + if (!DoPrecharge(txc, ctx)) { + timer.AddLoadingFail(); + return false; + } + } + + { + TMemoryProfileGuard g("ITxReader/" + StageName + "/Read"); + NColumnShard::TLoadTimeSignals::TLoadTimer timer = ReaderCounters.StartGuard(); + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("load_stage_name", "EXECUTE:" + StageName); + if (!DoExecute(txc, ctx)) { + timer.AddLoadingFail(); + return false; + } + } + IsReady = true; + NextReaderAfterLoad = BuildNextReaderAfterLoad(); + return NextReaderAfterLoad ? NextReaderAfterLoad->Execute(txc, ctx) : true; +} + +} \ No newline at end of file diff --git a/ydb/core/tx/columnshard/tx_reader/abstract.h b/ydb/core/tx/columnshard/tx_reader/abstract.h new file mode 100644 index 000000000000..72ed2a2b9e4a --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/abstract.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include + +#include + +namespace NKikimr { + +class ITxReader { +private: + YDB_READONLY_DEF(TString, StageName); + bool IsReady = false; + bool IsStarted = false; + std::shared_ptr NextReaderAfterLoad; + NColumnShard::TLoadTimeSignals PrechargeCounters; + NColumnShard::TLoadTimeSignals ReaderCounters; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) = 0; + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) = 0; + + virtual std::shared_ptr BuildNextReaderAfterLoad() { + return nullptr; + } + +public: + virtual ~ITxReader() = default; + void AddNamePrefix(const TString& prefix) { + StageName = prefix + StageName; + } + + ITxReader(const TString& stageName) + : StageName(stageName) + , PrechargeCounters("PRECHARGE:" + stageName) + , ReaderCounters("EXECUTE:" + stageName) { + AFL_VERIFY(StageName); + } + + bool GetIsStarted() const { + return IsStarted; + } + + bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx); +}; + +} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/tx_reader/composite.cpp b/ydb/core/tx/columnshard/tx_reader/composite.cpp new file mode 100644 index 000000000000..ae25a605db5d --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/composite.cpp @@ -0,0 +1,5 @@ +#include "composite.h" + +namespace NKikimr { + +} \ No newline at end of file diff --git a/ydb/core/tx/columnshard/tx_reader/composite.h b/ydb/core/tx/columnshard/tx_reader/composite.h new file mode 100644 index 000000000000..849e9c5c1c1a --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/composite.h @@ -0,0 +1,34 @@ +#pragma once +#include "abstract.h" + +namespace NKikimr { + +class TTxCompositeReader: public ITxReader { +private: + using TBase = ITxReader; + std::vector> Children; + + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { + for (auto&& i : Children) { + if (!i->Execute(txc, ctx)) { + return false; + } + } + return true; + } + +public: + using TBase::TBase; + + void AddChildren(const std::shared_ptr& reader) { + AFL_VERIFY(!GetIsStarted()); + reader->AddNamePrefix(GetStageName() + "/"); + Children.emplace_back(reader); + } +}; + +} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/tx_reader/lambda.cpp b/ydb/core/tx/columnshard/tx_reader/lambda.cpp new file mode 100644 index 000000000000..50222b22d030 --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/lambda.cpp @@ -0,0 +1,5 @@ +#include "lambda.h" + +namespace NKikimr { + +} \ No newline at end of file diff --git a/ydb/core/tx/columnshard/tx_reader/lambda.h b/ydb/core/tx/columnshard/tx_reader/lambda.h new file mode 100644 index 000000000000..a6e81bd50802 --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/lambda.h @@ -0,0 +1,33 @@ +#pragma once +#include "abstract.h" + +namespace NKikimr { + +class TTxLambdaReader: public ITxReader { +private: + using TBase = ITxReader; + +public: + using TFunction = std::function; + +private: + TFunction PrechargeFunction; + TFunction ExecuteFunction; + + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { + return PrechargeFunction(txc, ctx); + } + + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { + return ExecuteFunction(txc, ctx); + } + +public: + TTxLambdaReader(const TString& name, const TFunction& precharge, const TFunction& execute) + : TBase(name) + , PrechargeFunction(precharge) + , ExecuteFunction(execute) { + } +}; + +} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/tx_reader/ya.make b/ydb/core/tx/columnshard/tx_reader/ya.make new file mode 100644 index 000000000000..8e9d0533295a --- /dev/null +++ b/ydb/core/tx/columnshard/tx_reader/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + abstract.cpp + composite.cpp + lambda.cpp +) + +PEERDIR( + ydb/core/tablet_flat + ydb/core/tx/columnshard/counters +) + +END() diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index 407f25c1eb43..1e6f1c59376f 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -930,6 +930,7 @@ void TestCompactionInGranuleImpl(bool reboots, const TestTableDescription& table TTestBasicRuntime runtime; TTester::Setup(runtime); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); TActorId sender = runtime.AllocateEdgeActor(); CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); @@ -2166,7 +2167,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { void TestCompactionSplitGranuleImpl(const TestTableDescription& table, const TTestBlobOptions& testBlobOptions = {}) { TTestBasicRuntime runtime; TTester::Setup(runtime); - runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_NOTICE); + runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csDefaultControllerGuard->SetSmallSizeDetector(1LLU << 20); @@ -2530,29 +2531,29 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { Cerr << sb; } if (auto compact = dynamic_pointer_cast(msg->IndexChanges)) { - Y_ABORT_UNLESS(compact->SwitchedPortions.size()); + Y_ABORT_UNLESS(compact->GetSwitchedPortions().size()); ++compactionsHappened; TStringBuilder sb; sb << "Compaction old portions:"; ui64 srcPathId{ 0 }; - for (const auto& portionInfo : compact->SwitchedPortions) { - const ui64 pathId = portionInfo.GetPortionInfo().GetPathId(); + for (const auto& portionInfo : compact->GetSwitchedPortions()) { + const ui64 pathId = portionInfo->GetPathId(); UNIT_ASSERT(!srcPathId || srcPathId == pathId); srcPathId = pathId; - oldPortions.insert(portionInfo.GetPortionInfo().GetPortionId()); - sb << portionInfo.GetPortionInfo().GetPortionId() << ","; + oldPortions.insert(portionInfo->GetPortionId()); + sb << portionInfo->GetPortionId() << ","; } sb << Endl; Cerr << sb; } if (auto cleanup = dynamic_pointer_cast(msg->IndexChanges)) { - Y_ABORT_UNLESS(cleanup->PortionsToDrop.size()); + Y_ABORT_UNLESS(cleanup->GetPortionsToDrop().size()); ++cleanupsHappened; TStringBuilder sb; sb << "Cleanup old portions:"; - for (const auto& portion : cleanup->PortionsToDrop) { - sb << " " << portion.GetPortionInfo().GetPortionId(); - deletedPortions.insert(portion.GetPortionInfo().GetPortionId()); + for (const auto& portion : cleanup->GetPortionsToDrop()) { + sb << " " << portion->GetPortionId(); + deletedPortions.insert(portion->GetPortionId()); } sb << Endl; Cerr << sb; diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index f20664cf838e..542bb71cb5bd 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -55,6 +55,9 @@ PEERDIR( ydb/core/tx/columnshard/data_sharing ydb/core/tx/columnshard/subscriber ydb/core/tx/columnshard/export + ydb/core/tx/columnshard/tx_reader + ydb/core/tx/columnshard/loading + ydb/core/tx/columnshard/data_accessor ydb/core/tx/columnshard/resource_subscriber ydb/core/tx/columnshard/normalizer ydb/core/tx/columnshard/blobs_action/storages_manager diff --git a/ydb/library/formats/arrow/replace_key.h b/ydb/library/formats/arrow/replace_key.h index 1dda69cec323..159915f944a5 100644 --- a/ydb/library/formats/arrow/replace_key.h +++ b/ydb/library/formats/arrow/replace_key.h @@ -50,6 +50,19 @@ class TReplaceKeyTemplate { return sb; } + NJson::TJsonValue DebugJson() const { + NJson::TJsonValue result = NJson::JSON_ARRAY; + for (auto&& i : *Columns) { + auto res = i->GetScalar(Position); + if (!res.ok()) { + result.AppendValue(res.status().ToString()); + } else { + result.AppendValue((*res)->ToString()); + } + } + return result; + } + TReplaceKeyTemplate(TArrayVecPtr columns, const ui64 position) : Columns(columns) , Position(position) diff --git a/ydb/services/bg_tasks/abstract/interface.h b/ydb/services/bg_tasks/abstract/interface.h index b69558e5db13..6e57260ee4ac 100644 --- a/ydb/services/bg_tasks/abstract/interface.h +++ b/ydb/services/bg_tasks/abstract/interface.h @@ -103,36 +103,21 @@ class IProtoStringSerializable: public TBaseClass { }; template -class TCommonInterfaceContainer { +class TControlInterfaceContainer { protected: std::shared_ptr Object; - using TFactory = typename IInterface::TFactory; public: - TCommonInterfaceContainer() = default; - TCommonInterfaceContainer(std::shared_ptr object) + TControlInterfaceContainer() = default; + TControlInterfaceContainer(std::shared_ptr object) : Object(object) { } template - TCommonInterfaceContainer(std::shared_ptr object) + TControlInterfaceContainer(std::shared_ptr object) : Object(object) { static_assert(std::is_base_of::value); } - bool Initialize(const TString& className, const bool maybeExists = false) { - AFL_VERIFY(maybeExists || !Object)("problem", "initialize for not-empty-object"); - Object.reset(TFactory::Construct(className)); - if (!Object) { - ALS_ERROR(NKikimrServices::BG_TASKS) << "incorrect class name: " << className << " for " << typeid(IInterface).name(); - return false; - } - return true; - } - - TString GetClassName() const { - return Object ? Object->GetClassName() : "UNDEFINED"; - } - bool HasObject() const { return !!Object; } @@ -194,6 +179,32 @@ class TCommonInterfaceContainer { }; +template +class TCommonInterfaceContainer: public TControlInterfaceContainer { +private: + using TBase = TControlInterfaceContainer; +protected: + using TFactory = typename IInterface::TFactory; + using TBase::Object; +public: + using TBase::TBase; + + bool Initialize(const TString& className, const bool maybeExists = false) { + AFL_VERIFY(maybeExists || !Object)("problem", "initialize for not-empty-object"); + Object.reset(TFactory::Construct(className)); + if (!Object) { + ALS_ERROR(NKikimrServices::BG_TASKS) << "incorrect class name: " << className << " for " << typeid(IInterface).name(); + return false; + } + return true; + } + + TString GetClassName() const { + return Object ? Object->GetClassName() : "UNDEFINED"; + } + +}; + class TStringContainerProcessor { public: static bool DeserializeFromContainer(const TString& data, TString& className, TString& binary);