diff --git a/ydb/core/base/blobstorage.h b/ydb/core/base/blobstorage.h index e15d3bc31db1..ac9f905509b9 100644 --- a/ydb/core/base/blobstorage.h +++ b/ydb/core/base/blobstorage.h @@ -1013,14 +1013,16 @@ struct TEvBlobStorage { bool WrittenBeyondBarrier = false; // was this blob written beyond the barrier? mutable NLWTrace::TOrbit Orbit; std::shared_ptr ExecutionRelay; + const TString StorageId; TEvPutResult(NKikimrProto::EReplyStatus status, const TLogoBlobID &id, const TStorageStatusFlags statusFlags, - ui32 groupId, float approximateFreeSpaceShare) + ui32 groupId, float approximateFreeSpaceShare, const TString& storageId = Default()) : Status(status) , Id(id) , StatusFlags(statusFlags) , GroupId(groupId) , ApproximateFreeSpaceShare(approximateFreeSpaceShare) + , StorageId(storageId) {} TString Print(bool isFull) const { diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp index 0353207940b7..aeec2c01f225 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.cpp @@ -11,6 +11,10 @@ TConclusionStatus TAddColumnOperation::DoDeserialize(NYql::TObjectSettingsImpl:: } ColumnName = *fValue; } + StorageId = features.Extract("STORAGE_ID"); + if (StorageId && !*StorageId) { + return TConclusionStatus::Fail("STORAGE_ID cannot be empty string"); + } { auto fValue = features.Extract("TYPE"); if (!fValue) { @@ -31,6 +35,9 @@ void TAddColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSc auto column = schemaData.AddAddColumns(); column->SetName(ColumnName); column->SetType(ColumnType); + if (StorageId) { + column->SetStorageId(*StorageId); + } column->SetNotNull(NotNull); } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.h index 6bf452c9f860..a78207845dea 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.h +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/add_column.h @@ -12,6 +12,7 @@ class TAddColumnOperation : public ITableStoreOperation { private: TString ColumnName; TString ColumnType; + std::optional StorageId; bool NotNull = false; public: TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) override; diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp index c3b65e981194..bd430d71fc47 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.cpp @@ -10,6 +10,10 @@ TConclusionStatus TAlterColumnOperation::DoDeserialize(NYql::TObjectSettingsImpl } ColumnName = *fValue; } + StorageId = features.Extract("STORAGE_ID"); + if (StorageId && !*StorageId) { + return TConclusionStatus::Fail("STORAGE_ID cannot be empty string"); + } { auto result = DictionaryEncodingDiff.DeserializeFromRequestFeatures(features); if (!result) { @@ -28,6 +32,9 @@ TConclusionStatus TAlterColumnOperation::DoDeserialize(NYql::TObjectSettingsImpl void TAlterColumnOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& schemaData) const { auto* column = schemaData.AddAlterColumns(); column->SetName(ColumnName); + if (StorageId && !!*StorageId) { + column->SetStorageId(*StorageId); + } if (!!Serializer) { Serializer.SerializeToProto(*column->MutableSerializer()); } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h index 81c0e362be3d..c883ab035f9d 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_column.h @@ -13,6 +13,7 @@ class TAlterColumnOperation : public ITableStoreOperation { static inline auto Registrator = TFactory::TRegistrator(GetTypeName()); TString ColumnName; + std::optional StorageId; NArrow::NSerialization::TSerializerContainer Serializer; NArrow::NDictionary::TEncodingDiff DictionaryEncodingDiff; diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.cpp index 61914cb6e005..ae0f08e3333d 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.cpp @@ -12,6 +12,10 @@ TConclusionStatus TUpsertIndexOperation::DoDeserialize(NYql::TObjectSettingsImpl } IndexName = *fValue; } + StorageId = features.Extract("STORAGE_ID"); + if (StorageId && !*StorageId) { + return TConclusionStatus::Fail("STORAGE_ID cannot be empty string"); + } TString indexType; { auto fValue = features.Extract("TYPE"); @@ -42,6 +46,9 @@ TConclusionStatus TUpsertIndexOperation::DoDeserialize(NYql::TObjectSettingsImpl void TUpsertIndexOperation::DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& schemaData) const { auto* indexProto = schemaData.AddUpsertIndexes(); + if (StorageId) { + indexProto->SetStorageId(*StorageId); + } indexProto->SetName(IndexName); IndexMetaConstructor.SerializeToProto(*indexProto); } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.h index 267829a1a5f4..12305f85f0ae 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.h +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_index.h @@ -12,6 +12,7 @@ class TUpsertIndexOperation : public ITableStoreOperation { static inline auto Registrator = TFactory::TRegistrator(GetTypeName()); private: TString IndexName; + std::optional StorageId; NBackgroundTasks::TInterfaceProtoContainer IndexMetaConstructor; public: TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) override; diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 73463b798d04..23d9eaa68493 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -2127,6 +2128,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { TLocalHelper(kikimr).CreateTestOlapTable(); auto tableClient = kikimr.GetTableClient(); + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); { WriteTestData(kikimr, "/Root/olapStore/olapTable", 10000, 3000000, 1000); @@ -2137,6 +2139,11 @@ Y_UNIT_TEST_SUITE(KqpOlap) { WriteTestData(kikimr, "/Root/olapStore/olapTable", 20000, 2000000, 7000); WriteTestData(kikimr, "/Root/olapStore/olapTable", 30000, 1000000, 11000); } + while (csController->GetIndexations().Val() == 0) { + Cout << "Wait indexation..." << Endl; + Sleep(TDuration::Seconds(2)); + } + AFL_VERIFY(Singleton()->GetSize()); { TString query = R"( @@ -3637,12 +3644,12 @@ Y_UNIT_TEST_SUITE(KqpOlap) { auto settings = TKikimrSettings() .SetWithSampleTables(false); TKikimrRunner kikimr(settings); - auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); static ui32 numKinds = 2; + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); TLocalHelper(kikimr).CreateTestOlapTable(); for (ui64 i = 0; i < 100; ++i) { - WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000 + i*10000, 1000); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000 + i * 10000, 1000); } auto tableClient = kikimr.GetTableClient(); diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index ba7772bcca9c..9228eb7837a3 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -418,6 +418,7 @@ message TOlapColumnDiff { optional string Name = 1; optional TDictionaryEncodingSettings DictionaryEncoding = 4; optional TOlapColumn.TSerializer Serializer = 5; + optional string StorageId = 6; } message TOlapColumnDescription { @@ -435,6 +436,7 @@ message TOlapColumnDescription { optional TCompressionOptions Compression = 8[deprecated = true]; optional TDictionaryEncodingSettings DictionaryEncoding = 9; optional TOlapColumn.TSerializer Serializer = 10; + optional string StorageId = 11; } message TRequestedBloomFilter { @@ -445,8 +447,9 @@ message TRequestedBloomFilter { message TOlapIndexRequested { optional string Name = 1; optional TCompressionOptions Compression = 3; + optional string StorageId = 4; - optional string ClassName = 2; + optional string ClassName = 39; oneof Implementation { TRequestedBloomFilter BloomFilter = 40; } @@ -465,9 +468,11 @@ message TOlapIndexDescription { optional string Name = 2; optional TCompressionOptions Compression = 3; - optional string ClassName = 4; + optional string StorageId = 4; + + optional string ClassName = 40; oneof Implementation { - TBloomFilter BloomFilter = 40; + TBloomFilter BloomFilter = 41; } } diff --git a/ydb/core/testlib/cs_helper.cpp b/ydb/core/testlib/cs_helper.cpp index 8545bec37722..dc9412bf741b 100644 --- a/ydb/core/testlib/cs_helper.cpp +++ b/ydb/core/testlib/cs_helper.cpp @@ -184,9 +184,9 @@ TString THelper::GetTestTableSchema() const { TStringBuilder sb; sb << R"(Columns{ Name: "timestamp" Type : "Timestamp" NotNull : true })"; sb << R"(Columns{ Name: "resource_id" Type : "Utf8" })"; - sb << R"(Columns{ Name: "uid" Type : "Utf8" })"; + sb << R"(Columns{ Name: "uid" Type : "Utf8" StorageId : "__MEMORY" })"; sb << R"(Columns{ Name: "level" Type : "Int32" })"; - sb << R"(Columns{ Name: "message" Type : "Utf8" })"; + sb << R"(Columns{ Name: "message" Type : "Utf8" StorageId : "__MEMORY" })"; if (GetWithJsonDocument()) { sb << R"(Columns{ Name: "json_payload" Type : "JsonDocument" })"; } diff --git a/ydb/core/tx/columnshard/blob_cache.h b/ydb/core/tx/columnshard/blob_cache.h index d65dc6aa6242..75e0ccf0a677 100644 --- a/ydb/core/tx/columnshard/blob_cache.h +++ b/ydb/core/tx/columnshard/blob_cache.h @@ -74,12 +74,14 @@ struct TEvBlobCache { TString Data; const bool FromCache = false; const TInstant ConstructTime = Now(); + const TString DataSourceId; - TEvReadBlobRangeResult(const TBlobRange& blobRange, NKikimrProto::EReplyStatus status, const TString& data, const bool fromCache = false) + TEvReadBlobRangeResult(const TBlobRange& blobRange, NKikimrProto::EReplyStatus status, const TString& data, const bool fromCache = false, const TString& dataSourceId = Default()) : BlobRange(blobRange) , Status(status) , Data(data) , FromCache(fromCache) + , DataSourceId(dataSourceId) {} }; diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/action.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/action.cpp index fdd3fd5adba1..f1d2e4e051ee 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/action.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/action.cpp @@ -3,16 +3,4 @@ namespace NKikimr::NOlap { -std::shared_ptr TBlobsAction::GetWriting(const TPortionInfo& portionInfo) { - return GetStorageAction(portionInfo.GetBlobsStorage()->GetStorageId()).GetWriting(ConsumerId); -} - -std::shared_ptr TBlobsAction::GetReading(const TPortionInfo& portionInfo) { - return GetStorageAction(portionInfo.GetBlobsStorage()->GetStorageId()).GetReading(ConsumerId); -} - -std::shared_ptr TBlobsAction::GetRemoving(const TPortionInfo& portionInfo) { - return GetStorageAction(portionInfo.GetBlobsStorage()->GetStorageId()).GetRemoving(ConsumerId); -} - } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/action.h b/ydb/core/tx/columnshard/blobs_action/abstract/action.h index 21f756196fb5..27a8e001fdbd 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/action.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/action.h @@ -95,6 +95,14 @@ class TBlobsAction { } + TString GetStorageIds() const { + TStringBuilder sb; + for (auto&& i : StorageActions) { + sb << i.first << ","; + } + return sb; + } + ui32 GetWritingBlobsCount() const { ui32 result = 0; for (auto&& [_, action] : StorageActions) { @@ -160,20 +168,14 @@ class TBlobsAction { return GetStorageAction(storageId).GetRemoving(ConsumerId); } - std::shared_ptr GetRemoving(const TPortionInfo& portionInfo); - std::shared_ptr GetWriting(const TString& storageId) { return GetStorageAction(storageId).GetWriting(ConsumerId); } - std::shared_ptr GetWriting(const TPortionInfo& portionInfo); - std::shared_ptr GetReading(const TString& storageId) { return GetStorageAction(storageId).GetReading(ConsumerId); } - std::shared_ptr GetReading(const TPortionInfo& portionInfo); - }; } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h b/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h index febf2ca72e83..231bc7271dda 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h @@ -48,6 +48,10 @@ class TTabletsByBlob { private: THashMap> Data; public: + void Clear() { + Data.clear(); + } + NKikimrColumnShardBlobOperationsProto::TTabletsByBlob SerializeToProto() const; TConclusionStatus DeserializeFromProto(const NKikimrColumnShardBlobOperationsProto::TTabletsByBlob& proto); diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/read.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/read.cpp index 766dd21da79d..2e03ad225335 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/read.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/read.cpp @@ -1,57 +1,40 @@ #include "read.h" #include +#include namespace NKikimr::NOlap { -void IBlobsReadingAction::StartReading(THashMap>&& ranges) { +void IBlobsReadingAction::StartReading(THashSet&& ranges) { AFL_VERIFY(ranges.size()); AFL_VERIFY(Counters); for (auto&& i : ranges) { - AFL_VERIFY(i.second.size()); - for (auto&& br : i.second) { - Counters->OnRequest(br.Size); - } + Counters->OnRequest(i.Size); } - return DoStartReading(ranges); -} - -void IBlobsReadingAction::ExtractBlobsDataTo(THashMap& result) { - AFL_VERIFY(Started); - if (result.empty()) { - std::swap(result, Replies); - } else { - for (auto&& i : Replies) { - AFL_VERIFY(result.emplace(i.first, std::move(i.second)).second); - } - Replies.clear(); - } - RangesForResult.clear(); + return DoStartReading(std::move(ranges)); } void IBlobsReadingAction::Start(const THashSet& rangesInProgress) { Y_ABORT_UNLESS(!Started); + Started = true; + Y_ABORT_UNLESS(RangesForRead.size() + RangesForResult.size()); StartWaitingRanges = TMonotonic::Now(); - for (auto&& i : RangesForRead) { - WaitingRangesCount += i.second.size(); - } - THashMap> rangesFiltered; + WaitingRangesCount = RangesForRead.size(); + THashSet rangesFiltered; if (rangesInProgress.empty()) { rangesFiltered = RangesForRead; } else { - for (auto&& i : RangesForRead) { - for (auto&& r : i.second) { - if (!rangesInProgress.contains(r)) { - rangesFiltered[r.BlobId].emplace(r); - } + for (auto&& r : RangesForRead) { + if (!rangesInProgress.contains(r)) { + rangesFiltered.emplace(r); } } } if (rangesFiltered.size()) { StartReading(std::move(rangesFiltered)); } - Started = true; for (auto&& i : RangesForResult) { + AFL_VERIFY(i.second.size() == i.first.Size); AFL_VERIFY(Replies.emplace(i.first, i.second).second); } } @@ -60,6 +43,7 @@ void IBlobsReadingAction::OnReadResult(const TBlobRange& range, const TString& d AFL_VERIFY(Counters); AFL_VERIFY(--WaitingRangesCount >= 0); Counters->OnReply(range.Size, TMonotonic::Now() - StartWaitingRanges); + AFL_VERIFY(data.size() == range.Size); Replies.emplace(range, data); } @@ -70,13 +54,23 @@ void IBlobsReadingAction::OnReadError(const TBlobRange& range, const TErrorStatu Fails.emplace(range, replyStatus); } -void IBlobsReadingAction::AddRange(const TBlobRange& range, const TString& result /*= Default()*/) { +void IBlobsReadingAction::AddRange(const TBlobRange& range, const std::optional& result /*= {}*/) { Y_ABORT_UNLESS(!Started); if (!result) { - AFL_VERIFY(RangesForRead[range.BlobId].emplace(range).second)("range", range.ToString()); + AFL_VERIFY(!RangesForResult.contains(range)); + AFL_VERIFY(RangesForRead.emplace(range).second)("range", range.ToString()); } else { - AFL_VERIFY(RangesForResult.emplace(range, result).second)("range", range.ToString()); + AFL_VERIFY(result->size() == range.Size); + AFL_VERIFY(RangesForResult.emplace(range, *result).second)("range", range.ToString()); + } +} + +TString TActionReadBlobs::DebugString() const { + THashSet ranges; + for (auto&& i : Blobs) { + ranges.emplace(i.first); } + return JoinSeq(",", ranges); } } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/read.h b/ydb/core/tx/columnshard/blobs_action/abstract/read.h index 06a04c13c521..67f224392582 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/read.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/read.h @@ -8,13 +8,76 @@ namespace NKikimr::NOlap { +class TActionReadBlobs { +private: + THashMap Blobs; +public: + TString DebugString() const; + + TActionReadBlobs() = default; + + TActionReadBlobs(THashMap&& blobs) + : Blobs(std::move(blobs)) + { + for (auto&& i : Blobs) { + AFL_VERIFY(i.second.size()); + } + } + + void Merge(TActionReadBlobs&& item) { + for (auto&& i : item.Blobs) { + Add(i.first, std::move(i.second)); + } + } + + THashMap::iterator begin() { + return Blobs.begin(); + } + + THashMap::iterator end() { + return Blobs.end(); + } + + ui64 GetTotalBlobsSize() const { + ui64 result = 0; + for (auto&& i : Blobs) { + result += i.second.size(); + } + return result; + } + + void Add(THashMap&& blobs) { + for (auto&& i : blobs) { + AFL_VERIFY(i.second.size()); + AFL_VERIFY(Blobs.emplace(i.first, std::move(i.second)).second); + } + } + + void Add(const TBlobRange& range, TString&& data) { + AFL_VERIFY(data.size()); + AFL_VERIFY(Blobs.emplace(range, std::move(data)).second); + } + + TString Extract(const TBlobRange& bRange) { + auto it = Blobs.find(bRange); + AFL_VERIFY(it != Blobs.end()); + TString result = it->second; + Blobs.erase(it); + return result; + } + + bool IsEmpty() const { + return Blobs.empty(); + } +}; + class IBlobsReadingAction: public ICommonBlobsAction { public: using TErrorStatus = TConclusionSpecialStatus; private: using TBase = ICommonBlobsAction; - THashMap> RangesForRead; + THashSet RangesForRead; THashMap RangesForResult; TMonotonic StartWaitingRanges; i32 WaitingRangesCount = 0; @@ -22,12 +85,48 @@ class IBlobsReadingAction: public ICommonBlobsAction { THashMap Fails; std::shared_ptr Counters; bool Started = false; + bool DataExtracted = false; YDB_ACCESSOR(bool, IsBackgroundProcess, true); protected: - virtual void DoStartReading(const THashMap>& range) = 0; - void StartReading(THashMap>&& ranges); + virtual void DoStartReading(THashSet&& range) = 0; + void StartReading(THashSet&& ranges); public: + const THashSet& GetRangesForRead() const { + return RangesForRead; + } + + void Merge(const std::shared_ptr& action) { + AFL_VERIFY(action); + AFL_VERIFY(!Started); + for (auto&& i : action->RangesForResult) { + RangesForResult.emplace(i.first, i.second); + auto it = RangesForRead.find(i.first); + if (it != RangesForRead.end()) { + RangesForRead.erase(it); + } + } + for (auto&& i : action->RangesForResult) { + RangesForResult.emplace(i.first, i.second); + } + for (auto&& i : action->RangesForRead) { + if (!RangesForResult.contains(i)) { + RangesForRead.emplace(i); + } + } + } + + TActionReadBlobs ExtractBlobsData() { + AFL_VERIFY(Started); + AFL_VERIFY(IsFinished()); + AFL_VERIFY(!DataExtracted); + DataExtracted = true; + auto result = TActionReadBlobs(std::move(Replies)); + RangesForResult.clear(); + Replies.clear(); + return result; + } + void SetCounters(std::shared_ptr counters) { Counters = counters; } @@ -38,14 +137,10 @@ class IBlobsReadingAction: public ICommonBlobsAction { } - void ExtractBlobsDataTo(THashMap& result); - ui64 GetExpectedBlobsSize() const { ui64 result = 0; for (auto&& i : RangesForRead) { - for (auto&& b : i.second) { - result += b.Size; - } + result += i.Size; } for (auto&& i : RangesForResult) { result += i.first.Size; @@ -54,29 +149,10 @@ class IBlobsReadingAction: public ICommonBlobsAction { } ui64 GetExpectedBlobsCount() const { - ui64 result = 0; - for (auto&& i : RangesForRead) { - result += i.second.size(); - } - return result + RangesForResult.size(); - } - - void FillExpectedRanges(THashSet& ranges) const { - for (auto&& i : RangesForRead) { - for (auto&& b : i.second) { - Y_ABORT_UNLESS(ranges.emplace(b).second); - } - } - for (auto&& i : RangesForResult) { - Y_ABORT_UNLESS(ranges.emplace(i.first).second); - } - } - - const THashMap>& GetRangesForRead() const { - return RangesForRead; + return RangesForRead.size() + RangesForResult.size(); } - void AddRange(const TBlobRange& range, const TString& result = Default()); + void AddRange(const TBlobRange& range, const std::optional& result = {}); void Start(const THashSet& rangesInProgress); void OnReadResult(const TBlobRange& range, const TString& data); @@ -91,4 +167,46 @@ class IBlobsReadingAction: public ICommonBlobsAction { } }; +class TReadActionsCollection { +private: + THashMap> Actions; +public: + THashMap>::const_iterator begin() const { + return Actions.begin(); + } + + THashMap>::const_iterator end() const { + return Actions.end(); + } + + THashMap>::iterator begin() { + return Actions.begin(); + } + + THashMap>::iterator end() { + return Actions.end(); + } + + ui32 IsEmpty() const { + return Actions.empty(); + } + + void Add(const std::shared_ptr& action) { + auto it = Actions.find(action->GetStorageId()); + if (it == Actions.end()) { + Actions.emplace(action->GetStorageId(), action); + } else { + it->second->Merge(action); + } + } + + TReadActionsCollection() = default; + + TReadActionsCollection(const std::vector>& actions) { + for (auto&& a: actions) { + Add(a); + } + } +}; + } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp index 738f966bc381..9f9eb741b24e 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp @@ -5,13 +5,15 @@ namespace NKikimr::NOlap { std::shared_ptr IStoragesManager::GetOperatorVerified(const TString& storageId) { + AFL_VERIFY(Initialized); + AFL_VERIFY(storageId); TReadGuard rg(RWMutex); auto it = Constructed.find(storageId); - AFL_VERIFY(it != Constructed.end()); + AFL_VERIFY(it != Constructed.end())("storage_id", storageId); return it->second; } -std::shared_ptr IStoragesManager::GetOperator(const TString& storageId) { +std::shared_ptr IStoragesManager::GetOperatorGuarantee(const TString& storageId) { TReadGuard rg(RWMutex); auto it = Constructed.find(storageId); if (it == Constructed.end()) { @@ -26,26 +28,23 @@ std::shared_ptr IStoragesManager::GetOper return it->second; } -std::shared_ptr IStoragesManager::InitializePortionOperator(const TPortionInfo& portionInfo) { - Y_ABORT_UNLESS(!portionInfo.HasStorageOperator()); - if (portionInfo.GetMeta().GetTierName()) { - return GetOperator(portionInfo.GetMeta().GetTierName()); - } else { - return GetOperator(DefaultStorageId); - } +std::shared_ptr IStoragesManager::GetOperator(const TString& storageId) { + return GetOperatorGuarantee(storageId); } void IStoragesManager::OnTieringModified(const std::shared_ptr& tiers) { for (auto&& i : tiers->GetManagers()) { - GetOperator(i.second.GetTierName())->OnTieringModified(tiers); + GetOperatorGuarantee(i.second.GetTierName())->OnTieringModified(tiers); } } -void IStoragesManager::InitializeNecessaryStorages() { +void IStoragesManager::DoInitialize() { GetOperator(DefaultStorageId); + GetOperator(MemoryStorageId); } bool IStoragesManager::LoadIdempotency(NTable::TDatabase& database) { + AFL_VERIFY(Initialized); if (!DoLoadIdempotency(database)) { return false; } @@ -55,6 +54,8 @@ bool IStoragesManager::LoadIdempotency(NTable::TDatabase& database) { return false; } } + GetOperatorVerified(DefaultStorageId); + GetSharedBlobsManager()->GetStorageManagerVerified(DefaultStorageId); return true; } @@ -67,4 +68,19 @@ bool IStoragesManager::HasBlobsToDelete() const { return false; } +std::shared_ptr IStoragesManager::BuildOperator(const TString& storageId) { + auto result = DoBuildOperator(storageId); + AFL_VERIFY(result)("storage_id", storageId); + return result; +} + +void IStoragesManager::Stop() { + if (Initialized) { + for (auto&& i : Constructed) { + i.second->Stop(); + } + Initialized = false; + } +} + } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h index 128c3b0b3a77..cef1791747a3 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h @@ -8,34 +8,38 @@ class TPortionInfo; class IStoragesManager { private: TRWMutex RWMutex; + bool Initialized = false; protected: virtual std::shared_ptr DoBuildOperator(const TString& storageId) = 0; THashMap> Constructed; - std::shared_ptr BuildOperator(const TString& storageId) { - auto result = DoBuildOperator(storageId); - Y_ABORT_UNLESS(result); - return result; - } + std::shared_ptr BuildOperator(const TString& storageId); - virtual void InitializeNecessaryStorages(); + virtual void DoInitialize(); virtual bool DoLoadIdempotency(NTable::TDatabase& database) = 0; + virtual const std::shared_ptr& DoGetSharedBlobsManager() const = 0; public: static const inline TString DefaultStorageId = "__DEFAULT"; + static const inline TString MemoryStorageId = "__MEMORY"; virtual ~IStoragesManager() = default; + void Initialize() { + AFL_VERIFY(!Initialized); + Initialized = true; + DoInitialize(); + } + IStoragesManager() = default; - virtual const std::shared_ptr& GetSharedBlobsManager() const = 0; + const std::shared_ptr& GetSharedBlobsManager() const { + AFL_VERIFY(Initialized); + return DoGetSharedBlobsManager(); + } bool LoadIdempotency(NTable::TDatabase& database); bool HasBlobsToDelete() const; - void Stop() { - for (auto&& i : Constructed) { - i.second->Stop(); - } - } + void Stop(); std::shared_ptr GetDefaultOperator() { return GetOperator(DefaultStorageId); @@ -46,15 +50,15 @@ class IStoragesManager { } const THashMap>& GetStorages() { - InitializeNecessaryStorages(); + AFL_VERIFY(Initialized); return Constructed; } void OnTieringModified(const std::shared_ptr& tiers); std::shared_ptr GetOperator(const TString& storageIdExt); + std::shared_ptr GetOperatorGuarantee(const TString& storageIdExt); std::shared_ptr GetOperatorVerified(const TString& storageIdExt); - std::shared_ptr InitializePortionOperator(const TPortionInfo& portionInfo); }; diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/write.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/write.cpp index 60a3635e3fd1..19d360667185 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/write.cpp @@ -6,11 +6,18 @@ namespace NKikimr::NOlap { TUnifiedBlobId IBlobsWritingAction::AddDataForWrite(const TString& data) { Y_ABORT_UNLESS(!WritingStarted); auto blobId = AllocateNextBlobId(data); - AFL_VERIFY(BlobsForWrite.emplace(blobId, data).second); + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("generated_blob_id", blobId.ToStringNew()); + AddDataForWrite(blobId, data); + return blobId; +} + +void IBlobsWritingAction::AddDataForWrite(const TUnifiedBlobId& blobId, const TString& data) { + AFL_VERIFY(blobId.IsValid())("blob_id", blobId.ToStringNew()); + AFL_VERIFY(blobId.BlobSize() == data.size()); + AFL_VERIFY(BlobsForWrite.emplace(blobId, data).second)("blob_id", blobId.ToStringNew()); BlobsWaiting.emplace(blobId); BlobsWriteCount += 1; SumSize += data.size(); - return blobId; } void IBlobsWritingAction::OnBlobWriteResult(const TUnifiedBlobId& blobId, const NKikimrProto::EReplyStatus status) { diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/write.h b/ydb/core/tx/columnshard/blobs_action/abstract/write.h index 0599bcb459da..7d3992014453 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/write.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/write.h @@ -8,7 +8,6 @@ namespace NKikimr::NColumnShard { class TColumnShard; -class TBlobManagerDb; } namespace NKikimr::NOlap { @@ -24,6 +23,7 @@ class IBlobsWritingAction: public ICommonBlobsAction { THashSet BlobsWaiting; bool Aborted = false; std::shared_ptr Counters; + void AddDataForWrite(const TUnifiedBlobId& blobId, const TString& data); protected: virtual void DoOnExecuteTxBeforeWrite(NColumnShard::TColumnShard& self, TBlobManagerDb& dbBlobs) = 0; virtual void DoOnCompleteTxBeforeWrite(NColumnShard::TColumnShard& self) = 0; @@ -44,6 +44,14 @@ class IBlobsWritingAction: public ICommonBlobsAction { virtual ~IBlobsWritingAction(); bool IsReady() const; + void Merge(const std::shared_ptr& action) { + AFL_VERIFY(action); + AFL_VERIFY(!WritingStarted); + for (auto&& i : action->BlobsForWrite) { + AddDataForWrite(i.first, i.second); + } + } + void SetCounters(std::shared_ptr counters) { Counters = counters; } @@ -56,7 +64,6 @@ class IBlobsWritingAction: public ICommonBlobsAction { Aborted = true; } TUnifiedBlobId AddDataForWrite(const TString& data); - void OnBlobWriteResult(const TUnifiedBlobId& blobId, const NKikimrProto::EReplyStatus status); void OnExecuteTxBeforeWrite(NColumnShard::TColumnShard& self, TBlobManagerDb& dbBlobs) { @@ -87,4 +94,43 @@ class IBlobsWritingAction: public ICommonBlobsAction { void SendWriteBlobRequest(const TString& data, const TUnifiedBlobId& blobId); }; +class TWriteActionsCollection { +private: + THashMap> Actions; +public: + THashMap>::const_iterator begin() const { + return Actions.begin(); + } + + THashMap>::const_iterator end() const { + return Actions.end(); + } + + THashMap>::iterator begin() { + return Actions.begin(); + } + + THashMap>::iterator end() { + return Actions.end(); + } + + std::shared_ptr Add(const std::shared_ptr& action) { + auto it = Actions.find(action->GetStorageId()); + if (it == Actions.end()) { + return Actions.emplace(action->GetStorageId(), action).first->second; + } else if (action.get() != it->second.get()) { + it->second->Merge(action); + } + return it->second; + } + + TWriteActionsCollection() = default; + + TWriteActionsCollection(const std::vector>& actions) { + for (auto&& a : actions) { + Add(a); + } + } +}; + } diff --git a/ydb/core/tx/columnshard/blobs_action/bs/read.cpp b/ydb/core/tx/columnshard/blobs_action/bs/read.cpp index 93ed132aa4cc..93d6e45200c4 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/read.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/read.cpp @@ -1,15 +1,14 @@ #include "read.h" #include +#include namespace NKikimr::NOlap::NBlobOperations::NBlobStorage { -void TReadingAction::DoStartReading(const THashMap>& ranges) { - for (auto&& i : ranges) { - NBlobCache::TReadBlobRangeOptions readOpts{.CacheAfterRead = true, .IsBackgroud = GetIsBackgroundProcess(), .WithDeadline = false}; - std::vector rangesLocal(i.second.begin(), i.second.end()); - TActorContext::AsActorContext().Send(BlobCacheActorId, new NBlobCache::TEvBlobCache::TEvReadBlobRangeBatch(std::move(rangesLocal), std::move(readOpts))); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("blob_id", i.first)("count", i.second.size()); - } +void TReadingAction::DoStartReading(THashSet&& ranges) { + NBlobCache::TReadBlobRangeOptions readOpts{.CacheAfterRead = true, .IsBackgroud = GetIsBackgroundProcess(), .WithDeadline = false}; + std::vector rangesLocal(ranges.begin(), ranges.end()); + TActorContext::AsActorContext().Send(BlobCacheActorId, new NBlobCache::TEvBlobCache::TEvReadBlobRangeBatch(std::move(rangesLocal), std::move(readOpts))); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("blob_ids", JoinSeq(",", ranges))("count", ranges.size()); } } diff --git a/ydb/core/tx/columnshard/blobs_action/bs/read.h b/ydb/core/tx/columnshard/blobs_action/bs/read.h index 6361a47034b2..31a9dccb524f 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/read.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/read.h @@ -9,7 +9,7 @@ class TReadingAction: public IBlobsReadingAction { using TBase = IBlobsReadingAction; const TActorId BlobCacheActorId; protected: - virtual void DoStartReading(const THashMap>& ranges) override; + virtual void DoStartReading(THashSet&& ranges) override; public: TReadingAction(const TString& storageId, const TActorId& blobCacheActorId) diff --git a/ydb/core/tx/columnshard/blobs_action/memory.cpp b/ydb/core/tx/columnshard/blobs_action/memory.cpp deleted file mode 100644 index 66ff21fc5004..000000000000 --- a/ydb/core/tx/columnshard/blobs_action/memory.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "memory.h" - -namespace NKikimr::NOlap { - -} diff --git a/ydb/core/tx/columnshard/blobs_action/memory.h b/ydb/core/tx/columnshard/blobs_action/memory.h deleted file mode 100644 index 123a94ed2ccc..000000000000 --- a/ydb/core/tx/columnshard/blobs_action/memory.h +++ /dev/null @@ -1,172 +0,0 @@ -#pragma once - -#include -#include - -namespace NKikimr::NOlap { - -class TMemoryStorage { -private: - THashMap Data; - THashMap DataWriting; - THashSet DataForRemove; - TMutex Mutex; -public: - std::optional Read(const TUnifiedBlobId& id) { - TGuard g(Mutex); - auto it = Data.find(id); - if (it == Data.end()) { - return {}; - } else { - return it->second; - } - } - - void DeclareDataForRemove(const TTabletId /*tabletId*/, const TUnifiedBlobId& id) { - TGuard g(Mutex); - DataForRemove.emplace(id); - } - - void StartWriting(const TUnifiedBlobId& id, const TString& data) { - TGuard g(Mutex); - Y_ABORT_UNLESS(DataWriting.emplace(id, data).second); - } - - void CommitWriting(const TUnifiedBlobId& id) { - TGuard g(Mutex); - auto it = DataWriting.find(id); - Y_ABORT_UNLESS(it != DataWriting.end()); - Y_ABORT_UNLESS(Data.emplace(id, it->second).second); - DataWriting.erase(it); - } - - TMemoryStorage() = default; -}; - -class TMemoryWriteAction: public IBlobsWritingAction { -private: - using TBase = IBlobsWritingAction; - const std::shared_ptr Storage; -protected: - virtual void DoSendWriteBlobRequest(const TString& data, const TUnifiedBlobId& blobId) override { - Storage->StartWriting(blobId, data); - TActorContext::AsActorContext().Send(TActorContext::AsActorContext().SelfID, std::make_unique( - NKikimrProto::EReplyStatus::OK, blobId.GetLogoBlobId(), TStorageStatusFlags(), 0, 0)); - } - - virtual void DoOnBlobWriteResult(const TUnifiedBlobId& blobId, const NKikimrProto::EReplyStatus status) override { - Y_ABORT_UNLESS(status == NKikimrProto::EReplyStatus::OK); - Storage->CommitWriting(blobId); - } - - virtual void DoOnExecuteTxBeforeWrite(NColumnShard::TColumnShard& /*self*/, TBlobManagerDb& /*dbBlobs*/) override { - return; - } - - virtual void DoOnCompleteTxBeforeWrite(NColumnShard::TColumnShard& /*self*/) override { - return; - } - - virtual void DoOnExecuteTxAfterWrite(NColumnShard::TColumnShard& /*self*/, TBlobManagerDb& /*dbBlobs*/, const bool /*blobsWroteSuccessfully*/) override { - - } - virtual void DoOnCompleteTxAfterWrite(NColumnShard::TColumnShard& /*self*/, const bool /*blobsWroteSuccessfully*/) override { - - } -public: - virtual bool NeedDraftTransaction() const override { - return true; - } - - virtual TUnifiedBlobId AllocateNextBlobId(const TString& /*data*/) override { - return TUnifiedBlobId(); -// return BlobBatch.AllocateNextBlobId(data); - } - - TMemoryWriteAction(const TString& storageId, const std::shared_ptr& storage) - : TBase(storageId) - , Storage(storage) - { - - } -}; - -class TMemoryDeclareRemovingAction: public IBlobsDeclareRemovingAction { -private: - using TBase = IBlobsDeclareRemovingAction; - const std::shared_ptr Storage; -protected: - virtual void DoDeclareRemove(const TTabletId /*tabletId*/, const TUnifiedBlobId& /*blobId*/) { - - } - - virtual void DoOnExecuteTxAfterRemoving(NColumnShard::TColumnShard& /*self*/, TBlobManagerDb& /*dbBlobs*/, const bool /*blobsWroteSuccessfully*/) { - for (auto i = GetDeclaredBlobs().GetIterator(); i.IsValid(); ++i) { - Storage->DeclareDataForRemove(i.GetTabletId(), i.GetBlobId()); - } - } - virtual void DoOnCompleteTxAfterRemoving(NColumnShard::TColumnShard& /*self*/, const bool /*blobsWroteSuccessfully*/) { - - } -public: - - TMemoryDeclareRemovingAction(const TString& storageId, const TTabletId tabletId, const std::shared_ptr& counters, const std::shared_ptr& storage) - : TBase(storageId, tabletId, counters) - , Storage(storage) { - - } -}; - -class TMemoryReadingAction: public IBlobsReadingAction { -private: - using TBase = IBlobsReadingAction; - const std::shared_ptr Storage; -protected: - virtual void DoStartReading(const THashMap>& ranges) override { - for (auto&& i : ranges) { - auto data = Storage->Read(i.first); - for (auto&& r : i.second) { - if (!data) { - TActorContext::AsActorContext().Send(TActorContext::AsActorContext().SelfID, - new NBlobCache::TEvBlobCache::TEvReadBlobRangeResult(r, NKikimrProto::EReplyStatus::NODATA, "")); - } else { - Y_ABORT_UNLESS(r.Offset + r.Size <= data->size()); - TActorContext::AsActorContext().Send(TActorContext::AsActorContext().SelfID, - new NBlobCache::TEvBlobCache::TEvReadBlobRangeResult(r, NKikimrProto::EReplyStatus::OK, data->substr(r.Offset, r.Size))); - } - } - } - } -public: - - TMemoryReadingAction(const TString& storageId, const std::shared_ptr& storage) - : TBase(storageId) - , Storage(storage) - { - - } -}; - -class TMemoryOperator: public IBlobsStorageOperator { -private: - using TBase = IBlobsStorageOperator; - std::shared_ptr Storage; -protected: - virtual std::shared_ptr DoStartDeclareRemovingAction(const std::shared_ptr& counters) override { - return std::make_shared(GetStorageId(), GetSelfTabletId(), counters, Storage); - } - virtual std::shared_ptr DoStartWritingAction() override { - return std::make_shared(GetStorageId(), Storage); - } - virtual std::shared_ptr DoStartReadingAction() override { - return std::make_shared(GetStorageId(), Storage); - } -public: - TMemoryOperator(const TString& storageId, const std::shared_ptr& sharedBlobs) - : TBase(storageId, sharedBlobs) - { - Storage = std::make_shared(); - } -}; - -} diff --git a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp index 5c80c3a1d4a1..aea4f15aa5bc 100644 --- a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp @@ -4,6 +4,8 @@ #ifndef KIKIMR_DISABLE_S3_OPS #include #endif +#include +#include namespace NKikimr::NOlap { @@ -11,6 +13,14 @@ std::shared_ptr TStoragesManager::DoBuild if (storageId == TBase::DefaultStorageId) { return std::make_shared(storageId, Shard.SelfId(), Shard.Info(), Shard.Executor()->Generation(), SharedBlobsManager->GetStorageManagerGuarantee(storageId)); + } else if (storageId == TBase::MemoryStorageId) { + { + static TMutex mutexLocal; + TGuard g(mutexLocal); + Singleton()->SetSecretKey("fakeSecret"); + } + return std::make_shared(storageId, Shard.SelfId(), std::make_shared("fakeBucket", "fakeSecret"), + SharedBlobsManager->GetStorageManagerGuarantee(storageId)); } else if (!Shard.Tiers) { return nullptr; } else { @@ -23,12 +33,7 @@ std::shared_ptr TStoragesManager::DoBuild } bool TStoragesManager::DoLoadIdempotency(NTable::TDatabase& database) { - std::shared_ptr local = std::make_shared((TTabletId)Shard.TabletID()); - if (!local->Load(database)) { - return false; - } - SharedBlobsManager = local; - return true; + return SharedBlobsManager->LoadIdempotency(database); } TStoragesManager::TStoragesManager(NColumnShard::TColumnShard& shard) diff --git a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.h b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.h index f28c9ac9783e..00828c89a713 100644 --- a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.h +++ b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.h @@ -18,12 +18,12 @@ class TStoragesManager: public IStoragesManager { virtual std::shared_ptr DoBuildOperator(const TString& storageId) override; virtual bool DoLoadIdempotency(NTable::TDatabase& database) override; -public: - virtual const std::shared_ptr& GetSharedBlobsManager() const override { + virtual const std::shared_ptr& DoGetSharedBlobsManager() const override { AFL_VERIFY(SharedBlobsManager); return SharedBlobsManager; } +public: TStoragesManager(NColumnShard::TColumnShard& shard); }; diff --git a/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp b/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp index 97b8a307e10d..9d94dafe939d 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp @@ -16,9 +16,9 @@ std::unique_ptr TRepliesAdapter::RebuildReplyEvent(std::uni } if (ev->IsSuccess()) { AFL_VERIFY(!!ev->Body)("key", ev->Key)("interval_from", ev->GetReadInterval().first)("interval_to", ev->GetReadInterval().second); - return std::make_unique(bRange, NKikimrProto::EReplyStatus::OK, ev->Body); + return std::make_unique(bRange, NKikimrProto::EReplyStatus::OK, ev->Body, false, StorageId); } else { - return std::make_unique(bRange, NKikimrProto::EReplyStatus::ERROR, TStringBuilder() << ev->Result); + return std::make_unique(bRange, NKikimrProto::EReplyStatus::ERROR, TStringBuilder() << ev->Result, false, StorageId); } } @@ -28,9 +28,9 @@ std::unique_ptr TRepliesAdapter::RebuildReplyEvent(std::uni Y_ABORT_UNLESS(ev->Key); AFL_VERIFY(TLogoBlobID::Parse(logoBlobId, *ev->Key, error))("error", error)("str_blob_id", *ev->Key); if (ev->IsSuccess()) { - return std::make_unique(NKikimrProto::EReplyStatus::OK, logoBlobId, 0, Max(), 0); + return std::make_unique(NKikimrProto::EReplyStatus::OK, logoBlobId, 0, Max(), 0, StorageId); } else { - return std::make_unique(NKikimrProto::EReplyStatus::ERROR, logoBlobId, 0, Max(), 0); + return std::make_unique(NKikimrProto::EReplyStatus::ERROR, logoBlobId, 0, Max(), 0, StorageId); } } diff --git a/ydb/core/tx/columnshard/blobs_action/tier/adapter.h b/ydb/core/tx/columnshard/blobs_action/tier/adapter.h index 1c3e6af2e88e..4e146e019137 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/adapter.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/adapter.h @@ -4,7 +4,15 @@ namespace NKikimr::NOlap::NBlobOperations::NTier { class TRepliesAdapter: public NWrappers::NExternalStorage::IReplyAdapter { +private: + const TString StorageId; public: + TRepliesAdapter(const TString& storageId) + : StorageId(storageId) + { + + } + virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override; virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override; virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { diff --git a/ydb/core/tx/columnshard/blobs_action/tier/read.cpp b/ydb/core/tx/columnshard/blobs_action/tier/read.cpp index 754dd289d730..7af0c28583af 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/read.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/read.cpp @@ -2,17 +2,15 @@ namespace NKikimr::NOlap::NBlobOperations::NTier { -void TReadingAction::DoStartReading(const THashMap>& ranges) { - for (auto&& i : ranges) { - for (auto&& r : i.second) { - auto awsRequest = Aws::S3::Model::GetObjectRequest() - .WithKey(i.first.GetLogoBlobId().ToString()) - .WithRange(TStringBuilder() << "bytes=" << r.Offset << "-" << r.Offset + r.Size - 1); - auto request = std::make_unique(awsRequest); - auto hRequest = std::make_unique(NActors::TActorId(), TActorContext::AsActorContext().SelfID, request.release()); - TAutoPtr> evPtr((TEventHandle*)hRequest.release()); - ExternalStorageOperator->Execute(evPtr); - } +void TReadingAction::DoStartReading(THashSet&& ranges) { + for (auto&& r : ranges) { + auto awsRequest = Aws::S3::Model::GetObjectRequest() + .WithKey(r.BlobId.GetLogoBlobId().ToString()) + .WithRange(TStringBuilder() << "bytes=" << r.Offset << "-" << r.Offset + r.Size - 1); + auto request = std::make_unique(awsRequest); + auto hRequest = std::make_unique(NActors::TActorId(), TActorContext::AsActorContext().SelfID, request.release()); + TAutoPtr> evPtr((TEventHandle*)hRequest.release()); + ExternalStorageOperator->Execute(evPtr); } } diff --git a/ydb/core/tx/columnshard/blobs_action/tier/read.h b/ydb/core/tx/columnshard/blobs_action/tier/read.h index b9804d18b662..3e7baee57c3d 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/read.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/read.h @@ -10,7 +10,7 @@ class TReadingAction: public IBlobsReadingAction { using TBase = IBlobsReadingAction; const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr ExternalStorageOperator; protected: - virtual void DoStartReading(const THashMap>& ranges) override; + virtual void DoStartReading(THashSet&& ranges) override; public: TReadingAction(const TString& storageId, const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& storageOperator) diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp index a3e5e2b63c71..80be5a84f3c8 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp @@ -52,7 +52,15 @@ void TOperator::InitNewExternalOperator(const NColumnShard::NTiers::TManager* ti auto extStorageConfig = NWrappers::NExternalStorage::IExternalStorageConfig::Construct(settings); AFL_VERIFY(extStorageConfig); auto extStorageOperator = extStorageConfig->ConstructStorageOperator(false); - extStorageOperator->InitReplyAdapter(std::make_shared()); + extStorageOperator->InitReplyAdapter(std::make_shared(GetStorageId())); + TGuard changeLock(ChangeOperatorLock); + ExternalStorageOperator = extStorageOperator; +} + +void TOperator::InitNewExternalOperator() { + AFL_VERIFY(InitializationConfig); + auto extStorageOperator = InitializationConfig->ConstructStorageOperator(false); + extStorageOperator->InitReplyAdapter(std::make_shared(GetStorageId())); TGuard changeLock(ChangeOperatorLock); ExternalStorageOperator = extStorageOperator; } @@ -64,6 +72,15 @@ TOperator::TOperator(const TString& storageId, const NColumnShard::TColumnShard& InitNewExternalOperator(shard.GetTierManagerPointer(storageId)); } +TOperator::TOperator(const TString& storageId, const TActorId& shardActorId, const std::shared_ptr& storageConfig, + const std::shared_ptr& storageSharedBlobsManager) + : TBase(storageId, storageSharedBlobsManager) + , TabletActorId(shardActorId) + , InitializationConfig(storageConfig) +{ + InitNewExternalOperator(); +} + void TOperator::DoOnTieringModified(const std::shared_ptr& tiers) { auto* tierManager = tiers->GetManagerOptional(TBase::GetStorageId()); if (tierManager) { diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.h b/ydb/core/tx/columnshard/blobs_action/tier/storage.h index d45864f93fdd..c5118a9adcd6 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.h @@ -14,12 +14,14 @@ class TOperator: public IBlobsStorageOperator { const NActors::TActorId TabletActorId; std::shared_ptr GCInfo = std::make_shared(); + NWrappers::NExternalStorage::IExternalStorageConfig::TPtr InitializationConfig; NWrappers::NExternalStorage::IExternalStorageConfig::TPtr ExternalStorageConfig; TSpinLock ChangeOperatorLock; NWrappers::NExternalStorage::IExternalStorageOperator::TPtr ExternalStorageOperator; NWrappers::NExternalStorage::IExternalStorageOperator::TPtr GetCurrentOperator() const; void InitNewExternalOperator(const NColumnShard::NTiers::TManager* tierManager); + void InitNewExternalOperator(); virtual TString DoDebugString() const override { return GetCurrentOperator()->DebugString(); @@ -34,6 +36,8 @@ class TOperator: public IBlobsStorageOperator { public: TOperator(const TString& storageId, const NColumnShard::TColumnShard& shard, const std::shared_ptr& storageSharedBlobsManager); + TOperator(const TString& storageId, const TActorId& shardActorId, const std::shared_ptr& storageConfig, + const std::shared_ptr& storageSharedBlobsManager); virtual TTabletsByBlob GetBlobsToDelete() const override { auto result = GCInfo->GetBlobsToDelete(); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_draft.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_draft.cpp index ebbbb35568c8..2c212a5fc9fb 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_draft.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_draft.cpp @@ -5,14 +5,14 @@ namespace NKikimr::NColumnShard { bool TTxWriteDraft::Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) { NOlap::TBlobManagerDb blobManagerDb(txc.DB); for (auto&& action : WriteController->GetBlobActions()) { - action->OnExecuteTxBeforeWrite(*Self, blobManagerDb); + action.second->OnExecuteTxBeforeWrite(*Self, blobManagerDb); } return true; } void TTxWriteDraft::Complete(const TActorContext& ctx) { for (auto&& action : WriteController->GetBlobActions()) { - action->OnCompleteTxBeforeWrite(*Self); + action.second->OnCompleteTxBeforeWrite(*Self); } ctx.Register(NColumnShard::CreateWriteActor(Self->TabletID(), WriteController, TInstant::Max())); } diff --git a/ydb/core/tx/columnshard/blobs_action/ya.make b/ydb/core/tx/columnshard/blobs_action/ya.make index e40c02ad703d..12470a0aa080 100644 --- a/ydb/core/tx/columnshard/blobs_action/ya.make +++ b/ydb/core/tx/columnshard/blobs_action/ya.make @@ -2,7 +2,6 @@ LIBRARY() SRCS( blob_manager_db.cpp - memory.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/blobs_reader/actor.cpp b/ydb/core/tx/columnshard/blobs_reader/actor.cpp index c934ddef6d1a..3d27b9d8a0e8 100644 --- a/ydb/core/tx/columnshard/blobs_reader/actor.cpp +++ b/ydb/core/tx/columnshard/blobs_reader/actor.cpp @@ -14,13 +14,13 @@ void TActor::Handle(NBlobCache::TEvBlobCache::TEvReadBlobRangeResult::TPtr& ev) bool aborted = false; if (event.Status != NKikimrProto::EReplyStatus::OK) { - WaitingBlobsCount.Sub(Task->GetWaitingCount()); - if (!Task->AddError(event.BlobRange, IBlobsReadingAction::TErrorStatus::Fail(event.Status, "cannot get blob: " + event.Data.substr(0, 1024)))) { + WaitingBlobsCount.Sub(Task->GetWaitingRangesCount()); + if (!Task->AddError(event.DataSourceId, event.BlobRange, IBlobsReadingAction::TErrorStatus::Fail(event.Status, "cannot get blob: " + event.Data.substr(0, 1024)))) { aborted = true; } } else { WaitingBlobsCount.Dec(); - Task->AddData(event.BlobRange, event.Data); + Task->AddData(event.DataSourceId, event.BlobRange, event.Data); } if (aborted || Task->IsFinished()) { Task = nullptr; @@ -46,8 +46,7 @@ void TActor::Bootstrap() { NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("external_task_id", externalTaskId); Task->StartBlobsFetching({}); ACFL_DEBUG("task", Task->DebugString()); - WaitingBlobsCount.Add(Task->GetReadRangesCount()); - AFL_VERIFY(Task->GetAllRangesSize()); + WaitingBlobsCount.Add(Task->GetWaitingRangesCount()); Become(&TThis::StateWait); if (Task->IsFinished()) { PassAway(); diff --git a/ydb/core/tx/columnshard/blobs_reader/read_coordinator.cpp b/ydb/core/tx/columnshard/blobs_reader/read_coordinator.cpp index 5b6a255faffb..1901a7495972 100644 --- a/ydb/core/tx/columnshard/blobs_reader/read_coordinator.cpp +++ b/ydb/core/tx/columnshard/blobs_reader/read_coordinator.cpp @@ -8,42 +8,24 @@ void TReadCoordinatorActor::Handle(TEvStartReadTask::TPtr& ev) { const auto& externalTaskId = ev->Get()->GetTask()->GetExternalTaskId(); NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("external_task_id", externalTaskId); THashSet rangesInProgress; - for (auto&& agent : ev->Get()->GetTask()->GetAgents()) { - for (auto&& b : agent->GetRangesForRead()) { - for (auto&& r : b.second) { - auto it = BlobTasks.find(r); - if (it != BlobTasks.end()) { - ACFL_DEBUG("event", "TEvReadTask")("enqueued_blob_id", r); - rangesInProgress.emplace(r); - } else { - ACFL_DEBUG("event", "TEvReadTask")("blob_id", r); - it = BlobTasks.emplace(r, std::vector>()).first; - WaitingBlobsCount.Inc(); - } - it->second.emplace_back(ev->Get()->GetTask()); - } - } - } + BlobTasks.AddTask(ev->Get()->GetTask()); ev->Get()->GetTask()->StartBlobsFetching(rangesInProgress); ACFL_DEBUG("task", ev->Get()->GetTask()->DebugString()); - AFL_VERIFY(ev->Get()->GetTask()->GetAllRangesSize()); } void TReadCoordinatorActor::Handle(NBlobCache::TEvBlobCache::TEvReadBlobRangeResult::TPtr& ev) { ACFL_TRACE("event", "TEvReadBlobRangeResult")("blob_id", ev->Get()->BlobRange); auto& event = *ev->Get(); - auto it = BlobTasks.find(event.BlobRange); - AFL_VERIFY(it != BlobTasks.end())("blob_id", event.BlobRange); - for (auto&& i : it->second) { + auto tasks = BlobTasks.Extract(event.DataSourceId, event.BlobRange); + for (auto&& i : tasks) { if (event.Status != NKikimrProto::EReplyStatus::OK) { - i->AddError(event.BlobRange, IBlobsReadingAction::TErrorStatus::Fail(event.Status, "cannot get blob")); + i->AddError(event.DataSourceId, event.BlobRange, IBlobsReadingAction::TErrorStatus::Fail(event.Status, "cannot get blob")); } else { - i->AddData(event.BlobRange, event.Data); + i->AddData(event.DataSourceId, event.BlobRange, event.Data); } } WaitingBlobsCount.Dec(); - BlobTasks.erase(it); } TReadCoordinatorActor::TReadCoordinatorActor(ui64 tabletId, const TActorId& parent) @@ -53,10 +35,9 @@ TReadCoordinatorActor::TReadCoordinatorActor(ui64 tabletId, const TActorId& pare } TReadCoordinatorActor::~TReadCoordinatorActor() { - for (auto&& i : BlobTasks) { - for (auto&& t : i.second) { - t->Abort(); - } + auto tasks = BlobTasks.ExtractTasksAll(); + for (auto&& i : tasks) { + i->Abort(); } } diff --git a/ydb/core/tx/columnshard/blobs_reader/read_coordinator.h b/ydb/core/tx/columnshard/blobs_reader/read_coordinator.h index e89a29aee1b0..d9e311815cdb 100644 --- a/ydb/core/tx/columnshard/blobs_reader/read_coordinator.h +++ b/ydb/core/tx/columnshard/blobs_reader/read_coordinator.h @@ -12,11 +12,53 @@ namespace NKikimr::NOlap::NBlobOperations::NRead { +class TBlobsForRead { +private: + THashMap>>> BlobTasks; +public: + std::vector> ExtractTasksAll() { + THashMap> tasks; + for (auto&& i : BlobTasks) { + for (auto&& r : i.second) { + for (auto&& t : r.second) { + tasks.emplace(t->GetTaskIdentifier(), t); + } + } + } + std::vector> result; + for (auto&& i : tasks) { + result.emplace_back(i.second); + } + return result; + } + + std::vector> Extract(const TString& storageId, const TBlobRange& bRange) { + auto it = BlobTasks.find(storageId); + AFL_VERIFY(it != BlobTasks.end()); + auto itBlobRange = it->second.find(bRange); + auto result = std::move(itBlobRange->second); + it->second.erase(itBlobRange); + if (it->second.empty()) { + BlobTasks.erase(it); + } + return result; + } + + void AddTask(const std::shared_ptr& task) { + for (auto&& i : task->GetAgents()) { + auto& storage = BlobTasks[i.second->GetStorageId()]; + for (auto&& bRid : i.second->GetRangesForRead()) { + storage[bRid].emplace_back(task); + } + } + } +}; + class TReadCoordinatorActor: public NActors::TActorBootstrapped { private: ui64 TabletId; NActors::TActorId Parent; - THashMap>> BlobTasks; + TBlobsForRead BlobTasks; public: static TAtomicCounter WaitingBlobsCount; TReadCoordinatorActor(ui64 tabletId, const TActorId& parent); @@ -42,4 +84,4 @@ class TReadCoordinatorActor: public NActors::TActorBootstrapped>& ITask::GetAgents() const { - Y_ABORT_UNLESS(!BlobsFetchingStarted); - return Agents; -} - -bool ITask::AddError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { - ++BlobErrorsCount; +bool ITask::AddError(const TString& storageIdExt, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { + const TString storageId = storageIdExt ? storageIdExt : IStoragesManager::DefaultStorageId; + AFL_VERIFY(--BlobsWaitingCount >= 0); if (TaskFinishedWithError || AbortFlag) { - ACFL_WARN("event", "SkipError")("blob_range", range)("message", status.GetErrorMessage())("status", status.GetStatus())("external_task_id", ExternalTaskId)("consumer", TaskCustomer) + ACFL_WARN("event", "SkipError")("storage_id", storageId)("blob_range", range)("message", status.GetErrorMessage())("status", status.GetStatus())("external_task_id", ExternalTaskId)("consumer", TaskCustomer) ("abort", AbortFlag)("finished_with_error", TaskFinishedWithError); return false; } else { - ACFL_ERROR("event", "NewError")("blob_range", range)("message", status.GetErrorMessage())("status", status.GetStatus())("external_task_id", ExternalTaskId)("consumer", TaskCustomer); + ACFL_ERROR("event", "NewError")("storage_id", storageId)("blob_range", range)("message", status.GetErrorMessage())("status", status.GetStatus())("external_task_id", ExternalTaskId)("consumer", TaskCustomer); } { - auto it = BlobsWaiting.find(range); - AFL_VERIFY(it != BlobsWaiting.end()); + auto it = AgentsWaiting.find(storageId); + AFL_VERIFY(it != AgentsWaiting.end())("storage_id", storageId); it->second->OnReadError(range, status); - BlobsWaiting.erase(it); + if (it->second->IsFinished()) { + AgentsWaiting.erase(it); + } } - if (!OnError(range, status)) { + if (!OnError(storageId, range, status)) { TaskFinishedWithError = true; return false; } - if (BlobsWaiting.empty()) { + if (AgentsWaiting.empty()) { OnDataReady(); } return true; } -void ITask::AddData(const TBlobRange& range, const TString& data) { - ++BlobsDataCount; +void ITask::AddData(const TString& storageIdExt, const TBlobRange& range, const TString& data) { + const TString storageId = storageIdExt ? storageIdExt : IStoragesManager::DefaultStorageId; + AFL_VERIFY(--BlobsWaitingCount >= 0); if (TaskFinishedWithError || AbortFlag) { - ACFL_WARN("event", "SkipDataAfterError")("external_task_id", ExternalTaskId)("abort", AbortFlag)("finished_with_error", TaskFinishedWithError); + ACFL_WARN("event", "SkipDataAfterError")("storage_id", storageId)("external_task_id", ExternalTaskId)("abort", AbortFlag)("finished_with_error", TaskFinishedWithError); return; } else { - ACFL_TRACE("event", "NewData")("range", range.ToString())("external_task_id", ExternalTaskId); + ACFL_TRACE("event", "NewData")("storage_id", storageId)("range", range.ToString())("external_task_id", ExternalTaskId); } Y_ABORT_UNLESS(BlobsFetchingStarted); { - auto it = BlobsWaiting.find(range); - AFL_VERIFY(it != BlobsWaiting.end()); + auto it = AgentsWaiting.find(storageId); + AFL_VERIFY(it != AgentsWaiting.end())("storage_id", storageId); it->second->OnReadResult(range, data); - BlobsWaiting.erase(it); + if (it->second->IsFinished()) { + AgentsWaiting.erase(it); + } } - if (BlobsWaiting.empty()) { + if (AgentsWaiting.empty()) { OnDataReady(); } } @@ -59,24 +60,15 @@ void ITask::StartBlobsFetching(const THashSet& rangesInProgress) { ACFL_TRACE("task_id", ExternalTaskId)("event", "start"); Y_ABORT_UNLESS(!BlobsFetchingStarted); BlobsFetchingStarted = true; - ui64 allRangesSize = 0; - ui64 allRangesCount = 0; - ui64 readRangesCount = 0; + AFL_VERIFY(BlobsWaitingCount == 0); for (auto&& agent : Agents) { - allRangesSize += agent->GetExpectedBlobsSize(); - allRangesCount += agent->GetExpectedBlobsCount(); - for (auto&& b : agent->GetRangesForRead()) { - for (auto&& r : b.second) { - BlobsWaiting.emplace(r, agent); - ++readRangesCount; - } + agent.second->Start(rangesInProgress); + if (!agent.second->IsFinished()) { + AgentsWaiting.emplace(agent.second->GetStorageId(), agent.second); + BlobsWaitingCount += agent.second->GetRangesForRead().size(); } - agent->Start(rangesInProgress); } - ReadRangesCount = readRangesCount; - AllRangesCount = allRangesCount; - AllRangesSize = allRangesSize; - if (BlobsWaiting.empty()) { + if (AgentsWaiting.empty()) { OnDataReady(); } } @@ -91,15 +83,15 @@ void ITask::TReadSubscriber::DoOnAllocationSuccess(const std::shared_ptr>& actions, const TString& taskCustomer, const TString& externalTaskId) - : Agents(actions) - , TaskIdentifier(TaskIdentifierBuilder.Inc()) +ITask::ITask(const TReadActionsCollection& actions, const TString& taskCustomer, const TString& externalTaskId) + : TaskIdentifier(TaskIdentifierBuilder.Inc()) , ExternalTaskId(externalTaskId) , TaskCustomer(taskCustomer) { - AFL_VERIFY(Agents.size()); + Agents = actions; + AFL_VERIFY(!Agents.IsEmpty()); for (auto&& i : Agents) { - AFL_VERIFY(i->GetExpectedBlobsCount()); + AFL_VERIFY(i.second->GetExpectedBlobsCount()); } } @@ -108,11 +100,7 @@ TString ITask::DebugString() const { if (TaskFinishedWithError) { sb << "finished_with_error=" << TaskFinishedWithError << ";"; } - if (BlobErrorsCount) { - sb << "blob_errors=" << BlobErrorsCount << ";"; - } - sb << "data=" << BlobsDataCount << ";" - << "waiting=" << BlobsWaiting.size() << ";" + sb << "agents_waiting=" << AgentsWaiting.size() << ";" << "additional_info=(" << DoDebugString() << ");" ; return sb; @@ -125,22 +113,22 @@ void ITask::OnDataReady() { DoOnDataReady(ResourcesGuard); } -bool ITask::OnError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { +bool ITask::OnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { ACFL_DEBUG("event", "OnError")("status", status.GetStatus())("task", DebugString()); - return DoOnError(range, status); + return DoOnError(storageId, range, status); } ITask::~ITask() { AFL_VERIFY(!NActors::TlsActivationContext || DataIsReadyFlag || TaskFinishedWithError || AbortFlag || !BlobsFetchingStarted); } -THashMap ITask::ExtractBlobsData() { - AFL_VERIFY(BlobsWaiting.empty()); +TCompositeReadBlobs ITask::ExtractBlobsData() { + AFL_VERIFY(AgentsWaiting.empty()); AFL_VERIFY(!ResultsExtracted); ResultsExtracted = true; - THashMap result; + TCompositeReadBlobs result; for (auto&& i : Agents) { - i->ExtractBlobsDataTo(result); + result.Add(i.second->GetStorageId(), i.second->ExtractBlobsData()); } return std::move(result); } diff --git a/ydb/core/tx/columnshard/blobs_reader/task.h b/ydb/core/tx/columnshard/blobs_reader/task.h index 65525b03f434..bc2796e9dda1 100644 --- a/ydb/core/tx/columnshard/blobs_reader/task.h +++ b/ydb/core/tx/columnshard/blobs_reader/task.h @@ -9,47 +9,106 @@ namespace NKikimr::NOlap::NBlobOperations::NRead { +class TCompositeReadBlobs { +private: + THashMap BlobsByStorage; +public: + TString DebugString() const { + TStringBuilder sb; + sb << "{"; + for (auto&& i : BlobsByStorage) { + sb << "{storage_id:" << i.first << ";blobs:" << i.second.DebugString() << "};"; + } + sb << "}"; + return sb; + } + + void Merge(TCompositeReadBlobs&& blobs) { + for (auto&& i : blobs.BlobsByStorage) { + BlobsByStorage[i.first].Merge(std::move(i.second)); + } + } + + void Clear() { + BlobsByStorage.clear(); + } + + bool IsEmpty() const { + return BlobsByStorage.empty(); + } + + THashMap::iterator begin() { + return BlobsByStorage.begin(); + } + THashMap::iterator end() { + return BlobsByStorage.end(); + } + void Add(const TString& storageId, TActionReadBlobs&& data) { + AFL_VERIFY(BlobsByStorage.emplace(storageId, std::move(data)).second); + } + void Add(const TString& storageId, const TBlobRange& blobId, TString&& value) { + BlobsByStorage[storageId].Add(blobId, std::move(value)); + } + TString Extract(const TString& storageId, const TBlobRange& range) { + auto it = BlobsByStorage.find(storageId); + AFL_VERIFY(it != BlobsByStorage.end()); + auto result = it->second.Extract(range); + if (it->second.IsEmpty()) { + BlobsByStorage.erase(it); + } + return result; + } + + ui64 GetTotalBlobsSize() const { + ui64 result = 0; + for (auto&& i : BlobsByStorage) { + result += i.second.GetTotalBlobsSize(); + } + return result; + } +}; + class ITask: public NColumnShard::TMonitoringObjectsCounter { private: - THashMap> BlobsWaiting; - std::vector> Agents; + THashMap> AgentsWaiting; + YDB_READONLY_DEF(TReadActionsCollection, Agents); bool BlobsFetchingStarted = false; bool TaskFinishedWithError = false; bool DataIsReadyFlag = false; const ui64 TaskIdentifier = 0; const TString ExternalTaskId; bool AbortFlag = false; - std::optional AllRangesSize; - std::optional AllRangesCount; - std::optional ReadRangesCount; TString TaskCustomer; std::shared_ptr ResourcesGuard; - ui32 BlobErrorsCount = 0; - ui32 BlobsDataCount = 0; + i64 BlobsWaitingCount = 0; bool ResultsExtracted = false; protected: bool IsFetchingStarted() const { return BlobsFetchingStarted; } - THashMap ExtractBlobsData(); + TCompositeReadBlobs ExtractBlobsData(); virtual void DoOnDataReady(const std::shared_ptr& resourcesGuard) = 0; - virtual bool DoOnError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) = 0; + virtual bool DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) = 0; void OnDataReady(); - bool OnError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status); + bool OnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status); virtual TString DoDebugString() const { return ""; } public: + i64 GetWaitingRangesCount() const { + return BlobsWaitingCount; + } + void Abort() { AbortFlag = true; } bool IsFinished() const { - return BlobsWaiting.empty() && BlobsFetchingStarted; + return AgentsWaiting.empty() && BlobsFetchingStarted; } ui64 GetTaskIdentifier() const { @@ -62,43 +121,14 @@ class ITask: public NColumnShard::TMonitoringObjectsCounter { TString DebugString() const; - ui64 GetAllRangesSize() const { - Y_ABORT_UNLESS(AllRangesSize); - return *AllRangesSize; - } - - ui64 GetAllRangesCount() const { - Y_ABORT_UNLESS(AllRangesCount); - return *AllRangesCount; - } - - ui64 GetReadRangesCount() const { - Y_ABORT_UNLESS(ReadRangesCount); - return *ReadRangesCount; - } - - ui32 GetWaitingCount() const { - return BlobsWaiting.size(); - } - - THashSet GetExpectedRanges() const { - THashSet result; - for (auto&& i : Agents) { - i->FillExpectedRanges(result); - } - return result; - } - - const std::vector>& GetAgents() const; - virtual ~ITask(); - ITask(const std::vector>& actions, const TString& taskCustomer, const TString& externalTaskId = ""); + ITask(const TReadActionsCollection& actions, const TString& taskCustomer, const TString& externalTaskId = ""); void StartBlobsFetching(const THashSet& rangesInProgress); - bool AddError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status); - void AddData(const TBlobRange& range, const TString& data); + bool AddError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status); + void AddData(const TString& storageId, const TBlobRange& range, const TString& data); class TReadSubscriber: public NResourceBroker::NSubscribe::ITask { private: diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index c7dfeebdaba1..b8d9ffe9e285 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -3,6 +3,7 @@ #include "hooks/abstract/abstract.h" #include "resource_subscriber/actor.h" #include "engines/writer/buffer/actor.h" +#include "engines/column_engine_logs.h" #include @@ -59,6 +60,7 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { Executor()->RegisterExternalTabletCounters(TabletCountersPtr.release()); const auto selfActorId = SelfId(); + StoragesManager->Initialize(); Tiers = std::make_shared(TabletID(), SelfId(), [selfActorId](const TActorContext& ctx) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "tiering_new_event"); @@ -141,7 +143,11 @@ void TColumnShard::Handle(TEvPrivate::TEvReadFinished::TPtr& ev, const TActorCon Y_UNUSED(ctx); ui64 readCookie = ev->Get()->RequestCookie; LOG_S_DEBUG("Finished read cookie: " << readCookie << " at tablet " << TabletID()); - InFlightReadsTracker.RemoveInFlightRequest(ev->Get()->RequestCookie); + const NOlap::TVersionedIndex* index = nullptr; + if (HasIndex()) { + index = &GetIndexAs().GetVersionedIndex(); + } + InFlightReadsTracker.RemoveInFlightRequest(ev->Get()->RequestCookie, index); ui64 txId = ev->Get()->TxId; if (ScanTxInFlight.contains(txId)) { diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index e4c76ab6cccf..cea82f4500eb 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -846,7 +846,11 @@ void TTxScan::Complete(const TActorContext& ctx) { return; } - ui64 requestCookie = Self->InFlightReadsTracker.AddInFlightRequest(ReadMetadataRanges); + const NOlap::TVersionedIndex* index = nullptr; + if (Self->HasIndex()) { + index = &Self->GetIndexAs().GetVersionedIndex(); + } + ui64 requestCookie = Self->InFlightReadsTracker.AddInFlightRequest(ReadMetadataRanges, index); auto statsDelta = Self->InFlightReadsTracker.GetSelectStatsDelta(); Self->IncCounter(COUNTER_READ_INDEX_PORTIONS, statsDelta.Portions); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 33ef906f7289..4149282afd17 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -503,6 +503,8 @@ void TColumnShard::RunAlterStore(const NKikimrTxColumnShard::TAlterStore& proto, void TColumnShard::EnqueueBackgroundActivities(bool periodic, TBackgroundActivity activity) { TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())); ACFL_DEBUG("event", "EnqueueBackgroundActivities")("periodic", periodic)("activity", activity.DebugString()); + StoragesManager->GetOperatorVerified(NOlap::IStoragesManager::DefaultStorageId); + StoragesManager->GetSharedBlobsManager()->GetStorageManagerVerified(NOlap::IStoragesManager::DefaultStorageId); CSCounters.OnStartBackground(); SendPeriodicStats(); @@ -590,8 +592,8 @@ class TChangesReadTask: public NOlap::NBlobOperations::NRead::ITask { NConveyor::TCompServiceOperator::SendTaskToExecute(task); } } - virtual bool DoOnError(const NOlap::TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "DoOnError")("blob_id", range)("status", status.GetErrorMessage())("status_code", status.GetStatus()); + virtual bool DoOnError(const TString& storageId, const NOlap::TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "DoOnError")("storage_id", storageId)("blob_id", range)("status", status.GetErrorMessage())("status_code", status.GetStatus()); AFL_VERIFY(false)("blob_id", range)("status", status.GetStatus()); TxEvent->SetPutStatus(NKikimrProto::ERROR); TActorContext::AsActorContext().Send(ParentActorId, std::move(TxEvent)); diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp index 407aed03b43d..2a787d11e18d 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp @@ -123,7 +123,7 @@ bool WriteDataImpl(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shar } bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector>& ydbSchema, std::vector* writeIds) { + const std::vector& ydbSchema, std::vector* writeIds) { NLongTxService::TLongTxId longTxId; UNIT_ASSERT(longTxId.ParseString("ydb://long-tx/01ezvvxjdk2hd4vdgjs68knvp8?node_id=1")); return WriteDataImpl(runtime, sender, shardId, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), writeIds); @@ -131,7 +131,7 @@ bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, } bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector>& ydbSchema, bool waitResult, std::vector* writeIds) { + const std::vector& ydbSchema, bool waitResult, std::vector* writeIds) { NLongTxService::TLongTxId longTxId; UNIT_ASSERT(longTxId.ParseString("ydb://long-tx/01ezvvxjdk2hd4vdgjs68knvp8?node_id=1")); if (writeIds) { @@ -143,7 +143,7 @@ bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, std::optional WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, ui64 tableId, const ui64 writePartId, const TString& data, - const std::vector>& ydbSchema) + const std::vector& ydbSchema) { auto write = std::make_unique(sender, longTxId, tableId, "0", data, writePartId); write->SetArrowSchema(NArrow::SerializeSchema(*NArrow::MakeArrowSchema(ydbSchema))); @@ -295,18 +295,17 @@ std::vector MakeTestCells(const std::vector& types, ui32 value } -TString MakeTestBlob(std::pair range, const std::vector>& columns, +TString MakeTestBlob(std::pair range, const std::vector& columns, const TTestBlobOptions& options, const std::set& notNullColumns) { - TString err; NArrow::TArrowBatchBuilder batchBuilder(arrow::Compression::LZ4_FRAME, notNullColumns); - batchBuilder.Start(columns, 0, 0, err); + batchBuilder.Start(NArrow::NTest::TTestColumn::ConvertToPairs(columns)); std::vector nullPositions; std::vector samePositions; for (size_t i = 0; i < columns.size(); ++i) { - if (options.NullColumns.contains(columns[i].first)) { + if (options.NullColumns.contains(columns[i].GetName())) { nullPositions.push_back(i); - } else if (options.SameValueColumns.contains(columns[i].first)) { + } else if (options.SameValueColumns.contains(columns[i].GetName())) { samePositions.push_back(i); } } @@ -348,7 +347,7 @@ TString MakeTestBlob(std::pair range, const std::vector range, bool inclusiveFrom, bool inclusiveTo, - const std::vector>& columns) { + const std::vector& columns) { std::vector mem; std::vector types = TTestSchema::ExtractTypes(columns); std::vector cellsFrom = MakeTestCells(types, range.first, mem); @@ -392,24 +391,26 @@ NMetadata::NFetcher::ISnapshot::TPtr TTestSchema::BuildSnapshot(const TTableSpec } namespace NKikimr::NColumnShard { - NOlap::TIndexInfo BuildTableInfo(const std::vector>& ydbSchema, - const std::vector>& key) { + NOlap::TIndexInfo BuildTableInfo(const std::vector& ydbSchema, + const std::vector& key) { NOlap::TIndexInfo indexInfo = NOlap::TIndexInfo::BuildDefault(); for (ui32 i = 0; i < ydbSchema.size(); ++i) { ui32 id = i + 1; - auto& name = ydbSchema[i].first; - auto& type = ydbSchema[i].second; + auto& name = ydbSchema[i].GetName(); + auto& type = ydbSchema[i].GetType(); indexInfo.Columns[id] = NTable::TColumn(name, id, type, ""); indexInfo.ColumnNames[name] = id; } - for (const auto& [keyName, keyType] : key) { - indexInfo.KeyColumns.push_back(indexInfo.ColumnNames[keyName]); + for (const auto& c : key) { + indexInfo.KeyColumns.push_back(indexInfo.ColumnNames[c.GetName()]); } - indexInfo.SetAllKeys(); + auto storage = std::make_shared(); + storage->Initialize(); + indexInfo.SetAllKeys(std::make_shared()); return indexInfo; } @@ -430,7 +431,7 @@ namespace NKikimr::NColumnShard { PlanSchemaTx(runtime, sender, snap); } - void PrepareTablet(TTestBasicRuntime& runtime, const ui64 tableId, const std::vector>& schema, const ui32 keySize) { + void PrepareTablet(TTestBasicRuntime& runtime, const ui64 tableId, const std::vector& schema, const ui32 keySize) { using namespace NTxUT; CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); @@ -449,10 +450,10 @@ namespace NKikimr::NColumnShard { SetupSchema(runtime, sender, tableId, tableDescription); } - std::shared_ptr ReadAllAsBatch(TTestBasicRuntime& runtime, const ui64 tableId, const NOlap::TSnapshot& snapshot, const std::vector>& schema) { + std::shared_ptr ReadAllAsBatch(TTestBasicRuntime& runtime, const ui64 tableId, const NOlap::TSnapshot& snapshot, const std::vector& schema) { std::vector fields; for (auto&& f : schema) { - fields.emplace_back(f.first); + fields.emplace_back(f.GetName()); } NOlap::NTests::TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, snapshot); diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h index 58849aa342e1..2d81e956c122 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/columnshard_ut_common.h @@ -5,13 +5,13 @@ #include "blob_cache.h" #include +#include #include #include #include #include #include - namespace NKikimr::NTxUT { // Private events of different actors reuse the same ES_PRIVATE range @@ -95,6 +95,7 @@ struct TTestSchema { s3Config.SetProxyScheme(NKikimrSchemeOp::TS3Settings::HTTP); #else s3Config.SetEndpoint("fake"); + s3Config.SetSecretKey("fakeSecret"); #endif s3Config.SetRequestTimeoutMs(10000); s3Config.SetHttpRequestTimeoutMs(10000); @@ -128,106 +129,94 @@ struct TTestSchema { return *this; } }; - - static auto YdbSchema(const std::pair& firstKeyItem = {"timestamp", TTypeInfo(NTypeIds::Timestamp) }) { - std::vector> schema = { + using TTestColumn = NArrow::NTest::TTestColumn; + static auto YdbSchema(const TTestColumn& firstKeyItem = TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp))) { + std::vector schema = { // PK firstKeyItem, - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, - {"level", TTypeInfo(NTypeIds::Int32) }, - {"message", TTypeInfo(NTypeIds::Utf8) }, - {"json_payload", TTypeInfo(NTypeIds::Json) }, - {"ingested_at", TTypeInfo(NTypeIds::Timestamp) }, - {"saved_at", TTypeInfo(NTypeIds::Timestamp) }, - {"request_id", TTypeInfo(NTypeIds::Utf8) } + TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), + TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ).SetStorageId("__MEMORY"), + TTestColumn("level", TTypeInfo(NTypeIds::Int32) ), + TTestColumn("message", TTypeInfo(NTypeIds::Utf8) ).SetStorageId("__MEMORY"), + TTestColumn("json_payload", TTypeInfo(NTypeIds::Json) ), + TTestColumn("ingested_at", TTypeInfo(NTypeIds::Timestamp) ), + TTestColumn("saved_at", TTypeInfo(NTypeIds::Timestamp) ), + TTestColumn("request_id", TTypeInfo(NTypeIds::Utf8) ) }; return schema; }; static auto YdbExoticSchema() { - std::vector> schema = { + std::vector schema = { // PK - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, + TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), + TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ).SetStorageId("__MEMORY"), // - {"level", TTypeInfo(NTypeIds::Int32) }, - {"message", TTypeInfo(NTypeIds::String4k) }, - {"json_payload", TTypeInfo(NTypeIds::JsonDocument) }, - {"ingested_at", TTypeInfo(NTypeIds::Timestamp) }, - {"saved_at", TTypeInfo(NTypeIds::Timestamp) }, - {"request_id", TTypeInfo(NTypeIds::Yson) } + TTestColumn("level", TTypeInfo(NTypeIds::Int32) ), + TTestColumn("message", TTypeInfo(NTypeIds::String4k) ).SetStorageId("__MEMORY"), + TTestColumn("json_payload", TTypeInfo(NTypeIds::JsonDocument) ), + TTestColumn("ingested_at", TTypeInfo(NTypeIds::Timestamp) ), + TTestColumn("saved_at", TTypeInfo(NTypeIds::Timestamp) ), + TTestColumn("request_id", TTypeInfo(NTypeIds::Yson) ) }; return schema; }; static auto YdbPkSchema() { - std::vector> schema = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) } + std::vector schema = { + TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ).SetStorageId("__MEMORY"), + TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ).SetStorageId("__MEMORY") }; return schema; } static auto YdbAllTypesSchema() { - std::vector> schema = { - { "ts", TTypeInfo(NTypeIds::Timestamp) }, - - { "i8", TTypeInfo(NTypeIds::Int8) }, - { "i16", TTypeInfo(NTypeIds::Int16) }, - { "i32", TTypeInfo(NTypeIds::Int32) }, - { "i64", TTypeInfo(NTypeIds::Int64) }, - { "u8", TTypeInfo(NTypeIds::Uint8) }, - { "u16", TTypeInfo(NTypeIds::Uint16) }, - { "u32", TTypeInfo(NTypeIds::Uint32) }, - { "u64", TTypeInfo(NTypeIds::Uint64) }, - { "float", TTypeInfo(NTypeIds::Float) }, - { "double", TTypeInfo(NTypeIds::Double) }, - - { "byte", TTypeInfo(NTypeIds::Byte) }, + std::vector schema = { + TTestColumn("ts", TTypeInfo(NTypeIds::Timestamp) ), + + TTestColumn( "i8", TTypeInfo(NTypeIds::Int8) ), + TTestColumn( "i16", TTypeInfo(NTypeIds::Int16) ), + TTestColumn( "i32", TTypeInfo(NTypeIds::Int32) ), + TTestColumn( "i64", TTypeInfo(NTypeIds::Int64) ), + TTestColumn( "u8", TTypeInfo(NTypeIds::Uint8) ), + TTestColumn( "u16", TTypeInfo(NTypeIds::Uint16) ), + TTestColumn( "u32", TTypeInfo(NTypeIds::Uint32) ), + TTestColumn( "u64", TTypeInfo(NTypeIds::Uint64) ), + TTestColumn( "float", TTypeInfo(NTypeIds::Float) ), + TTestColumn( "double", TTypeInfo(NTypeIds::Double) ), + + TTestColumn("byte", TTypeInfo(NTypeIds::Byte) ), //{ "bool", TTypeInfo(NTypeIds::Bool) }, //{ "decimal", TTypeInfo(NTypeIds::Decimal) }, //{ "dynum", TTypeInfo(NTypeIds::DyNumber) }, - { "date", TTypeInfo(NTypeIds::Date) }, - { "datetime", TTypeInfo(NTypeIds::Datetime) }, + TTestColumn( "date", TTypeInfo(NTypeIds::Date) ), + TTestColumn( "datetime", TTypeInfo(NTypeIds::Datetime) ), //{ "interval", TTypeInfo(NTypeIds::Interval) }, - {"text", TTypeInfo(NTypeIds::Text) }, - {"bytes", TTypeInfo(NTypeIds::Bytes) }, - {"yson", TTypeInfo(NTypeIds::Yson) }, - {"json", TTypeInfo(NTypeIds::Json) }, - {"jsondoc", TTypeInfo(NTypeIds::JsonDocument) } + TTestColumn("text", TTypeInfo(NTypeIds::Text) ), + TTestColumn("bytes", TTypeInfo(NTypeIds::Bytes) ), + TTestColumn("yson", TTypeInfo(NTypeIds::Yson) ), + TTestColumn("json", TTypeInfo(NTypeIds::Json) ), + TTestColumn("jsondoc", TTypeInfo(NTypeIds::JsonDocument) ) }; return schema; }; - static NKikimrSchemeOp::TOlapColumnDescription CreateColumn(ui32 id, const TString& name, TTypeInfo type) { - NKikimrSchemeOp::TOlapColumnDescription col; - col.SetId(id); - col.SetName(name); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(type, ""); - col.SetTypeId(columnType.TypeId); - if (columnType.TypeInfo) { - *col.MutableTypeInfo() = *columnType.TypeInfo; - } - return col; - } - - static void InitSchema(const std::vector>& columns, - const std::vector>& pk, + static void InitSchema(const std::vector& columns, + const std::vector& pk, const TTableSpecials& specials, NKikimrSchemeOp::TColumnTableSchema* schema) { schema->SetEngine(NKikimrSchemeOp::COLUMN_ENGINE_REPLACING_TIMESERIES); for (ui32 i = 0; i < columns.size(); ++i) { - *schema->MutableColumns()->Add() = CreateColumn(i + 1, columns[i].first, columns[i].second); + *schema->MutableColumns()->Add() = columns[i].CreateColumn(i + 1); } Y_ABORT_UNLESS(pk.size() > 0); @@ -261,8 +250,8 @@ struct TTestSchema { return specials.HasTiers() || specials.HasTtl(); } - static TString CreateTableTxBody(ui64 pathId, const std::vector>& columns, - const std::vector>& pk, + static TString CreateTableTxBody(ui64 pathId, const std::vector& columns, + const std::vector& pk, const TTableSpecials& specialsExt = {}) { auto specials = specialsExt; @@ -288,8 +277,8 @@ struct TTestSchema { return out; } - static TString CreateInitShardTxBody(ui64 pathId, const std::vector>& columns, - const std::vector>& pk, + static TString CreateInitShardTxBody(ui64 pathId, const std::vector& columns, + const std::vector& pk, const TTableSpecials& specials = {}, const TString& ownerPath = "/Root/olap") { NKikimrTxColumnShard::TSchemaTxBody tx; auto* table = tx.MutableInitShard()->AddTables(); @@ -306,8 +295,8 @@ struct TTestSchema { return out; } - static TString CreateStandaloneTableTxBody(ui64 pathId, const std::vector>& columns, - const std::vector>& pk, + static TString CreateStandaloneTableTxBody(ui64 pathId, const std::vector& columns, + const std::vector& pk, const TTableSpecials& specials = {}) { NKikimrTxColumnShard::TSchemaTxBody tx; auto* table = tx.MutableEnsureTables()->AddTables(); @@ -377,20 +366,20 @@ struct TTestSchema { return txBody; } - static std::vector ExtractNames(const std::vector>& columns) { + static std::vector ExtractNames(const std::vector& columns) { std::vector out; out.reserve(columns.size()); for (auto& col : columns) { - out.push_back(col.first); + out.push_back(col.GetName()); } return out; } - static std::vector ExtractTypes(const std::vector>& columns) { + static std::vector ExtractTypes(const std::vector& columns) { std::vector types; types.reserve(columns.size()); - for (auto& [name, type] : columns) { - types.push_back(type); + for (auto& i : columns) { + types.push_back(i.GetType()); } return types; } @@ -403,14 +392,14 @@ void PlanSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot void PlanWriteTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot snap, bool waitResult = true); bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector>& ydbSchema, std::vector* writeIds); + const std::vector& ydbSchema, std::vector* writeIds); bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector>& ydbSchema, bool waitResult = true, std::vector* writeIds = nullptr); + const std::vector& ydbSchema, bool waitResult = true, std::vector* writeIds = nullptr); std::optional WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, ui64 tableId, const ui64 writePartId, const TString& data, - const std::vector>& ydbSchema); + const std::vector& ydbSchema); ui32 WaitWriteResult(TTestBasicRuntime& runtime, ui64 shardId, std::vector* writeIds = nullptr); @@ -436,10 +425,10 @@ struct TTestBlobOptions { }; TCell MakeTestCell(const TTypeInfo& typeInfo, ui32 value, std::vector& mem); -TString MakeTestBlob(std::pair range, const std::vector>& columns, +TString MakeTestBlob(std::pair range, const std::vector& columns, const TTestBlobOptions& options = {}, const std::set& notNullColumns = {}); TSerializedTableRange MakeTestRange(std::pair range, bool inclusiveFrom, bool inclusiveTo, - const std::vector>& columns); + const std::vector& columns); } @@ -525,20 +514,20 @@ namespace NKikimr::NColumnShard { } }; - NOlap::TIndexInfo BuildTableInfo(const std::vector>& ydbSchema, - const std::vector>& key); + NOlap::TIndexInfo BuildTableInfo(const std::vector& ydbSchema, + const std::vector& key); struct TestTableDescription { - std::vector> Schema = NTxUT::TTestSchema::YdbSchema(); - std::vector> Pk = NTxUT::TTestSchema::YdbPkSchema(); + std::vector Schema = NTxUT::TTestSchema::YdbSchema(); + std::vector Pk = NTxUT::TTestSchema::YdbPkSchema(); bool InStore = true; }; void SetupSchema(TTestBasicRuntime& runtime, TActorId& sender, ui64 pathId, const TestTableDescription& table = {}, TString codec = "none"); - void PrepareTablet(TTestBasicRuntime& runtime, const ui64 tableId, const std::vector>& schema, const ui32 keySize = 1); + void PrepareTablet(TTestBasicRuntime& runtime, const ui64 tableId, const std::vector& schema, const ui32 keySize = 1); - std::shared_ptr ReadAllAsBatch(TTestBasicRuntime& runtime, const ui64 tableId, const NOlap::TSnapshot& snapshot, const std::vector>& schema); + std::shared_ptr ReadAllAsBatch(TTestBasicRuntime& runtime, const ui64 tableId, const NOlap::TSnapshot& snapshot, const std::vector& schema); } diff --git a/ydb/core/tx/columnshard/counters/engine_logs.cpp b/ydb/core/tx/columnshard/counters/engine_logs.cpp index 07b2844e97de..8086ccbe7934 100644 --- a/ydb/core/tx/columnshard/counters/engine_logs.cpp +++ b/ydb/core/tx/columnshard/counters/engine_logs.cpp @@ -52,8 +52,11 @@ TEngineLogsCounters::TEngineLogsCounters() void TEngineLogsCounters::TPortionsInfoGuard::OnNewPortion(const std::shared_ptr& portion) const { const ui32 producedId = (ui32)(portion->HasRemoveSnapshot() ? NOlap::NPortion::EProduced::INACTIVE : portion->GetMeta().Produced); Y_ABORT_UNLESS(producedId < BlobGuards.size()); - for (auto&& i : portion->GetBlobIds()) { - BlobGuards[producedId]->Add(i.BlobSize(), i.BlobSize()); + for (auto&& i : portion->GetRecords()) { + BlobGuards[producedId]->Add(i.GetBlobRange().Size, i.GetBlobRange().Size); + } + for (auto&& i : portion->GetIndexes()) { + BlobGuards[producedId]->Add(i.GetBlobRange().Size, i.GetBlobRange().Size); } PortionRecordCountGuards[producedId]->Add(portion->GetRecordsCount(), 1); PortionSizeGuards[producedId]->Add(portion->GetBlobBytes(), 1); @@ -62,8 +65,11 @@ void TEngineLogsCounters::TPortionsInfoGuard::OnNewPortion(const std::shared_ptr void TEngineLogsCounters::TPortionsInfoGuard::OnDropPortion(const std::shared_ptr& portion) const { const ui32 producedId = (ui32)(portion->HasRemoveSnapshot() ? NOlap::NPortion::EProduced::INACTIVE : portion->GetMeta().Produced); Y_ABORT_UNLESS(producedId < BlobGuards.size()); - for (auto&& i : portion->GetBlobIds()) { - BlobGuards[producedId]->Sub(i.BlobSize(), i.BlobSize()); + for (auto&& i : portion->GetRecords()) { + BlobGuards[producedId]->Sub(i.GetBlobRange().Size, i.GetBlobRange().Size); + } + for (auto&& i : portion->GetIndexes()) { + BlobGuards[producedId]->Sub(i.GetBlobRange().Size, i.GetBlobRange().Size); } PortionRecordCountGuards[producedId]->Sub(portion->GetRecordsCount(), 1); PortionSizeGuards[producedId]->Sub(portion->GetBlobBytes(), 1); diff --git a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.cpp b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.cpp index 06b6edadbb42..eece3f8574e4 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.cpp @@ -1,13 +1,16 @@ #include "transfer.h" #include #include +#include namespace NKikimr::NOlap::NDataSharing::NEvents { -THashMap TPathIdData::BuildLinkTabletTasks(const std::shared_ptr& sharedBlobs, const TTabletId selfTabletId, const TTransferContext& context) { +THashMap TPathIdData::BuildLinkTabletTasks( + const std::shared_ptr& sharedBlobs, const TTabletId selfTabletId, const TTransferContext& context, const TVersionedIndex& index) { THashMap> blobIds; for (auto&& i : Portions) { - i.FillBlobIdsByStorage(blobIds); + auto schema = index.GetSchema(i.GetMinSnapshot()); + i.FillBlobIdsByStorage(blobIds, schema->GetIndexInfo()); } THashMap> blobsInfo; diff --git a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h index e82d2b0d8fc8..d4205a0db65d 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h +++ b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h @@ -6,6 +6,10 @@ #include +namespace NKikimr::NOlap { +class TVersionedIndex; +} + namespace NKikimr::NOlap::NDataSharing { class TSharedBlobsManager; class TTaskForTablet; @@ -45,7 +49,8 @@ class TPathIdData { std::vector DetachPortions() { return std::move(Portions); } - THashMap BuildLinkTabletTasks(const std::shared_ptr& sharedBlobs, const TTabletId selfTabletId, const TTransferContext& context); + THashMap BuildLinkTabletTasks(const std::shared_ptr& sharedBlobs, const TTabletId selfTabletId, + const TTransferContext& context, const TVersionedIndex& index); void InitPortionIds(ui64* lastPortionId, const std::optional pathId = {}) { AFL_VERIFY(lastPortionId); diff --git a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp index c7c1b77e5b3d..8942f3f85f14 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp @@ -10,7 +10,7 @@ namespace NKikimr::NOlap::NDataSharing { -NKikimr::TConclusionStatus TDestinationSession::DataReceived(THashMap&& data, TColumnEngineForLogs& index, const std::shared_ptr& manager) { +NKikimr::TConclusionStatus TDestinationSession::DataReceived(THashMap&& data, TColumnEngineForLogs& index, const std::shared_ptr& /*manager*/) { auto guard = index.GranulesStorage->StartPackModification(); for (auto&& i : data) { auto it = PathIds.find(i.first); @@ -19,7 +19,7 @@ NKikimr::TConclusionStatus TDestinationSession::DataReceived(THashMap> blobIds; - portion.FillBlobIdsByStorage(blobIds); + portion.FillBlobIdsByStorage(blobIds, index.GetVersionedIndex()); for (auto&& s : blobIds) { auto it = CurrentBlobIds.find(s.first); if (it == CurrentBlobIds.end()) { @@ -34,8 +34,6 @@ NKikimr::TConclusionStatus TDestinationSession::DataReceived(THashMapGetOperatorVerified(portion.GetMeta().GetTierName() ? portion.GetMeta().GetTierName() : IStoragesManager::DefaultStorageId); - portion.InitOperator(storage, false); portion.SetPathId(it->second); index.UpsertPortion(std::move(portion)); } @@ -178,7 +176,7 @@ bool TDestinationSession::DoStart(const NColumnShard::TColumnShard& shard, const THashMap> local; for (auto&& i : portions) { for (auto&& p : i.second) { - p->FillBlobIdsByStorage(local); + p->FillBlobIdsByStorage(local, shard.GetIndexAs().GetVersionedIndex()); } } std::swap(CurrentBlobIds, local); diff --git a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp index 403ee00848d0..40387937a93b 100644 --- a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp +++ b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp @@ -5,23 +5,22 @@ namespace NKikimr::NOlap::NDataSharing { -bool TSharedBlobsManager::Load(NTable::TDatabase& database) { +bool TSharedBlobsManager::LoadIdempotency(NTable::TDatabase& database) { NIceDb::TNiceDb db(database); using namespace NKikimr::NColumnShard; + THashMap>> sharedBlobIds; + THashMap> borrowedBlobIds; { auto rowset = db.Table().Select(); if (!rowset.IsReady()) return false; TString error; - while (!rowset.EndOfSet()) { const TString& storageId = rowset.GetValue(); - const TString blobIdStr = rowset.GetValue(); - NOlap::TUnifiedBlobId unifiedBlobId = NOlap::TUnifiedBlobId::ParseFromString(blobIdStr, nullptr, error); - Y_ABORT_UNLESS(unifiedBlobId.IsValid(), "%s", error.c_str()); - - AFL_VERIFY(GetStorageManagerGuarantee(storageId)->UpsertSharedBlobOnLoad(unifiedBlobId, (TTabletId)rowset.GetValue())); + auto unifiedBlobId = NOlap::TUnifiedBlobId::BuildFromString(rowset.GetValue(), nullptr); + AFL_VERIFY(!!unifiedBlobId)("error", unifiedBlobId.GetErrorMessage()); + AFL_VERIFY(sharedBlobIds[storageId][*unifiedBlobId].emplace((TTabletId)rowset.GetValue()).second)("blob_id", *unifiedBlobId)("storage_id", storageId); if (!rowset.Next()) return false; } @@ -36,15 +35,30 @@ bool TSharedBlobsManager::Load(NTable::TDatabase& database) { while (!rowset.EndOfSet()) { const TString& storageId = rowset.GetValue(); - const TString blobIdStr = rowset.GetValue(); - NOlap::TUnifiedBlobId unifiedBlobId = NOlap::TUnifiedBlobId::ParseFromString(blobIdStr, nullptr, error); - Y_ABORT_UNLESS(unifiedBlobId.IsValid(), "%s", error.c_str()); - - AFL_VERIFY(GetStorageManagerGuarantee(storageId)->UpsertBorrowedBlobOnLoad(unifiedBlobId, (TTabletId)rowset.GetValue())); + auto unifiedBlobId = NOlap::TUnifiedBlobId::BuildFromString(rowset.GetValue(), nullptr); + AFL_VERIFY(!!unifiedBlobId)("error", unifiedBlobId.GetErrorMessage()); + AFL_VERIFY(borrowedBlobIds[storageId].emplace(*unifiedBlobId, (TTabletId)rowset.GetValue()).second)("blob_id", *unifiedBlobId)("storage_id", storageId); if (!rowset.Next()) return false; } } + for (auto&& i : Storages) { + i.second->Clear(); + } + for (auto&& [storageId, blobs] : sharedBlobIds) { + auto storage = GetStorageManagerGuarantee(storageId); + for (auto&& b : blobs) { + for (auto&& t : b.second) { + AFL_VERIFY(storage->UpsertSharedBlobOnLoad(b.first, t)); + } + } + } + for (auto&& [storageId, blobs] : borrowedBlobIds) { + auto storage = GetStorageManagerGuarantee(storageId); + for (auto&& b : blobs) { + AFL_VERIFY(storage->UpsertBorrowedBlobOnLoad(b.first, b.second)); + } + } return true; } diff --git a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h index 6772061d49b6..c9cacb603aeb 100644 --- a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h +++ b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h @@ -154,6 +154,11 @@ class TStorageSharedBlobsManager { return BorrowedBlobIds.emplace(blobId, ownerTabletId).second; } + void Clear() { + SharedBlobIds.Clear(); + BorrowedBlobIds.clear(); + } + void OnTransactionExecuteAfterCleaning(const TBlobsCategories& removeTask, NTable::TDatabase& db); void OnTransactionCompleteAfterCleaning(const TBlobsCategories& removeTask); }; @@ -224,7 +229,7 @@ class TSharedBlobsManager { std::shared_ptr GetStorageManagerVerified(const TString& storageId) const { auto it = Storages.find(storageId); - AFL_VERIFY(it != Storages.end()); + AFL_VERIFY(it != Storages.end())("storage_id", storageId); return it->second; } @@ -236,7 +241,7 @@ class TSharedBlobsManager { return it->second; } - bool Load(NTable::TDatabase& database); + bool LoadIdempotency(NTable::TDatabase& database); }; } \ No newline at end of file diff --git a/ydb/core/tx/columnshard/data_sharing/source/session/cursor.cpp b/ydb/core/tx/columnshard/data_sharing/source/session/cursor.cpp index 47e1a05e4121..1ca8fac2ace7 100644 --- a/ydb/core/tx/columnshard/data_sharing/source/session/cursor.cpp +++ b/ydb/core/tx/columnshard/data_sharing/source/session/cursor.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NOlap::NDataSharing { -void TSourceCursor::BuildSelection(const std::shared_ptr& sharedBlobsManager) { +void TSourceCursor::BuildSelection(const std::shared_ptr& sharedBlobsManager, const TVersionedIndex& index) { THashMap result; auto itCurrentPath = PortionsForSend.find(StartPathId); AFL_VERIFY(itCurrentPath != PortionsForSend.end()); @@ -41,7 +41,7 @@ void TSourceCursor::BuildSelection(const std::shared_ptr& s THashMap tabletTasksResult; for (auto&& i : result) { - THashMap tabletTasks = i.second.BuildLinkTabletTasks(sharedBlobsManager, SelfTabletId, TransferContext); + THashMap tabletTasks = i.second.BuildLinkTabletTasks(sharedBlobsManager, SelfTabletId, TransferContext, index); for (auto&& t : tabletTasks) { auto it = tabletTasksResult.find(t.first); if (it == tabletTasksResult.end()) { @@ -56,7 +56,7 @@ void TSourceCursor::BuildSelection(const std::shared_ptr& s std::swap(Selected, result); } -bool TSourceCursor::Next(const std::shared_ptr& sharedBlobsManager) { +bool TSourceCursor::Next(const std::shared_ptr& sharedBlobsManager, const TVersionedIndex& index) { PreviousSelected = std::move(Selected); LinksModifiedTablets.clear(); Selected.clear(); @@ -71,7 +71,7 @@ bool TSourceCursor::Next(const std::shared_ptr& sharedBlobs NextPathId = {}; NextPortionId = {}; ++PackIdx; - BuildSelection(sharedBlobsManager); + BuildSelection(sharedBlobsManager, index); AFL_VERIFY(IsValid()); return true; } @@ -141,7 +141,7 @@ TSourceCursor::TSourceCursor(const TTabletId selfTabletId, const std::set& { } -bool TSourceCursor::Start(const std::shared_ptr& sharedBlobsManager, const THashMap>>& portions) { +bool TSourceCursor::Start(const std::shared_ptr& sharedBlobsManager, const THashMap>>& portions, const TVersionedIndex& index) { AFL_VERIFY(!IsStartedFlag); std::map>> local; std::vector> portionsLock; @@ -170,9 +170,9 @@ bool TSourceCursor::Start(const std::shared_ptr& sharedBlob NextPathId = PortionsForSend.begin()->first; NextPortionId = PortionsForSend.begin()->second.begin()->first; - AFL_VERIFY(Next(sharedBlobsManager)); + AFL_VERIFY(Next(sharedBlobsManager, index)); } else { - BuildSelection(sharedBlobsManager); + BuildSelection(sharedBlobsManager, index); } IsStartedFlag = true; return true; diff --git a/ydb/core/tx/columnshard/data_sharing/source/session/cursor.h b/ydb/core/tx/columnshard/data_sharing/source/session/cursor.h index cb0aea148b19..7e527758bf0a 100644 --- a/ydb/core/tx/columnshard/data_sharing/source/session/cursor.h +++ b/ydb/core/tx/columnshard/data_sharing/source/session/cursor.h @@ -5,6 +5,7 @@ namespace NKikimr::NOlap { class TColumnEngineForLogs; +class TVersionedIndex; } namespace NKikimr::NOlap::NDataSharing { @@ -30,7 +31,7 @@ class TSourceCursor { THashMap PathPortionHashes; bool IsStartedFlag = false; YDB_ACCESSOR(bool, StaticSaved, false); - void BuildSelection(const std::shared_ptr& sharedBlobsManager); + void BuildSelection(const std::shared_ptr& sharedBlobsManager, const TVersionedIndex& index); public: bool IsAckDataReceived() const { return AckReceivedForPackIdx == PackIdx; @@ -87,7 +88,7 @@ class TSourceCursor { return Links; } - bool Next(const std::shared_ptr& sharedBlobsManager); + bool Next(const std::shared_ptr& sharedBlobsManager, const TVersionedIndex& index); bool IsValid() { return Selected.size(); @@ -95,7 +96,7 @@ class TSourceCursor { TSourceCursor(const TTabletId selfTabletId, const std::set& pathIds, const TTransferContext transferContext); - bool Start(const std::shared_ptr& sharedBlobsManager, const THashMap>>& portions); + bool Start(const std::shared_ptr& sharedBlobsManager, const THashMap>>& portions, const TVersionedIndex& index); NKikimrColumnShardDataSharingProto::TSourceSession::TCursorDynamic SerializeDynamicToProto() const; NKikimrColumnShardDataSharingProto::TSourceSession::TCursorStatic SerializeStaticToProto() const; diff --git a/ydb/core/tx/columnshard/data_sharing/source/session/source.cpp b/ydb/core/tx/columnshard/data_sharing/source/session/source.cpp index 7bca95eeb604..3cf78859289f 100644 --- a/ydb/core/tx/columnshard/data_sharing/source/session/source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/source/session/source.cpp @@ -48,7 +48,7 @@ TConclusion> TSourceSession:: return ackResult; } if (Cursor->IsReadyForNext()) { - Cursor->Next(self->GetStoragesManager()->GetSharedBlobsManager()); + Cursor->Next(self->GetStoragesManager()->GetSharedBlobsManager(), self->GetIndexAs().GetVersionedIndex()); return std::unique_ptr(new TTxDataAckToSource(self, selfPtr, "ack_to_source_on_ack_data")); } else { return std::unique_ptr(new TTxWriteSourceCursor(self, selfPtr, "write_source_cursor_on_ack_data")); @@ -61,7 +61,7 @@ TConclusion> TSourceSession:: return ackResult; } if (Cursor->IsReadyForNext()) { - Cursor->Next(self->GetStoragesManager()->GetSharedBlobsManager()); + Cursor->Next(self->GetStoragesManager()->GetSharedBlobsManager(), self->GetIndexAs().GetVersionedIndex()); return std::unique_ptr(new TTxDataAckToSource(self, selfPtr, "ack_to_source_on_ack_links")); } else { return std::unique_ptr(new TTxWriteSourceCursor(self, selfPtr, "write_source_cursor_on_ack_links")); @@ -99,7 +99,7 @@ void TSourceSession::ActualizeDestination(const std::shared_ptr>>& portions) { AFL_VERIFY(Cursor); - if (Cursor->Start(shard.GetStoragesManager()->GetSharedBlobsManager(), portions)) { + if (Cursor->Start(shard.GetStoragesManager()->GetSharedBlobsManager(), portions, shard.GetIndexAs().GetVersionedIndex())) { ActualizeDestination(shard.GetDataLocksManager()); return true; } else { diff --git a/ydb/core/tx/columnshard/data_sharing/source/session/source.h b/ydb/core/tx/columnshard/data_sharing/source/session/source.h index ae227c95b55c..319302fe92d8 100644 --- a/ydb/core/tx/columnshard/data_sharing/source/session/source.h +++ b/ydb/core/tx/columnshard/data_sharing/source/session/source.h @@ -59,12 +59,12 @@ class TSourceSession: public TCommonSession { return Cursor; } - bool TryNextCursor(const ui32 packIdx, const std::shared_ptr& sharedBlobsManager) { + bool TryNextCursor(const ui32 packIdx, const std::shared_ptr& sharedBlobsManager, const TVersionedIndex& index) { AFL_VERIFY(Cursor); if (packIdx != Cursor->GetPackIdx()) { return false; } - Cursor->Next(sharedBlobsManager); + Cursor->Next(sharedBlobsManager, index); return true; } diff --git a/ydb/core/tx/columnshard/data_sharing/source/transactions/tx_data_ack_to_source.cpp b/ydb/core/tx/columnshard/data_sharing/source/transactions/tx_data_ack_to_source.cpp index 8269b6c2ea57..146a93bc706c 100644 --- a/ydb/core/tx/columnshard/data_sharing/source/transactions/tx_data_ack_to_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/source/transactions/tx_data_ack_to_source.cpp @@ -1,4 +1,5 @@ #include "tx_data_ack_to_source.h" +#include namespace NKikimr::NOlap::NDataSharing { @@ -7,9 +8,10 @@ bool TTxDataAckToSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc THashMap sharedTabletBlobIds; { THashMap> sharedBlobIds; + auto& index = Self->GetIndexAs().GetVersionedIndex(); for (auto&& [_, i] : Session->GetCursorVerified()->GetPreviousSelected()) { for (auto&& portion : i.GetPortions()) { - portion.FillBlobIdsByStorage(sharedBlobIds); + portion.FillBlobIdsByStorage(sharedBlobIds, index); } } for (auto&& i : sharedBlobIds) { diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp index 0c66ef049e7b..760eb6a05f82 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp @@ -17,13 +17,7 @@ TString TColumnEngineChanges::DebugString() const { TConclusionStatus TColumnEngineChanges::ConstructBlobs(TConstructionContext& context) noexcept { Y_ABORT_UNLESS(Stage == EStage::Started); - { - ui64 readBytes = 0; - for (auto&& i : Blobs) { - readBytes += i.first.Size; - } - context.Counters.CompactionInputSize(readBytes); - } + context.Counters.CompactionInputSize(Blobs.GetTotalBlobsSize()); const TMonotonic start = TMonotonic::Now(); TConclusionStatus result = DoConstructBlobs(context); if (result.Ok()) { diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index b8b4365a60e5..68c1e68b9928 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -1,5 +1,8 @@ #pragma once #include "settings.h" +#include +#include +#include #include #include #include @@ -8,8 +11,6 @@ #include #include #include -#include -#include #include #include @@ -39,12 +40,10 @@ struct TPortionEvictionFeatures { const TString TargetTierName; const ui64 PathId; // portion path id for cold-storage-key construct bool DataChanges = true; - const std::shared_ptr StorageOperator; - TPortionEvictionFeatures(const TString& targetTierName, const ui64 pathId, const std::shared_ptr& storageOperator) + TPortionEvictionFeatures(const TString& targetTierName, const ui64 pathId) : TargetTierName(targetTierName) , PathId(pathId) - , StorageOperator(storageOperator) {} }; @@ -222,12 +221,7 @@ class TColumnEngineChanges { void Compile(TFinalizationContext& context) noexcept; - void SetBlobs(THashMap&& blobs) { - Y_ABORT_UNLESS(!blobs.empty()); - Blobs = std::move(blobs); - } - - THashMap Blobs; + NBlobOperations::NRead::TCompositeReadBlobs Blobs; std::shared_ptr ResourcesGuard; std::vector> GetReadingActions() const { @@ -239,11 +233,7 @@ class TColumnEngineChanges { TString DebugString() const; ui64 TotalBlobsSize() const { - ui64 size = 0; - for (const auto& [_, blob] : Blobs) { - size += blob.size(); - } - return size; + return Blobs.GetTotalBlobsSize(); } }; diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup.cpp index c86be3c567e2..474b82b114b5 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup.cpp @@ -18,15 +18,20 @@ void TCleanupColumnEngineChanges::DoDebugString(TStringOutput& out) const { void TCleanupColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { THashSet pathIds; if (self) { + THashMap> blobIdsByStorage; for (auto&& p : PortionsToDrop) { p.RemoveFromDatabase(context.DBWrapper); - auto removing = BlobsAction.GetRemoving(p); - for (auto&& r : p.Records) { - removing->DeclareRemove((TTabletId)self->TabletID(), r.BlobRange.BlobId); - } + p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPathId()); } + for (auto&& i : blobIdsByStorage) { + auto action = BlobsAction.GetRemoving(i.first); + for (auto&& b : i.second) { + action->DeclareRemove((TTabletId)self->TabletID(), b); + } + } + if (context.DB) { for (auto&& p : pathIds) { self->TablesManager.TryFinalizeDropPath(*context.DB, p); diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index da1cb7f5eb2a..01f2a2cc777c 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -31,11 +31,17 @@ void TCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { TBase::DoStart(self); Y_ABORT_UNLESS(SwitchedPortions.size()); + THashMap> blobRanges; + auto& index = self.GetIndexAs().GetVersionedIndex(); for (const auto& p : SwitchedPortions) { Y_ABORT_UNLESS(!p.Empty()); - auto action = BlobsAction.GetReading(p); - for (const auto& rec : p.Records) { - action->AddRange(rec.BlobRange); + p.FillBlobRangesByStorage(blobRanges, index); + } + + for (const auto& p : blobRanges) { + auto action = BlobsAction.GetReading(p.first); + for (auto&& b: p.second) { + action->AddRange(b); } } diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 973d07952d58..c92b59ea33b5 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -16,8 +16,7 @@ namespace NKikimr::NOlap::NCompaction { -void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TConstructionContext& context) noexcept { - std::vector portions = TPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs); +void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TConstructionContext& context, std::vector&& portions) noexcept { std::vector> batchResults; auto resultSchema = context.SchemaVersions.GetLastSchema(); { @@ -42,14 +41,18 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TCon } } -void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstructionContext& context) noexcept { - std::vector portions = TPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs); +void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstructionContext& context, std::vector&& portions) noexcept { static const TString portionIdFieldName = "$$__portion_id"; static const TString portionRecordIndexFieldName = "$$__portion_record_idx"; static const std::shared_ptr portionIdField = std::make_shared(portionIdFieldName, std::make_shared()); static const std::shared_ptr portionRecordIndexField = std::make_shared(portionRecordIndexFieldName, std::make_shared()); auto resultSchema = context.SchemaVersions.GetLastSchema(); + TEntityGroups groups(IStoragesManager::DefaultStorageId); + for (auto&& i : resultSchema->GetIndexInfo().GetEntityIds()) { + groups.Add(i, resultSchema->GetIndexInfo().GetEntityStorageId(i, "")); + } + std::vector pkFieldNames = resultSchema->GetIndexInfo().GetReplaceKey()->field_names(); std::set pkFieldNamesSet(pkFieldNames.begin(), pkFieldNames.end()); for (auto&& i : TIndexInfo::GetSpecialColumnNames()) { @@ -159,6 +162,7 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc } ui32 batchIdx = 0; + for (auto&& columnChunks : chunkGroups) { auto batchResult = batchResults[batchIdx]; ++batchIdx; @@ -189,10 +193,9 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc ui32 recordIdx = 0; for (auto&& i : packs) { - TGeneralSerializedSlice slice(std::move(i)); + TGeneralSerializedSlice slice(std::move(i), GetSplitSettings()); auto b = batchResult->Slice(recordIdx, slice.GetRecordsCount()); - std::vector>> chunksByBlobs = slice.GroupChunksByBlobs(); - AppendedPortions.emplace_back(TPortionInfoWithBlobs::BuildByBlobs(chunksByBlobs, nullptr, GranuleMeta->GetPathId(), resultSchema->GetSnapshot(), SaverContext.GetStorageOperator())); + AppendedPortions.emplace_back(TPortionInfoWithBlobs::BuildByBlobs(slice.GroupChunksByBlobs(groups), nullptr, GranuleMeta->GetPathId(), resultSchema->GetSnapshot(), SaverContext.GetStoragesManager())); NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(resultSchema->GetIndexInfo().GetReplaceKey())); NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot()); AppendedPortions.back().GetPortionInfo().AddMetadata(*resultSchema, primaryKeys, snapshotKeys, SaverContext.GetTierName()); @@ -221,10 +224,13 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc NChanges::TGeneralCompactionCounters::OnPortionsKind(insertedPortionsSize, compactedPortionsSize, otherPortionsSize); NChanges::TGeneralCompactionCounters::OnRepackPortions(portionsCount, portionsSize); - if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { - BuildAppendedPortionsByChunks(context); - } else { - BuildAppendedPortionsByFullBatches(context); + { + std::vector portions = TPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs, context.SchemaVersions, SaverContext.GetStoragesManager()); + if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { + BuildAppendedPortionsByChunks(context, std::move(portions)); + } else { + BuildAppendedPortionsByFullBatches(context, std::move(portions)); + } } if (IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD)) { diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.h b/ydb/core/tx/columnshard/engines/changes/general_compaction.h index f57ae68f4282..bec51bef6a0d 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.h @@ -9,8 +9,8 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { using TBase = TCompactColumnEngineChanges; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; std::map CheckPoints; - void BuildAppendedPortionsByFullBatches(TConstructionContext& context) noexcept; - void BuildAppendedPortionsByChunks(TConstructionContext& context) noexcept; + void BuildAppendedPortionsByFullBatches(TConstructionContext& context, std::vector&& portions) noexcept; + void BuildAppendedPortionsByChunks(TConstructionContext& context, std::vector&& portions) noexcept; protected: virtual TConclusionStatus DoConstructBlobs(TConstructionContext& context) noexcept override; virtual TPortionMeta::EProduced GetResultProducedClass() const override { diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.cpp b/ydb/core/tx/columnshard/engines/changes/indexation.cpp index 4828ffeb6013..d4c730e8a992 100644 --- a/ydb/core/tx/columnshard/engines/changes/indexation.cpp +++ b/ydb/core/tx/columnshard/engines/changes/indexation.cpp @@ -22,7 +22,7 @@ void TInsertColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { Y_ABORT_UNLESS(DataToIndex.size()); auto reading = BlobsAction.GetReading(IStoragesManager::DefaultStorageId); for (auto&& insertedData : DataToIndex) { - reading->AddRange(insertedData.GetBlobRange(), insertedData.GetBlobData().value_or("")); + reading->AddRange(insertedData.GetBlobRange(), insertedData.GetBlobData()); } self.BackgroundController.StartIndexing(*this); @@ -74,12 +74,10 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont std::shared_ptr batch; { - auto itBlobData = Blobs.find(blobRange); - Y_ABORT_UNLESS(itBlobData != Blobs.end(), "Data for range %s has not been read", blobRange.ToString().c_str()); - Y_ABORT_UNLESS(!itBlobData->second.empty(), "Blob data not present"); + const auto blobData = Blobs.Extract(IStoragesManager::DefaultStorageId, blobRange); + Y_ABORT_UNLESS(blobData.size(), "Blob data not present"); // Prepare batch - batch = NArrow::DeserializeBatch(itBlobData->second, indexInfo.ArrowSchema()); - Blobs.erase(itBlobData); + batch = NArrow::DeserializeBatch(blobData, indexInfo.ArrowSchema()); AFL_VERIFY(batch)("event", "cannot_parse") ("data_snapshot", TStringBuilder() << inserted.GetSnapshot()) ("index_snapshot", TStringBuilder() << blobSchema->GetSnapshot()); @@ -92,7 +90,7 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(pathBatches[inserted.PathId].back(), resultSchema->GetIndexInfo().GetReplaceKey())); } - Y_ABORT_UNLESS(Blobs.empty()); + Y_ABORT_UNLESS(Blobs.IsEmpty()); const std::vector comparableColumns = resultSchema->GetIndexInfo().GetReplaceKey()->field_names(); for (auto& [pathId, batches] : pathBatches) { NIndexedReader::TMergePartialStream stream(resultSchema->GetIndexInfo().GetReplaceKey(), resultSchema->GetIndexInfo().ArrowSchemaWithSpecials(), false); diff --git a/ydb/core/tx/columnshard/engines/changes/ttl.cpp b/ydb/core/tx/columnshard/engines/changes/ttl.cpp index b88c4f7ec123..b5218fbcd057 100644 --- a/ydb/core/tx/columnshard/engines/changes/ttl.cpp +++ b/ydb/core/tx/columnshard/engines/changes/ttl.cpp @@ -14,12 +14,16 @@ void TTTLColumnEngineChanges::DoDebugString(TStringOutput& out) const { void TTTLColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { Y_ABORT_UNLESS(PortionsToEvict.size() || PortionsToRemove.size()); + THashMap> blobRanges; + auto& index = self.GetIndexAs().GetVersionedIndex(); for (const auto& p : PortionsToEvict) { Y_ABORT_UNLESS(!p.GetPortionInfo().Empty()); - - auto agent = BlobsAction.GetReading(p.GetPortionInfo()); - for (const auto& rec : p.GetPortionInfo().Records) { - agent->AddRange(rec.BlobRange); + p.GetPortionInfo().FillBlobRangesByStorage(blobRanges, index); + } + for (auto&& i : blobRanges) { + auto action = BlobsAction.GetReading(i.first); + for (auto&& b : i.second) { + action->AddRange(b); } } self.BackgroundController.StartTtl(); @@ -29,7 +33,7 @@ void TTTLColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TChan self.BackgroundController.FinishTtl(); } -std::optional TTTLColumnEngineChanges::UpdateEvictedPortion(TPortionForEviction& info, THashMap& srcBlobs, +std::optional TTTLColumnEngineChanges::UpdateEvictedPortion(TPortionForEviction& info, NBlobOperations::NRead::TCompositeReadBlobs& srcBlobs, TConstructionContext& context) const { const TPortionInfo& portionInfo = info.GetPortionInfo(); auto& evictFeatures = info.GetFeatures(); @@ -38,29 +42,26 @@ std::optional TTTLColumnEngineChanges::UpdateEvictedPorti auto* tiering = Tiering.FindPtr(evictFeatures.PathId); Y_ABORT_UNLESS(tiering); auto serializer = tiering->GetSerializer(evictFeatures.TargetTierName); + auto blobSchema = context.SchemaVersions.GetSchema(portionInfo.GetMinSnapshot()); + auto portionWithBlobs = TPortionInfoWithBlobs::RestorePortion(portionInfo, srcBlobs, blobSchema->GetIndexInfo(), SaverContext.GetStoragesManager()); if (!serializer) { // Nothing to recompress. We have no other kinds of evictions yet. evictFeatures.DataChanges = false; - auto result = TPortionInfoWithBlobs::RestorePortion(portionInfo, srcBlobs); - result.GetPortionInfo().InitOperator(evictFeatures.StorageOperator, true); - result.GetPortionInfo().MutableMeta().SetTierName(evictFeatures.TargetTierName); - return result; + portionWithBlobs.GetPortionInfo().MutableMeta().SetTierName(evictFeatures.TargetTierName); + return portionWithBlobs; } - auto blobSchema = context.SchemaVersions.GetSchema(portionInfo.GetMinSnapshot()); auto resultSchema = context.SchemaVersions.GetLastSchema(); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("portion_for_eviction", portionInfo.DebugString()); - TSaverContext saverContext(evictFeatures.StorageOperator, SaverContext.GetStoragesManager()); + TSaverContext saverContext(SaverContext.GetStoragesManager()); saverContext.SetTierName(evictFeatures.TargetTierName).SetExternalSerializer(*serializer); - auto withBlobs = TPortionInfoWithBlobs::RestorePortion(portionInfo, srcBlobs); - withBlobs.GetPortionInfo().InitOperator(evictFeatures.StorageOperator, true); - withBlobs.GetPortionInfo().MutableMeta().SetTierName(evictFeatures.TargetTierName); - return withBlobs.ChangeSaver(resultSchema, saverContext); + portionWithBlobs.GetPortionInfo().MutableMeta().SetTierName(evictFeatures.TargetTierName); + return portionWithBlobs.ChangeSaver(resultSchema, saverContext, SaverContext.GetStoragesManager()->GetOperatorVerified(saverContext.GetTierName())); } NKikimr::TConclusionStatus TTTLColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept { - Y_ABORT_UNLESS(!Blobs.empty()); + Y_ABORT_UNLESS(!Blobs.IsEmpty()); Y_ABORT_UNLESS(!PortionsToEvict.empty()); for (auto&& info : PortionsToEvict) { diff --git a/ydb/core/tx/columnshard/engines/changes/ttl.h b/ydb/core/tx/columnshard/engines/changes/ttl.h index 9784f1bcf4d3..8feb25efb69f 100644 --- a/ydb/core/tx/columnshard/engines/changes/ttl.h +++ b/ydb/core/tx/columnshard/engines/changes/ttl.h @@ -39,7 +39,7 @@ class TTTLColumnEngineChanges: public TChangesWithAppend { } }; - std::optional UpdateEvictedPortion(TPortionForEviction& info, THashMap& srcBlobs, + std::optional UpdateEvictedPortion(TPortionForEviction& info, NBlobOperations::NRead::TCompositeReadBlobs& srcBlobs, TConstructionContext& context) const; std::vector PortionsToEvict; // {portion, TPortionEvictionFeatures} diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 2b8ad23531c9..d5435459613e 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -97,6 +97,10 @@ std::vector TChangesWithAppend::MakeAppendedPortions(cons Y_ABORT_UNLESS(batch->num_rows()); auto resultSchema = context.SchemaVersions.GetSchema(snapshot); + TEntityGroups groups(IStoragesManager::DefaultStorageId); + for (auto&& i : resultSchema->GetIndexInfo().GetEntityIds()) { + groups.Add(i, resultSchema->GetIndexInfo().GetEntityStorageId(i, "")); + } std::shared_ptr stats = std::make_shared(); if (granuleMeta) { @@ -118,10 +122,9 @@ std::vector TChangesWithAppend::MakeAppendedPortions(cons ui32 recordIdx = 0; for (auto&& i : packs) { - TGeneralSerializedSlice slice(std::move(i)); + TGeneralSerializedSlice slice(std::move(i), GetSplitSettings()); auto b = batch->Slice(recordIdx, slice.GetRecordsCount()); - std::vector>> chunksByBlobs = slice.GroupChunksByBlobs(); - out.emplace_back(TPortionInfoWithBlobs::BuildByBlobs(chunksByBlobs, nullptr, pathId, snapshot, SaverContext.GetStorageOperator())); + out.emplace_back(TPortionInfoWithBlobs::BuildByBlobs(slice.GroupChunksByBlobs(groups), nullptr, pathId, snapshot, SaverContext.GetStoragesManager())); NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(resultSchema->GetIndexInfo().GetReplaceKey())); NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot()); out.back().GetPortionInfo().AddMetadata(*resultSchema, primaryKeys, snapshotKeys, SaverContext.GetTierName()); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 38f9f1b2685b..be90e55e90f3 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -225,8 +225,7 @@ bool TColumnEngineForLogs::LoadCounters(IDbWrapper& db) { std::shared_ptr TColumnEngineForLogs::StartInsert(std::vector&& dataToIndex) noexcept { Y_ABORT_UNLESS(dataToIndex.size()); - TSaverContext saverContext(StoragesManager->GetInsertOperator(), StoragesManager); - + TSaverContext saverContext(StoragesManager); auto changes = std::make_shared(std::move(dataToIndex), TSplitSettings(), saverContext); auto pkSchema = VersionedIndex.GetLastSchema()->GetIndexInfo().GetReplaceKey(); @@ -407,7 +406,7 @@ TDuration TColumnEngineForLogs::ProcessTiering(const ui64 pathId, const TTiering } if (currentTierName != tierName) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "tiering switch detected")("from", currentTierName)("to", tierName); - context.Changes->AddPortionToEvict(*info, TPortionEvictionFeatures(tierName, pathId, StoragesManager->GetOperator(tierName))); + context.Changes->AddPortionToEvict(*info, TPortionEvictionFeatures(tierName, pathId)); context.AppPortionForEvictionChecker(*info); SignalCounters.OnPortionToEvict(info->BlobsBytes()); } @@ -470,7 +469,7 @@ std::shared_ptr TColumnEngineForLogs::StartTtl(const TH ("internal", EvictionsController.MutableNextCheckInstantForTierings().size()) ; - TSaverContext saverContext(StoragesManager->GetDefaultOperator(), StoragesManager); + TSaverContext saverContext(StoragesManager); auto changes = std::make_shared(TSplitSettings(), saverContext); diff --git a/ydb/core/tx/columnshard/engines/portions/column_record.h b/ydb/core/tx/columnshard/engines/portions/column_record.h index 17dee9a00d53..8064410fcb88 100644 --- a/ydb/core/tx/columnshard/engines/portions/column_record.h +++ b/ydb/core/tx/columnshard/engines/portions/column_record.h @@ -114,6 +114,9 @@ class TColumnRecord { ui16 GetChunkIdx() const { return Chunk; } + const TBlobRange& GetBlobRange() const { + return BlobRange; + } NKikimrColumnShardDataSharingProto::TColumnRecord SerializeToProto() const; static TConclusion BuildFromProto(const NKikimrColumnShardDataSharingProto::TColumnRecord& proto, const TIndexInfo& indexInfo) { diff --git a/ydb/core/tx/columnshard/engines/portions/common.h b/ydb/core/tx/columnshard/engines/portions/common.h index 311563918469..1231a1e9f5f3 100644 --- a/ydb/core/tx/columnshard/engines/portions/common.h +++ b/ydb/core/tx/columnshard/engines/portions/common.h @@ -33,5 +33,11 @@ class TChunkAddress { TString DebugString() const; }; - } + +template<> +struct ::THash { + inline ui64 operator()(const NKikimr::NOlap::TChunkAddress& a) const { + return ((ui64)a.GetEntityId()) << 16 + a.GetChunkIdx(); + } +}; diff --git a/ydb/core/tx/columnshard/engines/portions/index_chunk.h b/ydb/core/tx/columnshard/engines/portions/index_chunk.h index 74eb910de1d3..32b4d81d2c41 100644 --- a/ydb/core/tx/columnshard/engines/portions/index_chunk.h +++ b/ydb/core/tx/columnshard/engines/portions/index_chunk.h @@ -35,6 +35,10 @@ class TIndexChunk { TIndexChunk() = default; TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TIndexChunk& proto); public: + TChunkAddress GetAddress() const { + return TChunkAddress(IndexId, ChunkIdx); + } + TIndexChunk(const ui32 indexId, const ui32 chunkIdx, const ui32 recordsCount, const ui64 rawBytes, const TBlobRange& blobRange) : IndexId(indexId) , ChunkIdx(chunkIdx) diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index b8ff79dc25cb..674b6d84afe1 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -1,12 +1,15 @@ #include "portion_info.h" -#include -#include +#include #include +#include +#include +#include #include -#include #include #include +#include + namespace NKikimr::NOlap { const TColumnRecord& TPortionInfo::AppendOneChunkColumn(TColumnRecord&& record) { @@ -60,7 +63,7 @@ std::shared_ptr TPortionInfo::MaxValue(ui32 columnId) const { } TPortionInfo TPortionInfo::CopyWithFilteredColumns(const THashSet& columnIds) const { - TPortionInfo result(PathId, Portion, GetMinSnapshot(), BlobsOperator); + TPortionInfo result(PathId, Portion, GetMinSnapshot()); result.Meta = Meta; result.Records.reserve(columnIds.size()); @@ -134,16 +137,13 @@ TString TPortionInfo::DebugString(const bool withDetails) const { if (RemoveSnapshot.Valid()) { sb << "remove_snapshot:(" << RemoveSnapshot.DebugString() << ");"; } - if (BlobsOperator) { - sb << "blobs_operator:" << BlobsOperator->DebugString() << ";"; - } sb << "chunks:(" << Records.size() << ");"; if (IS_TRACE_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD)) { - std::set blobIds; + THashSet blobRanges; for (auto&& i : Records) { - blobIds.emplace(::ToString(i.BlobRange.BlobId)); + blobRanges.emplace(i.BlobRange); } - sb << "blobs:" << JoinSeq(",", blobIds) << ";blobs_count:" << blobIds.size() << ";"; + sb << "blobs:" << JoinSeq(",", blobRanges) << ";ranges_count:" << blobRanges.size() << ";"; } return sb << ")"; } @@ -337,6 +337,89 @@ TConclusion TPortionInfo::BuildFromProto(const NKikimrColumnShardD return result; } +THashMap TPortionInfo::DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobs, const TIndexInfo& indexInfo) const { + THashMap result; + for (auto&& i : blobs) { + for (auto&& b : i.second) { + bool found = false; + TString columnStorageId; + ui32 columnId = 0; + for (auto&& record : Records) { + if (record.GetBlobRange() == b.first) { + if (columnId != record.GetColumnId()) { + columnStorageId = GetColumnStorageId(record.GetColumnId(), indexInfo); + } + if (columnStorageId != i.first) { + continue; + } + result.emplace(record.GetAddress(), std::move(b.second)); + found = true; + break; + } + } + if (found) { + continue; + } + for (auto&& record : Indexes) { + if (record.GetBlobRange() == b.first) { + if (columnId != record.GetIndexId()) { + columnStorageId = indexInfo.GetIndexStorageId(record.GetIndexId()); + } + if (columnStorageId != i.first) { + continue; + } + result.emplace(record.GetAddress(), std::move(b.second)); + found = true; + break; + } + } + AFL_VERIFY(found)("blobs", blobs.DebugString())("records", DebugString(true))("problem", b.first); + } + } + return result; +} + +const TString& TPortionInfo::GetColumnStorageId(const ui32 columnId, const TIndexInfo& indexInfo) const { + return indexInfo.GetColumnStorageId(columnId, GetMeta().GetTierName()); +} + +void TPortionInfo::FillBlobRangesByStorage(THashMap>& result, const TIndexInfo& indexInfo) const { + for (auto&& i : Records) { + const TString& storageId = GetColumnStorageId(i.GetColumnId(), indexInfo); + AFL_VERIFY(result[storageId].emplace(i.GetBlobRange()).second)("blob_id", i.GetBlobRange().ToString()); + } + for (auto&& i : Indexes) { + const TString& storageId = indexInfo.GetIndexStorageId(i.GetIndexId()); + AFL_VERIFY(result[storageId].emplace(i.GetBlobRange()).second)("blob_id", i.GetBlobRange().ToString()); + } +} + +void TPortionInfo::FillBlobRangesByStorage(THashMap>& result, const TVersionedIndex& index) const { + auto schema = index.GetSchema(GetMinSnapshot()); + return FillBlobRangesByStorage(result, schema->GetIndexInfo()); +} + +void TPortionInfo::FillBlobIdsByStorage(THashMap>& result, const TIndexInfo& indexInfo) const { + THashMap> local; + for (auto&& i : Records) { + const TString& storageId = GetColumnStorageId(i.GetColumnId(), indexInfo); + if (local[storageId].emplace(i.BlobRange.BlobId).second) { + AFL_VERIFY(result[storageId].emplace(i.GetBlobRange().BlobId).second)("blob_id", i.GetBlobRange().BlobId.ToStringNew()); + } + } + for (auto&& i : Indexes) { + const TString& storageId = indexInfo.GetIndexStorageId(i.GetIndexId()); + if (local[storageId].emplace(i.GetBlobRange().BlobId).second) { + AFL_VERIFY(result[storageId].emplace(i.GetBlobRange().BlobId).second)("blob_id", i.GetBlobRange().BlobId.ToStringNew()); + } + } +} + +void TPortionInfo::FillBlobIdsByStorage(THashMap>& result, const TVersionedIndex& index) const { + auto schema = index.GetSchema(GetMinSnapshot()); + return FillBlobIdsByStorage(result, schema->GetIndexInfo()); +} + std::shared_ptr TPortionInfo::TPreparedColumn::Assemble() const { Y_ABORT_UNLESS(!Blobs.empty()); diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 3d8e1522d4d5..09d006259ce5 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -17,7 +17,12 @@ class TPortionInfo; namespace NKikimr::NOlap { +namespace NBlobOperations::NRead { +class TCompositeReadBlobs; +} + struct TIndexInfo; +class TVersionedIndex; class IDbWrapper; class TPortionInfo { @@ -29,12 +34,15 @@ class TPortionInfo { TSnapshot RemoveSnapshot = TSnapshot::Zero(); // {XPlanStep, XTxId} is snapshot where the blob has been removed (i.e. compacted into another one) TPortionMeta Meta; - std::shared_ptr BlobsOperator; ui64 DeprecatedGranuleId = 0; YDB_READONLY_DEF(std::vector, Indexes); TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& info); public: + THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobs, const TIndexInfo& indexInfo) const; + + const TString& GetColumnStorageId(const ui32 columnId, const TIndexInfo& indexInfo) const; + ui64 GetTxVolume() const; // fake-correct method for determ volume on rewrite this portion in transaction progress class TPage { @@ -127,10 +135,6 @@ class TPortionInfo { return Portion; } - bool HasStorageOperator() const { - return !!BlobsOperator; - } - NJson::TJsonValue SerializeToJsonVisual() const { NJson::TJsonValue result = NJson::JSON_MAP; result.InsertValue("id", Portion); @@ -146,22 +150,8 @@ class TPortionInfo { return result; } - void InitOperator(const std::shared_ptr& bOperator, const bool rewrite) { - if (rewrite) { - AFL_VERIFY(!!BlobsOperator); - } else { - AFL_VERIFY(!BlobsOperator); - } - AFL_VERIFY(!!bOperator); - BlobsOperator = bOperator; - } - static constexpr const ui32 BLOB_BYTES_LIMIT = 8 * 1024 * 1024; - const std::shared_ptr& GetBlobsStorage() const { - Y_ABORT_UNLESS(BlobsOperator); - return BlobsOperator; - } std::vector GetColumnChunksPointers(const ui32 columnId) const; TSerializationStats GetSerializationStat(const ISnapshotSchema& schema) const { @@ -176,7 +166,6 @@ class TPortionInfo { void ResetMeta() { Meta = TPortionMeta(); - BlobsOperator = nullptr; } const TPortionMeta& GetMeta() const { @@ -215,11 +204,10 @@ class TPortionInfo { return TPortionInfo(); } - TPortionInfo(const ui64 pathId, const ui64 portionId, const TSnapshot& minSnapshot, const std::shared_ptr& blobsOperator) + TPortionInfo(const ui64 pathId, const ui64 portionId, const TSnapshot& minSnapshot) : PathId(pathId) , Portion(portionId) - , MinSnapshot(minSnapshot) - , BlobsOperator(blobsOperator) { + , MinSnapshot(minSnapshot) { } TString DebugString(const bool withDetails = false) const; @@ -369,37 +357,17 @@ class TPortionInfo { } - THashMap> GetBlobIdsByStorage() const { + THashMap> GetBlobIdsByStorage(const TIndexInfo& indexInfo) const { THashMap> result; - FillBlobIdsByStorage(result); + FillBlobIdsByStorage(result, indexInfo); return result; } - void FillBlobIdsByStorage(THashMap>& result) const { - const TString& storageId = GetMeta().GetTierName() ? GetMeta().GetTierName() : IStoragesManager::DefaultStorageId; - THashMap> local; - for (auto&& i : Records) { - if (local[storageId].emplace(i.BlobRange.BlobId).second) { - AFL_VERIFY(result[storageId].emplace(i.BlobRange.BlobId).second); - } - } - for (auto&& i : Indexes) { - if (local[storageId].emplace(i.GetBlobRange().BlobId).second) { - AFL_VERIFY(result[storageId].emplace(i.GetBlobRange().BlobId).second); - } - } - } + void FillBlobRangesByStorage(THashMap>& result, const TIndexInfo& indexInfo) const; + void FillBlobRangesByStorage(THashMap>& result, const TVersionedIndex& index) const; - THashSet GetBlobIds() const { - THashSet result; - for (auto&& i : Records) { - result.emplace(i.BlobRange.BlobId); - } - for (auto&& i : Indexes) { - result.emplace(i.GetBlobRange().BlobId); - } - return result; - } + void FillBlobIdsByStorage(THashMap>& result, const TIndexInfo& indexInfo) const; + void FillBlobIdsByStorage(THashMap>& result, const TVersionedIndex& index) const; ui32 GetRecordsCount() const { ui32 result = 0; @@ -613,7 +581,7 @@ class TPortionInfo { template TPreparedBatchData PrepareForAssemble(const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, - THashMap& blobsData) const { + THashMap& blobsData) const { std::vector columns; auto arrowResultSchema = resultSchema.GetSchema(); columns.reserve(arrowResultSchema->num_fields()); @@ -637,7 +605,7 @@ class TPortionInfo { AFL_VERIFY((ui32)resultPos < columns.size()); currentAssembler = &columns[resultPos]; } - auto it = blobsData.find(rec.BlobRange); + auto it = blobsData.find(rec.GetAddress()); Y_ABORT_UNLESS(it != blobsData.end()); currentAssembler->AddBlobInfo(rec.Chunk, std::move(it->second)); blobsData.erase(it); @@ -654,9 +622,8 @@ class TPortionInfo { return TPreparedBatchData(std::move(preparedColumns), arrowResultSchema, rowsCount); } - std::shared_ptr AssembleInBatch(const ISnapshotSchema& dataSchema, - const ISnapshotSchema& resultSchema, - THashMap& data) const { + std::shared_ptr AssembleInBatch(const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, + THashMap& data) const { auto batch = PrepareForAssemble(dataSchema, resultSchema, data).Assemble(); Y_ABORT_UNLESS(batch->Validate().ok()); return batch; diff --git a/ydb/core/tx/columnshard/engines/portions/with_blobs.cpp b/ydb/core/tx/columnshard/engines/portions/with_blobs.cpp index 9d8adcac99f2..4c9a3a6977c8 100644 --- a/ydb/core/tx/columnshard/engines/portions/with_blobs.cpp +++ b/ydb/core/tx/columnshard/engines/portions/with_blobs.cpp @@ -1,6 +1,8 @@ #include "with_blobs.h" #include #include +#include +#include namespace NKikimr::NOlap { @@ -54,10 +56,10 @@ std::shared_ptr TPortionInfoWithBlobs::GetBatch(const ISnaps Y_ABORT_UNLESS(data); if (columnNames.empty()) { if (!CachedBatch) { - THashMap blobs; + THashMap blobs; for (auto&& i : PortionInfo.Records) { - blobs[i.BlobRange] = GetBlobByRangeVerified(i.ColumnId, i.Chunk); - Y_ABORT_UNLESS(blobs[i.BlobRange].size() == i.BlobRange.Size); + blobs[i.GetAddress()] = GetBlobByRangeVerified(i.ColumnId, i.Chunk); + Y_ABORT_UNLESS(blobs[i.GetAddress()].size() == i.BlobRange.Size); } CachedBatch = PortionInfo.AssembleInBatch(*data, result, blobs); Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(*CachedBatch, result.GetIndexInfo().GetReplaceKey())); @@ -73,25 +75,28 @@ std::shared_ptr TPortionInfoWithBlobs::GetBatch(const ISnaps return result; } else { auto filteredSchema = std::make_shared(data, columnNames); - THashMap blobs; + THashMap blobs; for (auto&& i : PortionInfo.Records) { - blobs[i.BlobRange] = GetBlobByRangeVerified(i.ColumnId, i.Chunk); - Y_ABORT_UNLESS(blobs[i.BlobRange].size() == i.BlobRange.Size); + blobs[i.GetAddress()] = GetBlobByRangeVerified(i.ColumnId, i.Chunk); + Y_ABORT_UNLESS(blobs[i.GetAddress()].size() == i.BlobRange.Size); } return PortionInfo.AssembleInBatch(*data, *filteredSchema, blobs); } } -NKikimr::NOlap::TPortionInfoWithBlobs TPortionInfoWithBlobs::RestorePortion(const TPortionInfo& portion, THashMap& blobs) { +NKikimr::NOlap::TPortionInfoWithBlobs TPortionInfoWithBlobs::RestorePortion(const TPortionInfo& portion, NBlobOperations::NRead::TCompositeReadBlobs& blobs, const TIndexInfo& indexInfo, const std::shared_ptr& operators) { TPortionInfoWithBlobs result(portion); const auto pred = [](const TColumnRecord& l, const TColumnRecord& r) { return l.GetAddress() < r.GetAddress(); }; std::sort(result.PortionInfo.Records.begin(), result.PortionInfo.Records.end(), pred); - THashMap> recordsByBlob; + THashMap>> records; + for (auto&& c : result.PortionInfo.Records) { - auto& blobRecords = recordsByBlob[c.BlobRange.BlobId]; + const TString& storageId = portion.GetColumnStorageId(c.GetColumnId(), indexInfo); + auto& storageRecords = records[storageId]; + auto& blobRecords = storageRecords[c.BlobRange.BlobId]; blobRecords.emplace_back(&c); } @@ -99,33 +104,38 @@ NKikimr::NOlap::TPortionInfoWithBlobs TPortionInfoWithBlobs::RestorePortion(cons return l->BlobRange.Offset < r->BlobRange.Offset; }; - for (auto&& i : recordsByBlob) { - std::sort(i.second.begin(), i.second.end(), predOffset); - auto builder = result.StartBlob(); - for (auto&& d : i.second) { - auto itBlob = blobs.find(d->BlobRange); - Y_ABORT_UNLESS(itBlob != blobs.end()); - builder.RestoreChunk(std::make_shared(*d, itBlob->second)); - blobs.erase(itBlob); + for (auto&& [storageId, recordsByBlob]: records) { + auto storage = operators->GetOperatorVerified(storageId); + for (auto&& i : recordsByBlob) { + std::sort(i.second.begin(), i.second.end(), predOffset); + auto builder = result.StartBlob(storage); + for (auto&& d : i.second) { + auto blobData = blobs.Extract(portion.GetColumnStorageId(d->GetColumnId(), indexInfo), d->BlobRange); + builder.RestoreChunk(std::make_shared(*d, std::move(blobData))); + } } } return result; } -std::vector TPortionInfoWithBlobs::RestorePortions(const std::vector& portions, THashMap& blobs) { +std::vector TPortionInfoWithBlobs::RestorePortions(const std::vector& portions, NBlobOperations::NRead::TCompositeReadBlobs& blobs, + const TVersionedIndex& tables, const std::shared_ptr& operators) { std::vector result; for (auto&& i : portions) { - result.emplace_back(RestorePortion(i, blobs)); + const auto schema = tables.GetSchema(i.GetMinSnapshot()); + result.emplace_back(RestorePortion(i, blobs, schema->GetIndexInfo(), operators)); } return result; } -NKikimr::NOlap::TPortionInfoWithBlobs TPortionInfoWithBlobs::BuildByBlobs(std::vector>>& chunksByBlobs, std::shared_ptr batch, const ui64 granule, - const TSnapshot& snapshot, const std::shared_ptr& bStorageOperator) { - TPortionInfoWithBlobs result(TPortionInfo(granule, 0, snapshot, bStorageOperator), batch); - for (auto& blob : chunksByBlobs) { - auto blobInfo = result.StartBlob(); - for (auto&& chunk : blob) { +NKikimr::NOlap::TPortionInfoWithBlobs TPortionInfoWithBlobs::BuildByBlobs(std::vector&& chunks, + std::shared_ptr batch, const ui64 granule, const TSnapshot& snapshot, const std::shared_ptr& operators) +{ + TPortionInfoWithBlobs result(TPortionInfo(granule, 0, snapshot), batch); + for (auto&& blob: chunks) { + auto storage = operators->GetOperatorVerified(blob.GetGroupName()); + auto blobInfo = result.StartBlob(storage); + for (auto&& chunk : blob.GetChunks()) { blobInfo.AddChunk(chunk); } } @@ -137,7 +147,7 @@ NKikimr::NOlap::TPortionInfoWithBlobs TPortionInfoWithBlobs::BuildByBlobs(std::v return result; } -std::optional TPortionInfoWithBlobs::ChangeSaver(ISnapshotSchema::TPtr currentSchema, const TSaverContext& saverContext) const { +std::optional TPortionInfoWithBlobs::ChangeSaver(ISnapshotSchema::TPtr currentSchema, const TSaverContext& saverContext, const std::shared_ptr& bOperator) const { TPortionInfoWithBlobs result(PortionInfo, CachedBatch); result.PortionInfo.Records.clear(); std::optional bBuilder; @@ -153,7 +163,7 @@ std::optional TPortionInfoWithBlobs::Chan return {}; } if (!bBuilder || result.GetBlobs().back().GetSize() + newBlob.size() >= TPortionInfo::BLOB_BYTES_LIMIT) { - bBuilder = result.StartBlob(); + bBuilder = result.StartBlob(bOperator); } Y_ABORT_UNLESS(rb); Y_ABORT_UNLESS(rb->num_columns() == 1); diff --git a/ydb/core/tx/columnshard/engines/portions/with_blobs.h b/ydb/core/tx/columnshard/engines/portions/with_blobs.h index 1a7a6b7192cb..ea6997d37909 100644 --- a/ydb/core/tx/columnshard/engines/portions/with_blobs.h +++ b/ydb/core/tx/columnshard/engines/portions/with_blobs.h @@ -1,11 +1,15 @@ #pragma once #include "portion_info.h" -#include -#include #include +#include +#include + +#include namespace NKikimr::NOlap { +class TVersionedIndex; + class TPortionInfoWithBlobs { public: class TBlobInfo { @@ -13,12 +17,19 @@ class TPortionInfoWithBlobs { using TBlobChunks = std::map>; YDB_READONLY(ui64, Size, 0); YDB_READONLY_DEF(TBlobChunks, Chunks); + YDB_READONLY_DEF(std::shared_ptr, Operator); std::vector> ChunksOrdered; mutable std::optional ResultBlob; void AddChunk(TPortionInfoWithBlobs& owner, const std::shared_ptr& chunk); void RestoreChunk(const TPortionInfoWithBlobs& owner, const std::shared_ptr& chunk); public: + TBlobInfo(const std::shared_ptr& bOperator) + : Operator(bOperator) + { + + } + class TBuilder { private: TBlobInfo* OwnerBlob; @@ -71,14 +82,16 @@ class TPortionInfoWithBlobs { PortionInfo = portionInfo; } - TBlobInfo::TBuilder StartBlob() { - Blobs.emplace_back(TBlobInfo()); + TBlobInfo::TBuilder StartBlob(const std::shared_ptr& bOperator) { + Blobs.emplace_back(TBlobInfo(bOperator)); return TBlobInfo::TBuilder(Blobs.back(), *this); } public: - static std::vector RestorePortions(const std::vector& portions, THashMap& blobs); - static TPortionInfoWithBlobs RestorePortion(const TPortionInfo& portions, THashMap& blobs); + static std::vector RestorePortions(const std::vector& portions, NBlobOperations::NRead::TCompositeReadBlobs& blobs, + const TVersionedIndex& tables, const std::shared_ptr& operators); + static TPortionInfoWithBlobs RestorePortion(const TPortionInfo& portions, NBlobOperations::NRead::TCompositeReadBlobs& blobs, + const TIndexInfo& indexInfo, const std::shared_ptr& operators); std::shared_ptr GetBatch(const ISnapshotSchema::TPtr& data, const ISnapshotSchema& result, const std::set& columnNames = {}) const; @@ -90,10 +103,10 @@ class TPortionInfoWithBlobs { return PortionInfo.BlobsBytes(); } - static TPortionInfoWithBlobs BuildByBlobs(std::vector>>& chunksByBlobs, std::shared_ptr batch, const ui64 granule, const TSnapshot& snapshot, - const std::shared_ptr& bStorageOperator); + static TPortionInfoWithBlobs BuildByBlobs(std::vector&& chunks, + std::shared_ptr batch, const ui64 granule, const TSnapshot& snapshot, const std::shared_ptr& operators); - std::optional ChangeSaver(ISnapshotSchema::TPtr currentSchema, const TSaverContext& saverContext) const; + std::optional ChangeSaver(ISnapshotSchema::TPtr currentSchema, const TSaverContext& saverContext, const std::shared_ptr& bOperator) const; const TString& GetBlobByRangeVerified(const ui32 columnId, const ui32 chunkId) const { for (auto&& b : Blobs) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.cpp index a4ecce5de374..fd9add65bfc3 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.cpp @@ -4,15 +4,15 @@ namespace NKikimr::NOlap::NPlainReader { void TBlobsFetcherTask::DoOnDataReady(const std::shared_ptr& /*resourcesGuard*/) { - Source->MutableStageData().AddBlobs(ExtractBlobsData()); + Source->MutableStageData().AddBlobs(Source->DecodeBlobAddresses(ExtractBlobsData())); AFL_VERIFY(Step->GetNextStep()); auto task = std::make_shared(Source, Step->GetNextStep(), Context->GetCommonContext()->GetScanActorId()); NConveyor::TScanServiceOperator::SendTaskToExecute(task); } -bool TBlobsFetcherTask::DoOnError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { +bool TBlobsFetcherTask::DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_SCAN)("error_on_blob_reading", range.ToString())("scan_actor_id", Context->GetCommonContext()->GetScanActorId()) - ("status", status.GetErrorMessage())("status_code", status.GetStatus()); + ("status", status.GetErrorMessage())("status_code", status.GetStatus())("storage_id", storageId); NActors::TActorContext::AsActorContext().Send(Context->GetCommonContext()->GetScanActorId(), std::make_unique(TConclusionStatus::Fail("cannot read blob range " + range.ToString()))); return false; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.h index 3b5ceca5200e..bfeda1c6d4c9 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor.h @@ -16,7 +16,7 @@ class TBlobsFetcherTask: public NBlobOperations::NRead::ITask { const std::shared_ptr Context; virtual void DoOnDataReady(const std::shared_ptr& resourcesGuard) override; - virtual bool DoOnError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) override; + virtual bool DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) override; public: TBlobsFetcherTask(const std::vector>& readActions, const std::shared_ptr& sourcePtr, const std::shared_ptr& step, const std::shared_ptr& context, const TString& taskCustomer, const TString& externalTaskId) diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/fetched_data.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/fetched_data.h index e31f51bd0dac..04caa149a099 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/fetched_data.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/fetched_data.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -11,7 +12,7 @@ namespace NKikimr::NOlap { class TFetchedData { protected: - using TBlobs = THashMap; + using TBlobs = THashMap; YDB_ACCESSOR_DEF(TBlobs, Blobs); YDB_READONLY_DEF(std::shared_ptr, Table); YDB_READONLY_DEF(std::shared_ptr, Filter); @@ -33,8 +34,8 @@ class TFetchedData { return UseFilter ? nullptr : Filter; } - TString ExtractBlob(const TBlobRange& bRange) { - auto it = Blobs.find(bRange); + TString ExtractBlob(const TChunkAddress& address) { + auto it = Blobs.find(address); AFL_VERIFY(it != Blobs.end()); AFL_VERIFY(it->second.IsBlob()); auto result = it->second.GetData(); @@ -42,13 +43,13 @@ class TFetchedData { return result; } - void AddBlobs(THashMap&& blobs) { - for (auto&& i : blobs) { + void AddBlobs(THashMap&& blobData) { + for (auto&& i : blobData) { AFL_VERIFY(Blobs.emplace(i.first, std::move(i.second)).second); } } - void AddNulls(THashMap&& blobs) { + void AddNulls(THashMap&& blobs) { for (auto&& i : blobs) { AFL_VERIFY(Blobs.emplace(i.first, i.second).second); } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/source.cpp index 3ba21e93aef1..f2a5f44042b7 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/source.cpp @@ -45,7 +45,7 @@ void IDataSource::SetIsReady() { } void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, - const std::shared_ptr& readingAction, THashMap& nullBlocks, + TBlobsAction& blobsAction, THashMap& nullBlocks, const std::shared_ptr& filter) { const NArrow::TColumnFilter& cFilter = filter ? *filter : NArrow::TColumnFilter::BuildAllowFilter(); ui32 fetchedChunks = 0; @@ -60,17 +60,19 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, for (auto&& c : columnChunks) { Y_ABORT_UNLESS(!itFinished); if (!itFilter.IsBatchForSkip(c->GetMeta().GetNumRowsVerified())) { - readingAction->AddRange(c->BlobRange); + auto reading = blobsAction.GetReading(Schema->GetIndexInfo().GetColumnStorageId(c->GetColumnId(), Portion->GetMeta().GetTierName())); + reading->SetIsBackgroundProcess(false); + reading->AddRange(c->BlobRange); ++fetchedChunks; } else { - nullBlocks.emplace(c->BlobRange, c->GetMeta().GetNumRowsVerified()); + nullBlocks.emplace(c->GetAddress(), c->GetMeta().GetNumRowsVerified()); ++nullChunks; } itFinished = !itFilter.Next(c->GetMeta().GetNumRowsVerified()); } AFL_VERIFY(itFinished)("filter", itFilter.DebugString())("count", Portion->NumRows(i)); } - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks)("reading_action", readingAction->GetStorageId())("columns", columnIds.size()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks)("reading_actions", blobsAction.GetStorageIds())("columns", columnIds.size()); } bool TPortionDataSource::DoStartFetchingColumns(const std::shared_ptr& sourcePtr, const std::shared_ptr& step, const std::shared_ptr& columns) { @@ -80,20 +82,19 @@ bool TPortionDataSource::DoStartFetchingColumns(const std::shared_ptrGetColumnIds(); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", step->GetName())("fetching_info", step->DebugString()); - auto readAction = Portion->GetBlobsStorage()->StartReadingAction("CS::READ::" + step->GetName()); - readAction->SetIsBackgroundProcess(false); + TBlobsAction action(GetContext()->GetCommonContext()->GetStoragesManager(), "CS::READ::" + step->GetName()); { - THashMap nullBlocks; - NeedFetchColumns(columnIds, readAction, nullBlocks, StageData->GetAppliedFilter()); + THashMap nullBlocks; + NeedFetchColumns(columnIds, action, nullBlocks, StageData->GetAppliedFilter()); StageData->AddNulls(std::move(nullBlocks)); } - if (!readAction->GetExpectedBlobsSize()) { + auto readActions = action.GetReadingActions(); + if (!readActions.size()) { return false; } - std::vector> actions = {readAction}; - auto constructor = std::make_shared(actions, sourcePtr, step, GetContext(), "CS::READ::" + step->GetName(), ""); + auto constructor = std::make_shared(readActions, sourcePtr, step, GetContext(), "CS::READ::" + step->GetName(), ""); NActors::TActivationContext::AsActorContext().Register(new NOlap::NBlobOperations::NRead::TActor(constructor)); return true; } @@ -103,8 +104,7 @@ bool TPortionDataSource::DoStartFetchingIndexes(const std::shared_ptrGetIndexesCount()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", step->GetName())("fetching_info", step->DebugString()); - auto readAction = Portion->GetBlobsStorage()->StartReadingAction("CS::READ::" + step->GetName()); - readAction->SetIsBackgroundProcess(false); + TBlobsAction action(GetContext()->GetCommonContext()->GetStoragesManager(), "CS::READ::" + step->GetName()); { std::set indexIds; for (auto&& i : Portion->GetIndexes()) { @@ -112,20 +112,21 @@ bool TPortionDataSource::DoStartFetchingIndexes(const std::shared_ptrGetIndexInfo().GetIndexStorageId(i.GetIndexId())); + readAction->SetIsBackgroundProcess(false); readAction->AddRange(i.GetBlobRange()); } if (indexes->GetIndexIdsSet().size() != indexIds.size()) { return false; } } - - if (!readAction->GetExpectedBlobsSize()) { + auto readingActions = action.GetReadingActions(); + if (!readingActions.size()) { NYDBTest::TControllers::GetColumnShardController()->OnIndexSelectProcessed({}); return false; } - std::vector> actions = {readAction}; - auto constructor = std::make_shared(actions, sourcePtr, step, GetContext(), "CS::READ::" + step->GetName(), ""); + auto constructor = std::make_shared(readingActions, sourcePtr, step, GetContext(), "CS::READ::" + step->GetName(), ""); NActors::TActivationContext::AsActorContext().Register(new NOlap::NBlobOperations::NRead::TActor(constructor)); return true; } @@ -144,7 +145,7 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in if (!indexIds.contains(i->GetIndexId())) { continue; } - indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetBlobRange())); + indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetAddress())); } for (auto&& i : indexIds) { if (!indexBlobs.contains(i)) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/source.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/source.h index 3cce8e16eef7..746b081eb1f5 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/source.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/source.h @@ -2,12 +2,13 @@ #include "context.h" #include "columns_set.h" #include "fetched_data.h" -#include -#include #include +#include +#include +#include #include #include -#include +#include #include namespace NKikimr::NOlap { @@ -54,6 +55,8 @@ class IDataSource { virtual void DoAbort() = 0; virtual void DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexMeta) = 0; public: + virtual THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobsOriginal) const = 0; + const NArrow::TReplaceKey& GetStartReplaceKey() const { return StartReplaceKey; } @@ -157,8 +160,7 @@ class IDataSource { IDataSource(const ui32 sourceIdx, const std::shared_ptr& context, const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish, - const TSnapshot& recordSnapshotMax, const std::optional recordsCount - ) + const TSnapshot& recordSnapshotMax, const std::optional recordsCount) : SourceIdx(sourceIdx) , Start(context->GetReadMetadata()->BuildSortedPosition(start)) , Finish(context->GetReadMetadata()->BuildSortedPosition(finish)) @@ -184,9 +186,10 @@ class TPortionDataSource: public IDataSource { private: using TBase = IDataSource; std::shared_ptr Portion; + std::shared_ptr Schema; void NeedFetchColumns(const std::set& columnIds, - const std::shared_ptr& readingAction, THashMap& nullBlocks, + TBlobsAction& blobsAction, THashMap& nullBlocks, const std::shared_ptr& filter); virtual void DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) override; @@ -205,6 +208,10 @@ class TPortionDataSource: public IDataSource { virtual void DoAbort() override; public: + virtual THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobsOriginal) const override { + return Portion->DecodeBlobAddresses(std::move(blobsOriginal), Schema->GetIndexInfo()); + } + virtual ui64 GetRawBytes(const std::set& columnIds) const override { return Portion->GetRawBytes(columnIds); } @@ -225,6 +232,7 @@ class TPortionDataSource: public IDataSource { const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish) : TBase(sourceIdx, context, start, finish, portion->RecordSnapshotMax(), portion->GetRecordsCount()) , Portion(portion) + , Schema(GetContext()->GetReadMetadata()->GetLoadSchema(Portion->GetMinSnapshot())) { } }; @@ -255,6 +263,16 @@ class TCommittedDataSource: public IDataSource { return result; } public: + virtual THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobsOriginal) const override { + THashMap result; + for (auto&& i : blobsOriginal) { + for (auto&& b : i.second) { + result.emplace(TChunkAddress(1, 1), std::move(b.second)); + } + } + return result; + } + virtual ui64 GetRawBytes(const std::set& /*columnIds*/) const override { return CommittedBlob.GetBlobRange().Size; } diff --git a/ydb/core/tx/columnshard/engines/scheme/column_features.cpp b/ydb/core/tx/columnshard/engines/scheme/column_features.cpp index 72d5059c0053..fb9dcfb5c2dd 100644 --- a/ydb/core/tx/columnshard/engines/scheme/column_features.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/column_features.cpp @@ -26,9 +26,12 @@ void TColumnFeatures::InitLoader(const TIndexInfo& info) { Loader = std::make_shared(GetLoadTransformer(), Serializer, schema, ColumnId); } -std::optional TColumnFeatures::BuildFromProto(const NKikimrSchemeOp::TOlapColumnDescription& columnInfo, const TIndexInfo& indexInfo) { +std::optional TColumnFeatures::BuildFromProto(const NKikimrSchemeOp::TOlapColumnDescription& columnInfo, const TIndexInfo& indexInfo, + const std::shared_ptr& operators) +{ const ui32 columnId = columnInfo.GetId(); - TColumnFeatures result(columnId); + auto bOperator = operators->GetOperatorVerified(columnInfo.GetStorageId() ? columnInfo.GetStorageId() : IStoragesManager::DefaultStorageId); + TColumnFeatures result(columnId, bOperator); if (columnInfo.HasSerializer()) { AFL_VERIFY(result.Serializer.DeserializeFromProto(columnInfo.GetSerializer())); } else if (columnInfo.HasCompression()) { @@ -43,14 +46,17 @@ std::optional TColumnFeatures::BuildFromProto(c return result; } -NKikimr::NOlap::TColumnFeatures TColumnFeatures::BuildFromIndexInfo(const ui32 columnId, const TIndexInfo& indexInfo) { - TColumnFeatures result(columnId); +NKikimr::NOlap::TColumnFeatures TColumnFeatures::BuildFromIndexInfo(const ui32 columnId, const TIndexInfo& indexInfo, + const std::shared_ptr& blobsOperator) +{ + TColumnFeatures result(columnId, blobsOperator); result.InitLoader(indexInfo); return result; } -TColumnFeatures::TColumnFeatures(const ui32 columnId) +TColumnFeatures::TColumnFeatures(const ui32 columnId, const std::shared_ptr& blobsOperator) : ColumnId(columnId) + , Operator(blobsOperator) , Serializer(NArrow::NSerialization::TSerializerContainer::GetDefaultSerializer()) { diff --git a/ydb/core/tx/columnshard/engines/scheme/column_features.h b/ydb/core/tx/columnshard/engines/scheme/column_features.h index 7e99af80eb72..df5b0d17cc8c 100644 --- a/ydb/core/tx/columnshard/engines/scheme/column_features.h +++ b/ydb/core/tx/columnshard/engines/scheme/column_features.h @@ -17,9 +17,8 @@ class TSaverContext { YDB_READONLY_DEF(std::shared_ptr, StorageOperator); YDB_READONLY_DEF(std::shared_ptr, StoragesManager); public: - TSaverContext(const std::shared_ptr& storageOperator, const std::shared_ptr& storagesManager) - : StorageOperator(storageOperator) - , StoragesManager(storagesManager) { + TSaverContext(const std::shared_ptr& storagesManager) + : StoragesManager(storagesManager) { } @@ -135,14 +134,14 @@ struct TIndexInfo; class TColumnFeatures { private: ui32 ColumnId; + YDB_READONLY_DEF(std::shared_ptr, Operator); YDB_READONLY_DEF(NArrow::NSerialization::TSerializerContainer, Serializer); std::optional DictionaryEncoding; std::shared_ptr Loader; - NArrow::NTransformation::ITransformer::TPtr GetLoadTransformer() const; void InitLoader(const TIndexInfo& info); - TColumnFeatures(const ui32 columnId); + TColumnFeatures(const ui32 columnId, const std::shared_ptr& blobsOperator); public: TString DebugString() const { @@ -154,8 +153,8 @@ class TColumnFeatures { } NArrow::NTransformation::ITransformer::TPtr GetSaveTransformer() const; - static std::optional BuildFromProto(const NKikimrSchemeOp::TOlapColumnDescription& columnInfo, const TIndexInfo& indexInfo); - static TColumnFeatures BuildFromIndexInfo(const ui32 columnId, const TIndexInfo& indexInfo); + static std::optional BuildFromProto(const NKikimrSchemeOp::TOlapColumnDescription& columnInfo, const TIndexInfo& indexInfo, const std::shared_ptr& operators); + static TColumnFeatures BuildFromIndexInfo(const ui32 columnId, const TIndexInfo& indexInfo, const std::shared_ptr& blobsOperator); const std::shared_ptr& GetLoader() const { AFL_VERIFY(Loader); diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index e60785485be2..ff7d119b991a 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -231,7 +231,7 @@ std::shared_ptr TIndexInfo::ArrowColumnFieldOptional(const ui32 co } } -void TIndexInfo::SetAllKeys() { +void TIndexInfo::SetAllKeys(const std::shared_ptr& operators) { /// @note Setting replace and sorting key to PK we are able to: /// * apply REPLACE by MergeSort /// * apply PK predicate before REPLACE @@ -256,7 +256,7 @@ void TIndexInfo::SetAllKeys() { MinMaxIdxColumnsIds.insert(GetPKFirstColumnId()); if (!Schema) { AFL_VERIFY(!SchemaWithSpecials); - InitializeCaches(); + InitializeCaches(operators); } } @@ -364,7 +364,7 @@ std::shared_ptr TIndexInfo::GetColumnSchema(const ui32 columnId) return GetColumnsSchema({columnId}); } -bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema) { +bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema, const std::shared_ptr& operators) { if (schema.GetEngine() != NKikimrSchemeOp::COLUMN_ENGINE_REPLACING_TIMESERIES) { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "cannot_parse_index_info")("reason", "incorrect_engine_in_schema"); return false; @@ -384,9 +384,9 @@ bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& Columns[id] = NTable::TColumn(name, id, typeInfoMod.TypeInfo, typeInfoMod.TypeMod, notNull); ColumnNames[name] = id; } - InitializeCaches(); + InitializeCaches(operators); for (const auto& col : schema.GetColumns()) { - std::optional cFeatures = TColumnFeatures::BuildFromProto(col, *this); + std::optional cFeatures = TColumnFeatures::BuildFromProto(col, *this, operators); if (!cFeatures) { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "cannot_parse_column_feature"); return false; @@ -450,9 +450,9 @@ std::vector GetColumns(const NTable::TScheme::TTableSchema& table return out; } -std::optional TIndexInfo::BuildFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema) { +std::optional TIndexInfo::BuildFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema, const std::shared_ptr& operators) { TIndexInfo result(""); - if (!result.DeserializeFromProto(schema)) { + if (!result.DeserializeFromProto(schema, operators)) { return std::nullopt; } return result; @@ -462,17 +462,17 @@ std::shared_ptr TIndexInfo::SpecialColumnField(const ui32 columnId return ArrowSchemaSnapshot()->GetFieldByName(GetColumnName(columnId, true)); } -void TIndexInfo::InitializeCaches() { +void TIndexInfo::InitializeCaches(const std::shared_ptr& operators) { BuildArrowSchema(); BuildSchemaWithSpecials(); for (auto&& c : Columns) { AFL_VERIFY(ArrowColumnByColumnIdCache.emplace(c.first, GetColumnFieldVerified(c.first)).second); - AFL_VERIFY(ColumnFeatures.emplace(c.first, TColumnFeatures::BuildFromIndexInfo(c.first, *this)).second); + AFL_VERIFY(ColumnFeatures.emplace(c.first, TColumnFeatures::BuildFromIndexInfo(c.first, *this, operators->GetDefaultOperator())).second); } for (auto&& cId : GetSpecialColumnIds()) { AFL_VERIFY(ArrowColumnByColumnIdCache.emplace(cId, GetColumnFieldVerified(cId)).second); - AFL_VERIFY(ColumnFeatures.emplace(cId, TColumnFeatures::BuildFromIndexInfo(cId, *this)).second); + AFL_VERIFY(ColumnFeatures.emplace(cId, TColumnFeatures::BuildFromIndexInfo(cId, *this, operators->GetDefaultOperator())).second); } } diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index 7409e33bd910..52d26f9882c0 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -38,11 +38,11 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { THashMap> ArrowColumnByColumnIdCache; THashMap Indexes; TIndexInfo(const TString& name); - bool DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema); + bool DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema, const std::shared_ptr& operators); TColumnFeatures& GetOrCreateColumnFeatures(const ui32 columnId) const; void BuildSchemaWithSpecials(); void BuildArrowSchema(); - void InitializeCaches(); + void InitializeCaches(const std::shared_ptr& operators); public: static constexpr const char* SPEC_COL_PLAN_STEP = NOlap::NPortion::TSpecialColumns::SPEC_COL_PLAN_STEP; static constexpr const char* SPEC_COL_TX_ID = NOlap::NPortion::TSpecialColumns::SPEC_COL_TX_ID; @@ -53,6 +53,30 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { return Indexes; } + const TString& GetIndexStorageId(const ui32 indexId) const { + auto it = Indexes.find(indexId); + AFL_VERIFY(it != Indexes.end()); + return it->second->GetStorageId(); + } + + const TString& GetColumnStorageId(const ui32 columnId, const TString& specialTier) const { + if (specialTier) { + return specialTier; + } else { + auto it = ColumnFeatures.find(columnId); + AFL_VERIFY(it != ColumnFeatures.end()); + return it->second.GetOperator()->GetStorageId(); + } + } + + const TString& GetEntityStorageId(const ui32 entityId, const TString& specialTier) const { + auto it = Indexes.find(entityId); + if (it != Indexes.end()) { + return it->second->GetStorageId(); + } + return GetColumnStorageId(entityId, specialTier); + } + enum class ESpecialColumn : ui32 { PLAN_STEP = NOlap::NPortion::TSpecialColumns::SPEC_COL_PLAN_STEP_INDEX, TX_ID = NOlap::NPortion::TSpecialColumns::SPEC_COL_TX_ID_INDEX @@ -102,7 +126,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { return result; } - static std::optional BuildFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema); + static std::optional BuildFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema, const std::shared_ptr& operators); static const std::vector& SnapshotColumnNames() { static std::vector result = {SPEC_COL_PLAN_STEP, SPEC_COL_TX_ID}; @@ -157,6 +181,13 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { /// Returns names of columns defined by the specific ids. std::vector GetColumnNames(const std::vector& ids) const; std::vector GetColumnIds() const; + std::vector GetEntityIds() const { + auto result = GetColumnIds(); + for (auto&& i : Indexes) { + result.emplace_back(i.first); + } + return result; + } /// Returns info of columns defined by specific ids. std::vector GetColumns(const std::vector& ids) const; @@ -179,7 +210,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema { const std::shared_ptr& GetPrimaryKey() const { return PrimaryKey; } /// Initializes sorting, replace, index and extended keys. - void SetAllKeys(); + void SetAllKeys(const std::shared_ptr& operators); void CheckTtlColumn(const TString& ttlColumn) const { Y_ABORT_UNLESS(!ttlColumn.empty()); diff --git a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.cpp b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.cpp index cf87cf941d8f..38dd3fac3a1e 100644 --- a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.cpp @@ -1,4 +1,5 @@ #include "meta.h" +#include #include #include #include @@ -60,4 +61,13 @@ NKikimr::TConclusionStatus TIndexByColumns::CheckSameColumnsForModification(cons return TConclusionStatus::Success(); } +bool IIndexMeta::DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto) { + IndexId = proto.GetId(); + AFL_VERIFY(IndexId); + IndexName = proto.GetName(); + AFL_VERIFY(IndexName); + StorageId = proto.GetStorageId() ? proto.GetStorageId() : IStoragesManager::DefaultStorageId; + return DoDeserializeFromProto(proto); +} + } // namespace NKikimr::NOlap::NIndexes \ No newline at end of file diff --git a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.h b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.h index 9966d4ee31ef..19e2e3ad5fb9 100644 --- a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.h +++ b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/meta.h @@ -28,6 +28,7 @@ class IIndexMeta { private: YDB_READONLY_DEF(TString, IndexName); YDB_READONLY(ui32, IndexId, 0); + YDB_READONLY(TString, StorageId, IStoragesManager::DefaultStorageId); protected: virtual std::shared_ptr DoBuildIndex(const ui32 indexId, std::map>>& data, const TIndexInfo& indexInfo) const = 0; virtual void DoFillIndexCheckers(const std::shared_ptr& info, const NSchemeShard::TOlapSchema& schema) const = 0; @@ -67,19 +68,16 @@ class IIndexMeta { return DoFillIndexCheckers(info, schema); } - bool DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto) { - IndexId = proto.GetId(); - AFL_VERIFY(IndexId); - IndexName = proto.GetName(); - AFL_VERIFY(IndexName); - return DoDeserializeFromProto(proto); - } + bool DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& proto); void SerializeToProto(NKikimrSchemeOp::TOlapIndexDescription& proto) const { AFL_VERIFY(IndexId); proto.SetId(IndexId); AFL_VERIFY(IndexName); proto.SetName(IndexName); + if (StorageId) { + proto.SetStorageId(StorageId); + } return DoSerializeToProto(proto); } diff --git a/ydb/core/tx/columnshard/engines/storage/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule.cpp index aac2d2bdd1b7..402298be0d84 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule.cpp @@ -61,10 +61,6 @@ void TGranuleMeta::AddColumnRecord(const TIndexInfo& indexInfo, const TPortionIn AFL_VERIFY(it->second->IsEqualWithSnapshots(portion))("self", it->second->DebugString())("item", portion.DebugString()); } it->second->AddRecord(indexInfo, rec, portionMeta); - - if (portionMeta) { - it->second->InitOperator(Owner->GetStoragesManager()->InitializePortionOperator(*it->second), false); - } } void TGranuleMeta::OnAfterChangePortion(const std::shared_ptr portionAfter, NStorageOptimizer::IOptimizerPlanner::TModificationGuard* modificationGuard) { diff --git a/ydb/core/tx/columnshard/engines/storage/granule.h b/ydb/core/tx/columnshard/engines/storage/granule.h index 7c806630d6c7..b968364be48f 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule.h @@ -273,6 +273,14 @@ class TGranuleMeta: TNonCopyable { return Portions; } + std::vector> GetPortionsVector() const { + std::vector> result; + for (auto&& i : Portions) { + result.emplace_back(i.second); + } + return result; + } + ui64 GetPathId() const { return PathId; } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp index 0b9a05a0a659..e15eb551ca6d 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp @@ -30,7 +30,7 @@ std::shared_ptr TBlobsWithSizeLimit::Build } if (currentSum > SizeLimitToMerge || PortionsCount > CountLimitToMerge) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("portions", portions.size())("current_sum", currentSum); - TSaverContext saverContext(StoragesManager->GetOperator(tierName.value_or(IStoragesManager::DefaultStorageId)), StoragesManager); + TSaverContext saverContext(StoragesManager); return std::make_shared(limits.GetSplitSettings(), granule, portions, saverContext); } else { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("skip", "not_enough_data"); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp index 15965ecf18c4..d9431021b4d3 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp @@ -65,7 +65,7 @@ std::shared_ptr TIntervalsOptimizerPlanner::DoGetOptimizat } AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule")("features", features.DebugJson().GetStringRobust())("count", features.GetPortionsCount()); - TSaverContext saverContext(StoragesManager->GetOperator(tierName.value_or(IStoragesManager::DefaultStorageId)), StoragesManager); + TSaverContext saverContext(StoragesManager); return std::make_shared(limits.GetSplitSettings(), granule, portions, saverContext); } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h index 3d45347dbefc..8d9987739782 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h @@ -762,7 +762,7 @@ class TPortionsBucket: public TMoveOnly { } AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("stop_instant", stopInstant)("size", size)("next", NextBorder ? NextBorder->DebugString() : "") ("count", portions.size())("info", Others.DebugString())("event", "start_optimization")("stop_point", stopPoint ? stopPoint->DebugString() : ""); - TSaverContext saverContext(storagesManager->GetOperator(IStoragesManager::DefaultStorageId), storagesManager); + TSaverContext saverContext(storagesManager); auto result = std::make_shared(limits.GetSplitSettings(), granule, portions, saverContext); if (MainPortion) { NIndexedReader::TSortableBatchPosition pos(MainPortion->IndexKeyStart().ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h index 5c445261300a..67d7e1ae8402 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h @@ -450,7 +450,7 @@ class TLevel { Y_ABORT_UNLESS(position); positions.emplace_back(*position); } - TSaverContext saverContext(StoragesManager->GetOperator(IStoragesManager::DefaultStorageId), StoragesManager); + TSaverContext saverContext(StoragesManager); auto result = std::make_shared(CompactionLimits.GetSplitSettings(), granule, portions, saverContext); for (auto&& i : positions) { result->AddCheckPoint(i); diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index c493c8aeb0cf..6acca4223c56 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -12,7 +12,7 @@ #include #include #include - +#include namespace NKikimr { @@ -169,21 +169,21 @@ class TTestDbWrapper : public IDbWrapper { THashMap Indices; }; -static const std::vector> testColumns = { +static const std::vector testColumns = { // PK - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), // - {"message", TTypeInfo(NTypeIds::Utf8) } + NArrow::NTest::TTestColumn("message", TTypeInfo(NTypeIds::Utf8) ) }; -static const std::vector> testKey = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) } +static const std::vector testKey = { + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ) }; template @@ -249,11 +249,12 @@ TString MakeTestBlob(i64 start = 0, i64 end = 100) { return NArrow::SerializeBatchNoCompression(batch); } -void AddIdsToBlobs(std::vector& portions, THashMap& blobs, ui32& step) { +void AddIdsToBlobs(std::vector& portions, NBlobOperations::NRead::TCompositeReadBlobs& blobs, ui32& step) { for (auto& portion : portions) { for (auto& rec : portion.GetPortionInfo().Records) { rec.BlobRange.BlobId = MakeUnifiedBlobId(++step, portion.GetBlobFullSizeVerified(rec.ColumnId, rec.Chunk)); - blobs[rec.BlobRange] = portion.GetBlobByRangeVerified(rec.ColumnId, rec.Chunk); + TString data = portion.GetBlobByRangeVerified(rec.ColumnId, rec.Chunk); + blobs.Add(IStoragesManager::DefaultStorageId, rec.BlobRange, std::move(data)); } } } @@ -267,7 +268,7 @@ TCompactionLimits TestLimits() { } bool Insert(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, - std::vector&& dataToIndex, THashMap& blobs, ui32& step) { + std::vector&& dataToIndex, NBlobOperations::NRead::TCompositeReadBlobs& blobs, ui32& step) { for (ui32 i = 0; i < dataToIndex.size(); ++i) { // Commited data always has nonzero planstep (for WriteLoadRead tests) @@ -278,8 +279,8 @@ bool Insert(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, return false; } - changes->Blobs.insert(blobs.begin(), blobs.end()); - blobs.clear(); + changes->Blobs = std::move(blobs); + blobs.Clear(); changes->StartEmergency(); NOlap::TConstructionContext context(engine.GetVersionedIndex(), NColumnShard::TIndexationCounters("Indexation")); @@ -311,12 +312,12 @@ struct TExpected { ui32 NewGranules; }; -bool Compact(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, THashMap&& blobs, ui32& step, +bool Compact(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, NBlobOperations::NRead::TCompositeReadBlobs&& blobs, ui32& step, const TExpected& /*expected*/, THashMap* blobsPool = nullptr) { std::shared_ptr changes = dynamic_pointer_cast(engine.StartCompaction(TestLimits(), EmptyDataLocksManager)); UNIT_ASSERT(changes); // UNIT_ASSERT_VALUES_EQUAL(changes->SwitchedPortions.size(), expected.SrcPortions); - changes->SetBlobs(std::move(blobs)); + changes->Blobs = std::move(blobs); changes->StartEmergency(); NOlap::TConstructionContext context(engine.GetVersionedIndex(), NColumnShard::TIndexationCounters("Compaction")); Y_ABORT_UNLESS(changes->ConstructBlobs(context).Ok()); @@ -400,34 +401,22 @@ std::shared_ptr MakeStrPredicate(const std::string& key, NArrow::EOp } // namespace -class TTestStoragesManager: public NOlap::IStoragesManager { -private: - using TBase = NOlap::IStoragesManager; - TIntrusivePtr TabletInfo = new TTabletStorageInfo(); - std::shared_ptr SharedBlobsManager = std::make_shared(NOlap::TTabletId(0)); -protected: - virtual bool DoLoadIdempotency(NTable::TDatabase& /*database*/) override { - return true; - } - - virtual std::shared_ptr DoBuildOperator(const TString& storageId) override { - if (storageId == TBase::DefaultStorageId) { - return std::make_shared(storageId, NActors::TActorId(), TabletInfo, - 1, SharedBlobsManager->GetStorageManagerGuarantee(TBase::DefaultStorageId)); - } else - return nullptr; - } -public: - virtual const std::shared_ptr& GetSharedBlobsManager() const override { - return SharedBlobsManager; +std::shared_ptr InitializeStorageManager() { + static auto result = std::make_shared(); + static TMutex mutex; + static bool initialized = false; + TGuard g(mutex); + if (!initialized) { + result->Initialize(); } -}; + return result; +} -std::shared_ptr CommonStoragesManager = std::make_shared(); +std::shared_ptr CommonStoragesManager = InitializeStorageManager(); Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { - void WriteLoadRead(const std::vector>& ydbSchema, - const std::vector>& key) { + void WriteLoadRead(const std::vector& ydbSchema, + const std::vector& key) { TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -457,20 +446,24 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // write ui32 step = 1000; - THashMap blobs; - blobs[blobRanges[0]] = testBlob; - blobs[blobRanges[1]] = testBlob; - Insert(engine, db, TSnapshot(1, 2), std::move(dataToIndex), blobs, step); + { + NBlobOperations::NRead::TCompositeReadBlobs blobs; + TString str1 = testBlob; + blobs.Add(IStoragesManager::DefaultStorageId, blobRanges[0], std::move(str1)); + str1 = testBlob; + blobs.Add(IStoragesManager::DefaultStorageId, blobRanges[1], std::move(str1)); + Insert(engine, db, TSnapshot(1, 2), std::move(dataToIndex), blobs, step); + } // selects auto lastSchema = engine.GetVersionedIndex().GetLastSchema(); UNIT_ASSERT_EQUAL(lastSchema->GetSnapshot(), indexSnaphot); const TIndexInfo& indexInfo = lastSchema->GetIndexInfo(); - THashSet oneColumnId = { indexInfo.GetColumnId(testColumns[0].first) }; + THashSet oneColumnId = { indexInfo.GetColumnId(testColumns[0].GetName()) }; THashSet columnIds; - for (auto& [column, typeId] : testColumns) { - columnIds.insert(indexInfo.GetColumnId(column)); + for (auto& c : testColumns) { + columnIds.insert(indexInfo.GetColumnId(c.GetName())); } { // select from snap before insert @@ -508,18 +501,18 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexWriteLoadReadStrPK) { - std::vector> key = { - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, - {"timestamp", TTypeInfo(NTypeIds::Timestamp) } + std::vector key = { + NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ) }; WriteLoadRead(testColumns, key); } - void ReadWithPredicates(const std::vector>& ydbSchema, - const std::vector>& key) { + void ReadWithPredicates(const std::vector& ydbSchema, + const std::vector& key) { TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -540,8 +533,9 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { for (ui64 txId = 1; txId <= 20; ++txId, rowPos += numRows) { TString testBlob = MakeTestBlob(rowPos, rowPos + numRows); auto blobRange = MakeBlobRange(++step, testBlob.size()); - THashMap blobs; - blobs[blobRange] = testBlob; + NBlobOperations::NRead::TCompositeReadBlobs blobs; + TString str1 = testBlob; + blobs.Add(IStoragesManager::DefaultStorageId, blobRange, std::move(str1)); // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] std::vector dataToIndex; @@ -565,7 +559,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { planStep = 3; const TIndexInfo& indexInfo = engine.GetVersionedIndex().GetLastSchema()->GetIndexInfo(); - THashSet oneColumnId = { indexInfo.GetColumnId(key[0].first) }; + THashSet oneColumnId = { indexInfo.GetColumnId(key[0].GetName()) }; { // full scan ui64 txId = 1; @@ -578,7 +572,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { { ui64 txId = 1; std::shared_ptr gt10k = MakePredicate(10000, NArrow::EOperation::Greater); - if (key[0].second == TTypeInfo(NTypeIds::Utf8)) { + if (key[0].GetType() == TTypeInfo(NTypeIds::Utf8)) { gt10k = MakeStrPredicate("10000", NArrow::EOperation::Greater); } NOlap::TPKRangesFilter pkFilter(false); @@ -590,7 +584,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { { ui64 txId = 1; std::shared_ptr lt10k = MakePredicate(8999, NArrow::EOperation::Less); // TODO: better border checks - if (key[0].second == TTypeInfo(NTypeIds::Utf8)) { + if (key[0].GetType() == TTypeInfo(NTypeIds::Utf8)) { lt10k = MakeStrPredicate("08999", NArrow::EOperation::Less); } NOlap::TPKRangesFilter pkFilter(false); @@ -605,11 +599,11 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexReadWithPredicatesStrPK) { - std::vector> key = { - {"resource_type", TTypeInfo(NTypeIds::Utf8) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, - {"timestamp", TTypeInfo(NTypeIds::Timestamp) } + std::vector key = { + NArrow::NTest::TTestColumn("resource_type", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ) }; ReadWithPredicates(testColumns, key); @@ -634,12 +628,12 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 numRows = 1000; ui64 rowPos = 0; - THashMap blobsAll; + NBlobOperations::NRead::TCompositeReadBlobs blobsAll; for (ui64 txId = 1; txId <= 100; ++txId, rowPos += numRows) { TString testBlob = MakeTestBlob(rowPos, rowPos + numRows); auto blobRange = MakeBlobRange(++step, testBlob.size()); - THashMap blobs; - blobs[blobRange] = testBlob; + NBlobOperations::NRead::TCompositeReadBlobs blobs; + blobs.Add(IStoragesManager::DefaultStorageId, blobRange, std::move(testBlob)); // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] std::vector dataToIndex; @@ -647,9 +641,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { TInsertedData(txId, pathId, "", blobRange.BlobId, TLocalHelper::GetMetaProto(), 0, {})); bool ok = Insert(engine, db, TSnapshot(planStep, txId), std::move(dataToIndex), blobs, step); - for (auto&& i : blobs) { - blobsAll[i.first] = i.second; - } + blobsAll.Merge(std::move(blobs)); UNIT_ASSERT(ok); } @@ -672,8 +664,8 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { for (ui64 txId = 1; txId <= 2; ++txId, rowPos += numRows) { TString testBlob = MakeTestBlob(rowPos, rowPos + numRows); auto blobRange = MakeBlobRange(++step, testBlob.size()); - THashMap blobs; - blobs[blobRange] = testBlob; + NBlobOperations::NRead::TCompositeReadBlobs blobs; + blobs.Add(IStoragesManager::DefaultStorageId, blobRange, std::move(testBlob)); // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] std::vector dataToIndex; @@ -713,8 +705,9 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { for (ui64 txId = 1; txId <= 20; ++txId, rowPos += numRows) { TString testBlob = MakeTestBlob(rowPos, rowPos + numRows); auto blobRange = MakeBlobRange(++step, testBlob.size()); - THashMap blobs; - blobs[blobRange] = testBlob; + NBlobOperations::NRead::TCompositeReadBlobs blobs; + TString str1 = testBlob; + blobs.Add(IStoragesManager::DefaultStorageId, blobRange, std::move(str1)); // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] std::vector dataToIndex; @@ -735,7 +728,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { planStep = 3; const TIndexInfo& indexInfo = engine.GetVersionedIndex().GetLastSchema()->GetIndexInfo(); - THashSet oneColumnId = {indexInfo.GetColumnId(testColumns[0].first)}; + THashSet oneColumnId = {indexInfo.GetColumnId(testColumns[0].GetName())}; { // full scan ui64 txId = 1; @@ -776,7 +769,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { engine.Load(db); const TIndexInfo& indexInfo = engine.GetVersionedIndex().GetLastSchema()->GetIndexInfo(); - THashSet oneColumnId = {indexInfo.GetColumnId(testColumns[0].first)}; + THashSet oneColumnId = {indexInfo.GetColumnId(testColumns[0].GetName())}; { // full scan ui64 txId = 1; diff --git a/ydb/core/tx/columnshard/engines/ut/ut_program.cpp b/ydb/core/tx/columnshard/engines/ut/ut_program.cpp index 4cfe3282267a..8df5b086e796 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_program.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_program.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -20,16 +21,16 @@ using TTypeId = NScheme::TTypeId; using TTypeInfo = NScheme::TTypeInfo; namespace { - static const std::vector> testColumns = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, - {"sum", TTypeInfo(NTypeIds::Int32) }, - {"vat", TTypeInfo(NTypeIds::Int32) }, + static const std::vector testColumns = { + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("sum", TTypeInfo(NTypeIds::Int32)), + NArrow::NTest::TTestColumn("vat", TTypeInfo(NTypeIds::Int32)), }; - static const std::vector> testKey = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"uid", TTypeInfo(NTypeIds::Utf8) } + static const std::vector testKey = { + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ) }; } diff --git a/ydb/core/tx/columnshard/engines/ut/ya.make b/ydb/core/tx/columnshard/engines/ut/ya.make index fc8159e31764..41a7d7b2aac3 100644 --- a/ydb/core/tx/columnshard/engines/ut/ya.make +++ b/ydb/core/tx/columnshard/engines/ut/ya.make @@ -23,6 +23,7 @@ PEERDIR( ydb/library/yql/sql/pg_dummy ydb/library/yql/core/arrow_kernels/request ydb/core/testlib/default + ydb/core/tx/columnshard/test_helper ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/hooks/testing diff --git a/ydb/core/tx/columnshard/engines/writer/blob_constructor.h b/ydb/core/tx/columnshard/engines/writer/blob_constructor.h index c2df91c9a605..962f86857037 100644 --- a/ydb/core/tx/columnshard/engines/writer/blob_constructor.h +++ b/ydb/core/tx/columnshard/engines/writer/blob_constructor.h @@ -23,7 +23,7 @@ class TBlobWriteInfo { private: YDB_READONLY_DEF(TUnifiedBlobId, BlobId); YDB_READONLY_DEF(TString, Data); - YDB_READONLY_DEF(std::shared_ptr, WriteOperator); + YDB_ACCESSOR_DEF(std::shared_ptr, WriteOperator); TBlobWriteInfo(const TString& data, const std::shared_ptr& writeOperator) : Data(data) diff --git a/ydb/core/tx/columnshard/engines/writer/compacted_blob_constructor.cpp b/ydb/core/tx/columnshard/engines/writer/compacted_blob_constructor.cpp index 59cd3a62042e..f2bbaac5d4ef 100644 --- a/ydb/core/tx/columnshard/engines/writer/compacted_blob_constructor.cpp +++ b/ydb/core/tx/columnshard/engines/writer/compacted_blob_constructor.cpp @@ -18,9 +18,8 @@ TCompactedWriteController::TCompactedWriteController(const TActorId& dstActor, T auto* pInfo = changes.GetWritePortionInfo(i); Y_ABORT_UNLESS(pInfo); TPortionInfoWithBlobs& portionWithBlobs = *pInfo; - auto action = changes.MutableBlobsAction().GetWriting(portionWithBlobs.GetPortionInfo()); for (auto&& b : portionWithBlobs.GetBlobs()) { - auto& task = AddWriteTask(TBlobWriteInfo::BuildWriteTask(b.GetBlob(), action)); + auto& task = AddWriteTask(TBlobWriteInfo::BuildWriteTask(b.GetBlob(), changes.MutableBlobsAction().GetWriting(b.GetOperator()->GetStorageId()))); b.RegisterBlobId(portionWithBlobs, task.GetBlobId()); } } diff --git a/ydb/core/tx/columnshard/engines/writer/write_controller.cpp b/ydb/core/tx/columnshard/engines/writer/write_controller.cpp index abae873b3c5b..0b3322b955c4 100644 --- a/ydb/core/tx/columnshard/engines/writer/write_controller.cpp +++ b/ydb/core/tx/columnshard/engines/writer/write_controller.cpp @@ -1,5 +1,25 @@ #include "write_controller.h" +#include namespace NKikimr::NColumnShard { +void IWriteController::OnBlobWriteResult(const TEvBlobStorage::TEvPutResult& result) { + NOlap::TUnifiedBlobId blobId(result.GroupId, result.Id); + auto it = WaitingActions.find(result.StorageId ? result.StorageId : NOlap::IStoragesManager::DefaultStorageId); + AFL_VERIFY(it != WaitingActions.end()); + it->second->OnBlobWriteResult(blobId, result.Status); + if (it->second->IsReady()) { + WaitingActions.erase(it); + } + DoOnBlobWriteResult(result); +} + +NKikimr::NOlap::TBlobWriteInfo& IWriteController::AddWriteTask(NOlap::TBlobWriteInfo&& task) { + auto fullAction = WritingActions.Add(task.GetWriteOperator()); + task.SetWriteOperator(fullAction); + WaitingActions.emplace(fullAction->GetStorageId(), fullAction); + WriteTasks.emplace_back(std::move(task)); + return WriteTasks.back(); +} + } diff --git a/ydb/core/tx/columnshard/engines/writer/write_controller.h b/ydb/core/tx/columnshard/engines/writer/write_controller.h index 6652289cd156..32cd2885837f 100644 --- a/ydb/core/tx/columnshard/engines/writer/write_controller.h +++ b/ydb/core/tx/columnshard/engines/writer/write_controller.h @@ -28,8 +28,8 @@ class TBlobPutResult: public NColumnShard::TPutStatus { class IWriteController { private: - THashMap> BlobActions; - THashMap> WritingActions; + THashMap> WaitingActions; + NOlap::TWriteActionsCollection WritingActions; std::deque WriteTasks; protected: virtual void DoOnReadyResult(const NActors::TActorContext& ctx, const TBlobPutResult::TPtr& putResult) = 0; @@ -40,12 +40,12 @@ class IWriteController { } - NOlap::TBlobWriteInfo& AddWriteTask(NOlap::TBlobWriteInfo&& task) { - WritingActions.emplace(task.GetWriteOperator()->GetActionId(), task.GetWriteOperator()); - WriteTasks.emplace_back(std::move(task)); - return WriteTasks.back(); - } + NOlap::TBlobWriteInfo& AddWriteTask(NOlap::TBlobWriteInfo&& task); public: + const NOlap::TWriteActionsCollection& GetBlobActions() const { + return WritingActions; + } + TString DebugString() const { TStringBuilder sb; for (auto&& i : WritingActions) { @@ -76,14 +76,7 @@ class IWriteController { DoOnReadyResult(ctx, putResult); } - void OnBlobWriteResult(const TEvBlobStorage::TEvPutResult& result) { - NOlap::TUnifiedBlobId blobId(result.GroupId, result.Id); - auto it = BlobActions.find(blobId); - AFL_VERIFY(it != BlobActions.end()); - it->second->OnBlobWriteResult(blobId, result.Status); - BlobActions.erase(it); - DoOnBlobWriteResult(result); - } + void OnBlobWriteResult(const TEvBlobStorage::TEvPutResult& result); std::optional Next() { if (WriteTasks.empty()) { @@ -91,19 +84,11 @@ class IWriteController { } auto result = std::move(WriteTasks.front()); WriteTasks.pop_front(); - BlobActions.emplace(result.GetBlobId(), result.GetWriteOperator()); return result; } - bool IsBlobActionsReady() const { - return BlobActions.empty(); - } - std::vector> GetBlobActions() const { - std::vector> actions; - for (auto&& i : WritingActions) { - actions.emplace_back(i.second); - } - return actions; + bool IsReady() const { + return WaitingActions.empty(); } }; diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp index 30465f853e8a..75c015e2434b 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp @@ -53,7 +53,7 @@ void TController::CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& s THashMap> ids; for (auto&& i : granules) { for (auto&& p : i->GetPortions()) { - p.second->FillBlobIdsByStorage(ids); + p.second->FillBlobIdsByStorage(ids, index.GetVersionedIndex()); } } for (auto&& i : ids) { diff --git a/ydb/core/tx/columnshard/inflight_request_tracker.cpp b/ydb/core/tx/columnshard/inflight_request_tracker.cpp new file mode 100644 index 000000000000..ac18968b1f3b --- /dev/null +++ b/ydb/core/tx/columnshard/inflight_request_tracker.cpp @@ -0,0 +1,85 @@ +#include "inflight_request_tracker.h" +#include "engines/column_engine.h" + +namespace NKikimr::NColumnShard { + +void TInFlightReadsTracker::RemoveInFlightRequest(ui64 cookie, const NOlap::TVersionedIndex* index) { + Y_ABORT_UNLESS(RequestsMeta.contains(cookie), "Unknown request cookie %" PRIu64, cookie); + const auto& readMetaList = RequestsMeta[cookie]; + + for (const auto& readMetaBase : readMetaList) { + NOlap::TReadMetadata::TConstPtr readMeta = std::dynamic_pointer_cast(readMetaBase); + + if (!readMeta) { + continue; + } + + THashMap> portionBlobIds; + for (const auto& portion : readMeta->SelectInfo->PortionsOrderedPK) { + const ui64 portionId = portion->GetPortion(); + AFL_VERIFY(index); + portion->FillBlobIdsByStorage(portionBlobIds, *index); + auto it = PortionUseCount.find(portionId); + Y_ABORT_UNLESS(it != PortionUseCount.end(), "Portion id %" PRIu64 " not found in request %" PRIu64, portionId, cookie); + if (it->second == 1) { + PortionUseCount.erase(it); + } else { + it->second--; + } + } + + for (auto&& i : portionBlobIds) { + auto storage = StoragesManager->GetOperatorVerified(i.first); + auto tracker = storage->GetBlobsTracker(); + for (auto& blobId : i.second) { + tracker->FreeBlob(blobId); + } + } + + auto insertStorage = StoragesManager->GetInsertOperator(); + auto tracker = insertStorage->GetBlobsTracker(); + for (const auto& committedBlob : readMeta->CommittedBlobs) { + tracker->FreeBlob(committedBlob.GetBlobRange().GetBlobId()); + } + } + + RequestsMeta.erase(cookie); +} + +void TInFlightReadsTracker::AddToInFlightRequest(const ui64 cookie, NOlap::TReadMetadataBase::TConstPtr readMetaBase, const NOlap::TVersionedIndex* index) { + RequestsMeta[cookie].push_back(readMetaBase); + + NOlap::TReadMetadata::TConstPtr readMeta = std::dynamic_pointer_cast(readMetaBase); + + if (!readMeta) { + return; + } + + auto selectInfo = readMeta->SelectInfo; + Y_ABORT_UNLESS(selectInfo); + SelectStatsDelta += selectInfo->Stats(); + + THashMap> portionBlobIds; + for (const auto& portion : readMeta->SelectInfo->PortionsOrderedPK) { + const ui64 portionId = portion->GetPortion(); + PortionUseCount[portionId]++; + AFL_VERIFY(index); + portion->FillBlobIdsByStorage(portionBlobIds, *index); + } + + for (auto&& i : portionBlobIds) { + auto storage = StoragesManager->GetOperatorVerified(i.first); + auto tracker = storage->GetBlobsTracker(); + for (auto& blobId : i.second) { + tracker->UseBlob(blobId); + } + } + + auto insertStorage = StoragesManager->GetInsertOperator(); + auto tracker = insertStorage->GetBlobsTracker(); + for (const auto& committedBlob : readMeta->CommittedBlobs) { + tracker->UseBlob(committedBlob.GetBlobRange().GetBlobId()); + } +} + +} diff --git a/ydb/core/tx/columnshard/inflight_request_tracker.h b/ydb/core/tx/columnshard/inflight_request_tracker.h index 8b6c35bcac9a..2685e79bd206 100644 --- a/ydb/core/tx/columnshard/inflight_request_tracker.h +++ b/ydb/core/tx/columnshard/inflight_request_tracker.h @@ -3,6 +3,10 @@ #include "blob.h" #include +namespace NKikimr::NOlap { +class TVersionedIndex; +} + namespace NKikimr::NColumnShard { using NOlap::TReadMetadata; @@ -11,57 +15,23 @@ using NOlap::IBlobInUseTracker; class TInFlightReadsTracker { public: // Returns a unique cookie associated with this request - ui64 AddInFlightRequest(NOlap::TReadMetadataBase::TConstPtr readMeta) { + ui64 AddInFlightRequest(NOlap::TReadMetadataBase::TConstPtr readMeta, const NOlap::TVersionedIndex* index) { const ui64 cookie = NextCookie++; - AddToInFlightRequest(cookie, readMeta); + AddToInFlightRequest(cookie, readMeta, index); return cookie; } // Returns a unique cookie associated with this request template - ui64 AddInFlightRequest(const TReadMetadataList& readMetaList) { + ui64 AddInFlightRequest(const TReadMetadataList& readMetaList, const NOlap::TVersionedIndex* index) { const ui64 cookie = NextCookie++; for (const auto& readMetaPtr : readMetaList) { - AddToInFlightRequest(cookie, readMetaPtr); + AddToInFlightRequest(cookie, readMetaPtr, index); } return cookie; } - void RemoveInFlightRequest(ui64 cookie) { - Y_ABORT_UNLESS(RequestsMeta.contains(cookie), "Unknown request cookie %" PRIu64, cookie); - const auto& readMetaList = RequestsMeta[cookie]; - - for (const auto& readMetaBase : readMetaList) { - NOlap::TReadMetadata::TConstPtr readMeta = std::dynamic_pointer_cast(readMetaBase); - - if (!readMeta) { - continue; - } - - for (const auto& portion : readMeta->SelectInfo->PortionsOrderedPK) { - const ui64 portionId = portion->GetPortion(); - auto it = PortionUseCount.find(portionId); - Y_ABORT_UNLESS(it != PortionUseCount.end(), "Portion id %" PRIu64 " not found in request %" PRIu64, portionId, cookie); - if (it->second == 1) { - PortionUseCount.erase(it); - } else { - it->second--; - } - auto tracker = portion->GetBlobsStorage()->GetBlobsTracker(); - for (auto& rec : portion->Records) { - tracker->FreeBlob(rec.BlobRange.BlobId); - } - } - - auto insertStorage = StoragesManager->GetInsertOperator(); - auto tracker = insertStorage->GetBlobsTracker(); - for (const auto& committedBlob : readMeta->CommittedBlobs) { - tracker->FreeBlob(committedBlob.GetBlobRange().GetBlobId()); - } - } - - RequestsMeta.erase(cookie); - } + void RemoveInFlightRequest(ui64 cookie, const NOlap::TVersionedIndex* index); // Checks if the portion is in use by any in-flight request bool IsPortionUsed(ui64 portionId) const { @@ -81,34 +51,7 @@ class TInFlightReadsTracker { } private: - void AddToInFlightRequest(const ui64 cookie, NOlap::TReadMetadataBase::TConstPtr readMetaBase) { - RequestsMeta[cookie].push_back(readMetaBase); - - NOlap::TReadMetadata::TConstPtr readMeta = std::dynamic_pointer_cast(readMetaBase); - - if (!readMeta) { - return; - } - - auto selectInfo = readMeta->SelectInfo; - Y_ABORT_UNLESS(selectInfo); - SelectStatsDelta += selectInfo->Stats(); - - for (const auto& portion : readMeta->SelectInfo->PortionsOrderedPK) { - const ui64 portionId = portion->GetPortion(); - PortionUseCount[portionId]++; - auto tracker = portion->GetBlobsStorage()->GetBlobsTracker(); - for (auto& rec : portion->Records) { - tracker->UseBlob(rec.BlobRange.BlobId); - } - } - - auto insertStorage = StoragesManager->GetInsertOperator(); - auto tracker = insertStorage->GetBlobsTracker(); - for (const auto& committedBlob : readMeta->CommittedBlobs) { - tracker->UseBlob(committedBlob.GetBlobRange().GetBlobId()); - } - } + void AddToInFlightRequest(const ui64 cookie, NOlap::TReadMetadataBase::TConstPtr readMetaBase, const NOlap::TVersionedIndex* index); private: std::shared_ptr StoragesManager; diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp index 81525fc0f905..c30f5fdee626 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp @@ -40,7 +40,7 @@ class TRowsAndBytesChangesTask: public NConveyor::ITask { public: using TDataContainer = std::vector; private: - THashMap Blobs; + NBlobOperations::NRead::TCompositeReadBlobs Blobs; std::vector Chunks; TNormalizationContext NormContext; protected: @@ -48,13 +48,12 @@ class TRowsAndBytesChangesTask: public NConveyor::ITask { for (auto&& chunkInfo : Chunks) { const auto& blobRange = chunkInfo.GetBlobRange(); - auto blobIt = Blobs.find(blobRange); - Y_ABORT_UNLESS(blobIt != Blobs.end()); + auto blobData = Blobs.Extract(IStoragesManager::DefaultStorageId, blobRange); auto columnLoader = chunkInfo.GetLoader(); Y_ABORT_UNLESS(!!columnLoader); - TPortionInfo::TAssembleBlobInfo assembleBlob(blobIt->second); + TPortionInfo::TAssembleBlobInfo assembleBlob(blobData); auto batch = assembleBlob.BuildRecordBatch(*columnLoader); Y_ABORT_UNLESS(!!batch); @@ -68,7 +67,7 @@ class TRowsAndBytesChangesTask: public NConveyor::ITask { } public: - TRowsAndBytesChangesTask(THashMap&& blobs, const TNormalizationContext& nCtx, std::vector&& chunks, std::shared_ptr>) + TRowsAndBytesChangesTask(NBlobOperations::NRead::TCompositeReadBlobs&& blobs, const TNormalizationContext& nCtx, std::vector&& chunks, std::shared_ptr>) : Blobs(std::move(blobs)) , Chunks(std::move(chunks)) , NormContext(nCtx) diff --git a/ydb/core/tx/columnshard/normalizer/portion/min_max.cpp b/ydb/core/tx/columnshard/normalizer/portion/min_max.cpp index 974efa329712..e5da4fa0e75a 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/min_max.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/min_max.cpp @@ -14,7 +14,7 @@ class TMinMaxSnapshotChangesTask: public NConveyor::ITask { public: using TDataContainer = std::vector>; private: - THashMap Blobs; + NBlobOperations::NRead::TCompositeReadBlobs Blobs; TDataContainer Portions; std::shared_ptr> Schemas; TNormalizationContext NormContext; @@ -26,11 +26,10 @@ class TMinMaxSnapshotChangesTask: public NConveyor::ITask { for (auto&& portionInfo : Portions) { auto blobSchema = Schemas->FindPtr(portionInfo->GetPortionId()); - THashMap blobsDataAssemble; + THashMap blobsDataAssemble; for (auto&& i : portionInfo->Records) { - auto blobIt = Blobs.find(i.BlobRange); - Y_ABORT_UNLESS(blobIt != Blobs.end()); - blobsDataAssemble.emplace(i.BlobRange, blobIt->second); + auto blobData = Blobs.Extract((*blobSchema)->GetIndexInfo().GetColumnStorageId(i.GetColumnId(), portionInfo->GetMeta().GetTierName()), i.BlobRange); + blobsDataAssemble.emplace(i.GetAddress(), blobData); } AFL_VERIFY(!!blobSchema)("details", portionInfo->DebugString()); @@ -47,7 +46,7 @@ class TMinMaxSnapshotChangesTask: public NConveyor::ITask { } public: - TMinMaxSnapshotChangesTask(THashMap&& blobs, const TNormalizationContext& nCtx, TDataContainer&& portions, std::shared_ptr> schemas) + TMinMaxSnapshotChangesTask(NBlobOperations::NRead::TCompositeReadBlobs&& blobs, const TNormalizationContext& nCtx, TDataContainer&& portions, std::shared_ptr> schemas) : Blobs(std::move(blobs)) , Portions(std::move(portions)) , Schemas(schemas) diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.h b/ydb/core/tx/columnshard/normalizer/portion/normalizer.h index 7c86476e724f..eb4ef7db4a29 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.h +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.h @@ -37,8 +37,8 @@ class TReadPortionsTask: public NOlap::NBlobOperations::NRead::ITask { NConveyor::TCompServiceOperator::SendTaskToExecute(task); } - virtual bool DoOnError(const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) override { - Y_UNUSED(status, range); + virtual bool DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) override { + Y_UNUSED(status, range, storageId); return false; } diff --git a/ydb/core/tx/columnshard/splitter/batch_slice.cpp b/ydb/core/tx/columnshard/splitter/batch_slice.cpp index 47dfe990eb11..14f10eea25fd 100644 --- a/ydb/core/tx/columnshard/splitter/batch_slice.cpp +++ b/ydb/core/tx/columnshard/splitter/batch_slice.cpp @@ -4,14 +4,18 @@ namespace NKikimr::NOlap { -bool TGeneralSerializedSlice::GroupBlobs(std::vector& blobs) { +bool TGeneralSerializedSlice::GroupBlobsImpl(const TString& currentGroupName, const std::set& entityIds, std::vector& blobs) { std::vector> chunksInProgress; std::sort(Data.begin(), Data.end()); for (auto&& i : Data) { + if (!entityIds.empty() && !entityIds.contains(i.GetEntityId())) { + continue; + } for (auto&& p : i.GetChunks()) { chunksInProgress.emplace_back(p); } } + AFL_VERIFY(chunksInProgress.size()); std::vector result; Y_ABORT_UNLESS(Settings.GetMaxBlobSize() >= 2 * Settings.GetMinBlobSize()); while (chunksInProgress.size()) { @@ -20,7 +24,7 @@ bool TGeneralSerializedSlice::GroupBlobs(std::vector& blobs) { fullSize += i->GetPackedSize(); } if (fullSize < Settings.GetMaxBlobSize()) { - result.emplace_back(TSplittedBlob()); + result.emplace_back(TSplittedBlob(currentGroupName)); for (auto&& i : chunksInProgress) { result.back().Take(i); } @@ -39,7 +43,7 @@ bool TGeneralSerializedSlice::GroupBlobs(std::vector& blobs) { Y_ABORT_UNLESS(otherSize >= Settings.GetMinBlobSize()); Y_ABORT_UNLESS(partSize < Settings.GetMaxBlobSize()); if (partSize >= Settings.GetMinBlobSize()) { - result.emplace_back(TSplittedBlob()); + result.emplace_back(TSplittedBlob(currentGroupName)); for (ui32 chunk = 0; chunk < i; ++chunk) { result.back().Take(chunksInProgress[chunk]); } @@ -60,7 +64,7 @@ bool TGeneralSerializedSlice::GroupBlobs(std::vector& blobs) { chunksInProgress.insert(chunksInProgress.begin() + i, newChunks.begin(), newChunks.end()); } - TSplittedBlob newBlob; + TSplittedBlob newBlob(currentGroupName); for (ui32 chunk = 0; chunk <= i; ++chunk) { newBlob.Take(chunksInProgress[chunk]); } diff --git a/ydb/core/tx/columnshard/splitter/batch_slice.h b/ydb/core/tx/columnshard/splitter/batch_slice.h index 0c0aca979feb..421191230185 100644 --- a/ydb/core/tx/columnshard/splitter/batch_slice.h +++ b/ydb/core/tx/columnshard/splitter/batch_slice.h @@ -110,6 +110,7 @@ class TGeneralSerializedSlice { Y_ABORT_UNLESS(false); return Data.front(); } + bool GroupBlobsImpl(const TString& currentGroupName, const std::set& entityIds, std::vector& blobs); public: @@ -137,17 +138,15 @@ class TGeneralSerializedSlice { return Size; } - std::vector>> GroupChunksByBlobs() { - std::vector>> result; + std::vector GroupChunksByBlobs(const TEntityGroups& groups) { std::vector blobs; - GroupBlobs(blobs); - for (auto&& i : blobs) { - result.emplace_back(i.GetChunks()); - } - return result; + AFL_VERIFY(GroupBlobs(blobs, groups)); + return blobs; } - explicit TGeneralSerializedSlice(TVectorView&& objects) { + explicit TGeneralSerializedSlice(TVectorView&& objects, const TSplitSettings& settings) + : Settings(settings) + { Y_ABORT_UNLESS(objects.size()); std::swap(*this, objects.front()); for (ui32 i = 1; i < objects.size(); ++i) { @@ -159,7 +158,22 @@ class TGeneralSerializedSlice { void MergeSlice(TGeneralSerializedSlice&& slice); - bool GroupBlobs(std::vector& blobs); + bool GroupBlobs(std::vector& blobs, const TEntityGroups& groups) { + if (groups.IsEmpty()) { + return GroupBlobsImpl(groups.GetDefaultGroupName(), {}, blobs); + } else { + std::vector result; + for (auto&& i : groups) { + std::vector blobsLocal; + if (!GroupBlobsImpl(i.first, i.second, blobsLocal)) { + return false; + } + result.insert(result.end(), blobsLocal.begin(), blobsLocal.end()); + } + std::swap(result, blobs); + return true; + } + } bool operator<(const TGeneralSerializedSlice& item) const { return Size < item.Size; diff --git a/ydb/core/tx/columnshard/splitter/blob_info.h b/ydb/core/tx/columnshard/splitter/blob_info.h index a4515f9f7849..d96b55fd0f9c 100644 --- a/ydb/core/tx/columnshard/splitter/blob_info.h +++ b/ydb/core/tx/columnshard/splitter/blob_info.h @@ -6,10 +6,17 @@ namespace NKikimr::NOlap { class TSplittedBlob { private: + YDB_READONLY_DEF(TString, GroupName); YDB_READONLY(i64, Size, 0); YDB_READONLY_DEF(std::vector>, Chunks); public: + TSplittedBlob(const TString& groupName) + : GroupName(groupName) + { + + } + void Take(const std::shared_ptr& chunk); bool operator<(const TSplittedBlob& item) const { return Size > item.Size; diff --git a/ydb/core/tx/columnshard/splitter/rb_splitter.cpp b/ydb/core/tx/columnshard/splitter/rb_splitter.cpp index 698a9e6f3e4a..85e36ee74c05 100644 --- a/ydb/core/tx/columnshard/splitter/rb_splitter.cpp +++ b/ydb/core/tx/columnshard/splitter/rb_splitter.cpp @@ -26,12 +26,12 @@ TRBSplitLimiter::TRBSplitLimiter(std::shared_ptrnum_rows()); } -bool TRBSplitLimiter::Next(std::vector>>& portionBlobs, std::shared_ptr& batch) { +bool TRBSplitLimiter::Next(std::vector>>& portionBlobs, std::shared_ptr& batch, const TEntityGroups& groups) { if (!Slices.size()) { return false; } std::vector blobs; - Slices.front().GroupBlobs(blobs); + Slices.front().GroupBlobs(blobs, groups); std::vector>> result; std::map columnChunks; for (auto&& i : blobs) { diff --git a/ydb/core/tx/columnshard/splitter/rb_splitter.h b/ydb/core/tx/columnshard/splitter/rb_splitter.h index e5d25f2dcef9..ec9112184759 100644 --- a/ydb/core/tx/columnshard/splitter/rb_splitter.h +++ b/ydb/core/tx/columnshard/splitter/rb_splitter.h @@ -63,7 +63,7 @@ class TRBSplitLimiter { return std::move(Slices); } - bool Next(std::vector>>& portionBlobs, std::shared_ptr& batch); + bool Next(std::vector>>& portionBlobs, std::shared_ptr& batch, const TEntityGroups& groups); }; } diff --git a/ydb/core/tx/columnshard/splitter/settings.h b/ydb/core/tx/columnshard/splitter/settings.h index 34bc29783799..07f55449b2ee 100644 --- a/ydb/core/tx/columnshard/splitter/settings.h +++ b/ydb/core/tx/columnshard/splitter/settings.h @@ -2,10 +2,45 @@ #include +#include + #include +#include +#include +#include +#include namespace NKikimr::NOlap { +class TEntityGroups { +private: + THashMap> GroupEntities; + THashSet UsedEntityIds; + YDB_READONLY_DEF(TString, DefaultGroupName); +public: + TEntityGroups(const TString& defaultGroupName) + : DefaultGroupName(defaultGroupName) { + + } + + bool IsEmpty() const { + return GroupEntities.empty(); + } + + void Add(const ui32 entityId, const TString& groupName) { + AFL_VERIFY(UsedEntityIds.emplace(entityId).second); + AFL_VERIFY(GroupEntities[groupName].emplace(entityId).second); + } + + THashMap>::const_iterator begin() const { + return GroupEntities.begin(); + } + + THashMap>::const_iterator end() const { + return GroupEntities.end(); + } +}; + class TSplitSettings { private: static const inline i64 DefaultMaxBlobSize = 8 * 1024 * 1024; diff --git a/ydb/core/tx/columnshard/splitter/ut/ut_splitter.cpp b/ydb/core/tx/columnshard/splitter/ut/ut_splitter.cpp index 72aa5539f6b2..1aa886ad6e5e 100644 --- a/ydb/core/tx/columnshard/splitter/ut/ut_splitter.cpp +++ b/ydb/core/tx/columnshard/splitter/ut/ut_splitter.cpp @@ -79,7 +79,7 @@ Y_UNIT_TEST_SUITE(Splitter) { ui32 blobsCount = 0; ui32 slicesCount = 0; std::shared_ptr sliceBatch; - while (limiter.Next(chunksForBlob, sliceBatch)) { + while (limiter.Next(chunksForBlob, sliceBatch, NKikimr::NOlap::TEntityGroups("default"))) { ++slicesCount; TStringBuilder sb; std::map recordsCountByColumn; diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 87c9aa7c87c6..2bfaa93beeb3 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -304,8 +304,10 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& } void TTablesManager::IndexSchemaVersion(const NOlap::TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) { - NOlap::TIndexInfo indexInfo = DeserializeIndexInfoFromProto(schema); - indexInfo.SetAllKeys(); + std::optional indexInfoOptional = NOlap::TIndexInfo::BuildFromProto(schema, StoragesManager); + Y_ABORT_UNLESS(indexInfoOptional); + NOlap::TIndexInfo indexInfo = std::move(*indexInfoOptional); + indexInfo.SetAllKeys(StoragesManager); const bool isFirstPrimaryIndexInitialization = !PrimaryIndex; if (!PrimaryIndex) { PrimaryIndex = std::make_unique(TabletId, NOlap::TCompactionLimits(), StoragesManager); @@ -319,12 +321,6 @@ void TTablesManager::IndexSchemaVersion(const NOlap::TSnapshot& snapshot, const PrimaryIndex->OnTieringModified(nullptr, Ttl); } -NOlap::TIndexInfo TTablesManager::DeserializeIndexInfoFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema) { - std::optional indexInfo = NOlap::TIndexInfo::BuildFromProto(schema); - Y_ABORT_UNLESS(indexInfo); - return *indexInfo; -} - TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId) : StoragesManager(storagesManager) , TabletId(tabletId) diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 8349ccb9cd3b..5dd3bf039983 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -236,7 +236,6 @@ class TTablesManager { bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); private: void IndexSchemaVersion(const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema); - static NOlap::TIndexInfo DeserializeIndexInfoFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema); }; } diff --git a/ydb/core/tx/columnshard/test_helper/helper.cpp b/ydb/core/tx/columnshard/test_helper/helper.cpp new file mode 100644 index 000000000000..32bfd83882b8 --- /dev/null +++ b/ydb/core/tx/columnshard/test_helper/helper.cpp @@ -0,0 +1,86 @@ +#include "helper.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NKikimr::NArrow::NTest { + +NKikimrSchemeOp::TOlapColumnDescription TTestColumn::CreateColumn(const ui32 id) const { + NKikimrSchemeOp::TOlapColumnDescription col; + col.SetId(id); + col.SetName(Name); + if (StorageId) { + col.SetStorageId(StorageId); + } + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(Type, ""); + col.SetTypeId(columnType.TypeId); + if (columnType.TypeInfo) { + *col.MutableTypeInfo() = *columnType.TypeInfo; + } + return col; +} + +std::vector> TTestColumn::ConvertToPairs(const std::vector& columns) { + std::vector> result; + for (auto&& i : columns) { + result.emplace_back(std::make_pair(i.GetName(), i.GetType())); + } + return result; +} + +std::vector TTestColumn::BuildFromPairs(const std::vector>& columns) { + std::vector result; + for (auto&& i : columns) { + result.emplace_back(i.first, i.second); + } + return result; +} + +THashMap TTestColumn::ConvertToHash(const std::vector& columns) { + THashMap result; + for (auto&& i : columns) { + result.emplace(i.GetName(), i.GetType()); + } + return result; +} + +std::vector TTestColumn::CropSchema(const std::vector& input, const ui32 size) { + AFL_VERIFY(input.size() >= size); + return std::vector(input.begin(), input.begin() + size); +} + +} + +namespace NKikimr::NArrow { + +std::vector> MakeArrowFields(const std::vector& columns, const std::set& notNullColumns /*= {}*/) { + return MakeArrowFields(NTest::TTestColumn::ConvertToPairs(columns), notNullColumns); +} + +std::shared_ptr MakeArrowSchema(const std::vector& columns, const std::set& notNullColumns /*= {}*/) { + return MakeArrowSchema(NTest::TTestColumn::ConvertToPairs(columns), notNullColumns); +} + +} + +namespace NKikimr::NOlap { + +std::shared_ptr TTestStoragesManager::DoBuildOperator(const TString& storageId) { + if (storageId == TBase::DefaultStorageId) { + return std::make_shared(storageId, NActors::TActorId(), TabletInfo, + 1, SharedBlobsManager->GetStorageManagerGuarantee(TBase::DefaultStorageId)); + } else if (storageId == TBase::MemoryStorageId) { + Singleton()->SetSecretKey("fakeSecret"); + return std::make_shared(storageId, NActors::TActorId(), std::make_shared("fakeBucket", "fakeSecret"), + SharedBlobsManager->GetStorageManagerGuarantee(storageId)); + } else { + return nullptr; + } +} + +} \ No newline at end of file diff --git a/ydb/core/tx/columnshard/test_helper/helper.h b/ydb/core/tx/columnshard/test_helper/helper.h new file mode 100644 index 000000000000..afef460918aa --- /dev/null +++ b/ydb/core/tx/columnshard/test_helper/helper.h @@ -0,0 +1,63 @@ +#pragma once +#include +#include + +#include + +#include + +namespace NKikimrSchemeOp { +class TOlapColumnDescription; +} + +namespace NKikimr::NOlap { + +class TTestStoragesManager: public NOlap::IStoragesManager { +private: + using TBase = NOlap::IStoragesManager; + TIntrusivePtr TabletInfo = new TTabletStorageInfo(); + std::shared_ptr SharedBlobsManager = std::make_shared(NOlap::TTabletId(0)); +protected: + virtual bool DoLoadIdempotency(NTable::TDatabase& /*database*/) override { + return true; + } + + virtual std::shared_ptr DoBuildOperator(const TString& storageId) override; + virtual const std::shared_ptr& DoGetSharedBlobsManager() const override { + return SharedBlobsManager; + } +public: +}; + + +} + +namespace NKikimr::NArrow::NTest { + +class TTestColumn { +private: + YDB_ACCESSOR_DEF(TString, Name); + YDB_ACCESSOR_DEF(NScheme::TTypeInfo, Type); + YDB_ACCESSOR_DEF(TString, StorageId); +public: + explicit TTestColumn(const TString& name, const NScheme::TTypeInfo& type) + : Name(name) + , Type(type) { + + } + + NKikimrSchemeOp::TOlapColumnDescription CreateColumn(const ui32 id) const; + static std::vector> ConvertToPairs(const std::vector& columns); + static THashMap ConvertToHash(const std::vector& columns); + static std::vector BuildFromPairs(const std::vector>& columns); + static std::vector CropSchema(const std::vector& input, const ui32 size); +}; + +} + +namespace NKikimr::NArrow { + +std::vector> MakeArrowFields(const std::vector& columns, const std::set& notNullColumns = {}); +std::shared_ptr MakeArrowSchema(const std::vector& columns, const std::set& notNullColumns = {}); + +} diff --git a/ydb/core/tx/columnshard/test_helper/ya.make b/ydb/core/tx/columnshard/test_helper/ya.make new file mode 100644 index 000000000000..e6245a5b40f1 --- /dev/null +++ b/ydb/core/tx/columnshard/test_helper/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +PEERDIR( + ydb/core/protos + contrib/libs/apache/arrow + ydb/library/actors/core + ydb/core/tx/columnshard/blobs_action/bs + ydb/core/tx/columnshard/blobs_action/tier + ydb/core/wrappers +) + +SRCS( + helper.cpp +) + +END() + diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index 3677c0d14cbe..11a18e24b50f 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -352,12 +352,12 @@ void TestWrite(const TestTableDescription& table) { SetupSchema(runtime, sender, tableId, table); - const std::vector>& ydbSchema = table.Schema; + const auto& ydbSchema = table.Schema; bool ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, ydbSchema), ydbSchema); UNIT_ASSERT(ok); - std::vector> schema = ydbSchema; + auto schema = ydbSchema; // no data @@ -373,7 +373,7 @@ void TestWrite(const TestTableDescription& table) { // missing columns - schema.resize(4); + schema = NArrow::NTest::TTestColumn::CropSchema(schema, 4); ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, schema), schema); UNIT_ASSERT(ok); @@ -381,7 +381,7 @@ void TestWrite(const TestTableDescription& table) { // It fails only if we specify source schema. No way to detect it from serialized batch data. schema = ydbSchema; - schema[0].second = TTypeInfo(NTypeIds::Int64); + schema[0].SetType(TTypeInfo(NTypeIds::Int64)); ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, schema), schema); UNIT_ASSERT(!ok); @@ -390,7 +390,7 @@ void TestWrite(const TestTableDescription& table) { for (size_t i = 0; i < ydbSchema.size(); ++i) { schema = ydbSchema; - schema[i].second = TTypeInfo(NTypeIds::Int8); + schema[i].SetType(TTypeInfo(NTypeIds::Int8)); ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, schema), schema); UNIT_ASSERT(!ok); } @@ -399,24 +399,24 @@ void TestWrite(const TestTableDescription& table) { for (size_t i = 0; i < ydbSchema.size(); ++i) { schema = ydbSchema; - schema[i].second = TTypeInfo(NTypeIds::Int64); + schema[i].SetType(TTypeInfo(NTypeIds::Int64)); ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, schema), schema); - UNIT_ASSERT(ok == (ydbSchema[i].second == TTypeInfo(NTypeIds::Int64))); + UNIT_ASSERT(ok == (ydbSchema[i].GetType() == TTypeInfo(NTypeIds::Int64))); } schema = ydbSchema; - schema[1].second = TTypeInfo(NTypeIds::Utf8); - schema[5].second = TTypeInfo(NTypeIds::Int32); + schema[1].SetType(TTypeInfo(NTypeIds::Utf8)); + schema[5].SetType(TTypeInfo(NTypeIds::Int32)); ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, schema), schema); UNIT_ASSERT(!ok); // reordered columns - THashMap remap(ydbSchema.begin(), ydbSchema.end()); + THashMap remap = NArrow::NTest::TTestColumn::ConvertToHash(ydbSchema); - schema.resize(0); + schema.clear(); for (auto& [name, typeInfo] : remap) { - schema.push_back({name, typeInfo}); + schema.push_back(NArrow::NTest::TTestColumn(name, typeInfo)); } ok = WriteData(runtime, sender, writeId++, tableId, MakeTestBlob({0, 100}, schema), schema); @@ -624,7 +624,7 @@ void TestWriteRead(bool reboots, const TestTableDescription& table = {}, TString runtime.DispatchEvents(options); auto write = [&](TTestBasicRuntime& runtime, TActorId& sender, ui64 writeId, ui64 tableId, - const TString& data, const std::vector>& ydbSchema, std::vector& intWriteIds) { + const TString& data, const std::vector& ydbSchema, std::vector& intWriteIds) { bool ok = WriteData(runtime, sender, writeId, tableId, data, ydbSchema, true, &intWriteIds); if (reboots) { RebootTablet(runtime, TTestTxConfig::TxTablet0, sender); @@ -654,8 +654,8 @@ void TestWriteRead(bool reboots, const TestTableDescription& table = {}, TString SetupSchema(runtime, sender, tableId, table, codec); - const std::vector>& ydbSchema = table.Schema; - const std::vector>& testYdbPk = table.Pk; + const std::vector& ydbSchema = table.Schema; + const std::vector& testYdbPk = table.Pk; // ----xx // -----xx.. @@ -946,7 +946,7 @@ void TestCompactionInGranuleImpl(bool reboots, const TestTableDescription& table runtime.DispatchEvents(options); auto write = [&](TTestBasicRuntime& runtime, TActorId& sender, ui64 writeId, ui64 tableId, - const TString& data, const std::vector>& ydbSchema, std::vector& writeIds) { + const TString& data, const std::vector& ydbSchema, std::vector& writeIds) { bool ok = WriteData(runtime, sender, writeId, tableId, data, ydbSchema, true, &writeIds); if (reboots) { RebootTablet(runtime, TTestTxConfig::TxTablet0, sender); @@ -1037,7 +1037,7 @@ void TestCompactionInGranuleImpl(bool reboots, const TestTableDescription& table UNIT_ASSERT(rb); UNIT_ASSERT(reader.IsCorrectlyFinished()); - if (ydbPk[0].second == TTypeInfo(NTypeIds::String) || ydbPk[0].second == TTypeInfo(NTypeIds::Utf8)) { + if (ydbPk[0].GetType() == TTypeInfo(NTypeIds::String) || ydbPk[0].GetType() == TTypeInfo(NTypeIds::Utf8)) { UNIT_ASSERT(DataHas({rb}, triggerPortion, true)); UNIT_ASSERT(DataHas({rb}, smallWrites, true)); } else { @@ -1429,7 +1429,7 @@ struct TReadAggregateResult { std::vector Counts = {100}; }; -void TestReadAggregate(const std::vector>& ydbSchema, const TString& testDataBlob, +void TestReadAggregate(const std::vector& ydbSchema, const TString& testDataBlob, bool addProjection, const std::vector& aggKeys = {}, const TReadAggregateResult& expectedResult = {}, const TReadAggregateResult& expectedFiltered = {1, {1}, {1}, {1}}) { @@ -1448,8 +1448,7 @@ void TestReadAggregate(const std::vector>& ydbSche ui64 planStep = 100; ui64 txId = 100; - auto pk = ydbSchema; - pk.resize(4); + auto pk = NArrow::NTest::TTestColumn::CropSchema(ydbSchema, 4); TestTableDescription table{.Schema = ydbSchema, .Pk = pk}; SetupSchema(runtime, sender, tableId, table); @@ -1479,8 +1478,8 @@ void TestReadAggregate(const std::vector>& ydbSche ui32 prog = 0; for (ui32 i = 0; i < ydbSchema.size(); ++i, ++prog) { - if (intTypes.contains(ydbSchema[i].second.GetTypeId()) || - strTypes.contains(ydbSchema[i].second.GetTypeId())) { + if (intTypes.contains(ydbSchema[i].GetType().GetTypeId()) || + strTypes.contains(ydbSchema[i].GetType().GetTypeId())) { checkResult.insert(prog); } @@ -1496,8 +1495,8 @@ void TestReadAggregate(const std::vector>& ydbSche for (ui32 i = 0; i < ydbSchema.size(); ++i, ++prog) { isFiltered.insert(prog); - if (intTypes.contains(ydbSchema[i].second.GetTypeId()) || - strTypes.contains(ydbSchema[i].second.GetTypeId())) { + if (intTypes.contains(ydbSchema[i].GetType().GetTypeId()) || + strTypes.contains(ydbSchema[i].GetType().GetTypeId())) { checkResult.insert(prog); } @@ -1515,8 +1514,8 @@ void TestReadAggregate(const std::vector>& ydbSche std::vector unnamedColumns = {"100", "101", "102", "103"}; if (!addProjection) { for (auto& key : aggKeys) { - namedColumns.push_back(ydbSchema[key].first); - unnamedColumns.push_back(ydbSchema[key].first); + namedColumns.push_back(ydbSchema[key].GetName()); + unnamedColumns.push_back(ydbSchema[key].GetName()); } } @@ -1566,9 +1565,9 @@ Y_UNIT_TEST_SUITE(EvWrite) { const ui64 ownerId = 0; const ui64 tableId = 1; const ui64 schemaVersion = 1; - const std::vector> schema = { - {"key", TTypeInfo(NTypeIds::Uint64) }, - {"field", TTypeInfo(NTypeIds::Utf8) } + const std::vector schema = { + NArrow::NTest::TTestColumn("key", TTypeInfo(NTypeIds::Uint64)), + NArrow::NTest::TTestColumn("field", TTypeInfo(NTypeIds::Utf8)) }; const std::vector columnsIds = {1, 2}; PrepareTablet(runtime, tableId, schema); @@ -1617,10 +1616,10 @@ Y_UNIT_TEST_SUITE(EvWrite) { const ui64 ownerId = 0; const ui64 tableId = 1; const ui64 schemaVersion = 1; - const std::vector> schema = { - {"key", TTypeInfo(NTypeIds::Uint64) }, - {"field", TTypeInfo(NTypeIds::Utf8) } - }; + const std::vector schema = { + NArrow::NTest::TTestColumn("key", TTypeInfo(NTypeIds::Uint64)), + NArrow::NTest::TTestColumn("field", TTypeInfo(NTypeIds::Utf8)) + }; const std::vector columnsIds = {1, 2}; PrepareTablet(runtime, tableId, schema); const ui64 txId = 111; @@ -1665,10 +1664,10 @@ Y_UNIT_TEST_SUITE(EvWrite) { const ui64 ownerId = 0; const ui64 tableId = 1; const ui64 schemaVersion = 1; - const std::vector> schema = { - {"key", TTypeInfo(NTypeIds::Uint64) }, - {"field", TTypeInfo(NTypeIds::Utf8) } - }; + const std::vector schema = { + NArrow::NTest::TTestColumn("key", TTypeInfo(NTypeIds::Uint64)), + NArrow::NTest::TTestColumn("field", TTypeInfo(NTypeIds::Utf8)) + }; const std::vector columnsIds = {1, 2}; PrepareTablet(runtime, tableId, schema); const ui64 txId = 111; @@ -1710,9 +1709,9 @@ Y_UNIT_TEST_SUITE(EvWrite) { const ui64 ownerId = 0; const ui64 tableId = 1; const ui64 schemaVersion = 1; - const std::vector> schema = { - {"key", TTypeInfo(NTypeIds::Uint64) }, - {"field", TTypeInfo(NTypeIds::Utf8) } + const std::vector schema = { + NArrow::NTest::TTestColumn("key", TTypeInfo(NTypeIds::Uint64) ), + NArrow::NTest::TTestColumn("field", TTypeInfo(NTypeIds::Utf8) ) }; const std::vector columnsIds = {1, 2}; PrepareTablet(runtime, tableId, schema); @@ -1886,8 +1885,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { auto schema = TTestSchema::YdbSchema(); auto pk = TTestSchema::YdbPkSchema(); - schema[0].second = TTypeInfo(typeId); - pk[0].second = TTypeInfo(typeId); + schema[0].SetType(TTypeInfo(typeId)); + pk[0].SetType(TTypeInfo(typeId)); TestTableDescription table{.Schema = schema, .Pk = pk}; TestCompactionInGranuleImpl(reboot, table); } @@ -1968,14 +1967,14 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { Y_UNIT_TEST(ReadSomePrograms) { TestTableDescription table; table.Schema = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"resource_id", TTypeInfo(NTypeIds::Utf8) }, - {"uid", TTypeInfo(NTypeIds::Utf8) }, - {"level", TTypeInfo(NTypeIds::Int32) }, - {"message", TTypeInfo(NTypeIds::Utf8) } + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + NArrow::NTest::TTestColumn("resource_id", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("uid", TTypeInfo(NTypeIds::Utf8) ), + NArrow::NTest::TTestColumn("level", TTypeInfo(NTypeIds::Int32) ), + NArrow::NTest::TTestColumn("message", TTypeInfo(NTypeIds::Utf8) ) }; table.Pk = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) } + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ) }; TestSomePrograms(table); @@ -2012,7 +2011,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { Cerr << "-- group by key: " << key << "\n"; // the type has the same values in test batch so result would be grouped in one row - if (sameValTypes.contains(schema[key].second.GetTypeId())) { + if (sameValTypes.contains(schema[key].GetType().GetTypeId())) { TestReadAggregate(schema, testBlob, (key % 2), { key }, resGrouped, resFiltered); } else { TestReadAggregate(schema, testBlob, (key % 2), { key }, resDefault, resFiltered); @@ -2020,8 +2019,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } for (ui32 key = 0; key < schema.size() - 1; ++key) { Cerr << "-- group by key: " << key << ", " << key + 1 << "\n"; - if (sameValTypes.contains(schema[key].second.GetTypeId()) && - sameValTypes.contains(schema[key + 1].second.GetTypeId())) { + if (sameValTypes.contains(schema[key].GetType().GetTypeId()) && + sameValTypes.contains(schema[key + 1].GetType().GetTypeId())) { TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1 }, resGrouped, resFiltered); } else { TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1 }, resDefault, resFiltered); @@ -2029,9 +2028,9 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } for (ui32 key = 0; key < schema.size() - 2; ++key) { Cerr << "-- group by key: " << key << ", " << key + 1 << ", " << key + 2 << "\n"; - if (sameValTypes.contains(schema[key].second.GetTypeId()) && - sameValTypes.contains(schema[key + 1].second.GetTypeId()) && - sameValTypes.contains(schema[key + 1].second.GetTypeId())) { + if (sameValTypes.contains(schema[key].GetType().GetTypeId()) && + sameValTypes.contains(schema[key + 1].GetType().GetTypeId()) && + sameValTypes.contains(schema[key + 1].GetType().GetTypeId())) { TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1, key + 2 }, resGrouped, resFiltered); } else { TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1, key + 2 }, resDefault, resFiltered); @@ -2044,10 +2043,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TTestBasicRuntime& Runtime; const ui64 PlanStep; const ui64 TxId; - const std::vector> YdbPk; + const std::vector YdbPk; public: - TTabletReadPredicateTest(TTestBasicRuntime& runtime, const ui64 planStep, const ui64 txId, const std::vector>& ydbPk) + TTabletReadPredicateTest(TTestBasicRuntime& runtime, const ui64 planStep, const ui64 txId, const std::vector& ydbPk) : Runtime(runtime) , PlanStep(planStep) , TxId(txId) @@ -2067,14 +2066,14 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { bool GetInclude() const noexcept { return Include; } - std::vector GetCellVec(const std::vector>& pk, + std::vector GetCellVec(const std::vector& pk, std::vector& mem, bool trailingNulls = false) const { UNIT_ASSERT(Border.size() <= pk.size()); std::vector cells; size_t i = 0; for (; i < Border.size(); ++i) { - cells.push_back(MakeTestCell(pk[i].second, Border[i], mem)); + cells.push_back(MakeTestCell(pk[i].GetType(), Border[i], mem)); } for (; trailingNulls && i < pk.size(); ++i) { cells.push_back(TCell()); @@ -2094,7 +2093,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TTestCaseOptions& SetTo(const TBorder& border) { To = border; return *this; } TTestCaseOptions& SetExpectedCount(ui32 count) { ExpectedCount = count; return *this; } - TSerializedTableRange MakeRange(const std::vector>& pk) const { + TSerializedTableRange MakeRange(const std::vector& pk) const { std::vector mem; auto cellsFrom = From ? From->GetCellVec(pk, mem, false) : std::vector(); auto cellsTo = To ? To->GetCellVec(pk, mem) : std::vector(); @@ -2172,7 +2171,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { SetupSchema(runtime, sender, tableId, table, "lz4"); TAutoPtr handle; - bool isStrPk0 = table.Pk[0].second == TTypeInfo(NTypeIds::String) || table.Pk[0].second == TTypeInfo(NTypeIds::Utf8); + bool isStrPk0 = table.Pk[0].GetType() == TTypeInfo(NTypeIds::String) || table.Pk[0].GetType() == TTypeInfo(NTypeIds::Utf8); // Write different keys: grow on compaction @@ -2321,7 +2320,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { if (!keyColumnId) { keyColumnId = internalColumnId; } - Cerr << "[" << __LINE__ << "] " << activity << " " << table.Pk[0].second.GetTypeId() << " " + Cerr << "[" << __LINE__ << "] " << activity << " " << table.Pk[0].GetType().GetTypeId() << " " << pathId << " " << kindStr << " " << numRows << " " << numBytes << " " << numRawBytes << "\n"; if (pathId == tableId) { @@ -2359,12 +2358,12 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { auto schema = TTestSchema::YdbSchema(); auto pk = TTestSchema::YdbPkSchema(); TTestBlobOptions opts; - opts.SameValueColumns.emplace(pk[0].first); + opts.SameValueColumns.emplace(pk[0].GetName()); - schema[0].second = TTypeInfo(typeId); - pk[0].second = TTypeInfo(typeId); - schema[1].second = TTypeInfo(typeId); - pk[1].second = TTypeInfo(typeId); + schema[0].SetType(TTypeInfo(typeId)); + pk[0].SetType(TTypeInfo(typeId)); + schema[1].SetType(TTypeInfo(typeId)); + pk[1].SetType(TTypeInfo(typeId)); TestTableDescription table{.Schema = schema, .Pk = pk}; TestCompactionSplitGranuleImpl(table, opts); } diff --git a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp index fe213d28f72e..d48b6c8189b5 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp @@ -228,10 +228,10 @@ Y_UNIT_TEST_SUITE(Normalizers) { const ui64 ownerId = 0; const ui64 tableId = 1; const ui64 schemaVersion = 1; - const std::vector> schema = { - {"key1", TTypeInfo(NTypeIds::Uint64) }, - {"key2", TTypeInfo(NTypeIds::Uint64) }, - {"field", TTypeInfo(NTypeIds::Utf8) } + const std::vector schema = { + NArrow::NTest::TTestColumn("key1", TTypeInfo(NTypeIds::Uint64)), + NArrow::NTest::TTestColumn("key2", TTypeInfo(NTypeIds::Uint64)), + NArrow::NTest::TTestColumn("field", TTypeInfo(NTypeIds::Utf8) ) }; const std::vector columnsIds = { 1, 2, 3}; PrepareTablet(runtime, tableId, schema, 2); diff --git a/ydb/core/tx/columnshard/ut_rw/ya.make b/ydb/core/tx/columnshard/ut_rw/ya.make index 5932bccf759a..20e556696db2 100644 --- a/ydb/core/tx/columnshard/ut_rw/ya.make +++ b/ydb/core/tx/columnshard/ut_rw/ya.make @@ -22,6 +22,7 @@ PEERDIR( ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/hooks/testing ydb/core/tx/columnshard/common/tests + ydb/core/tx/columnshard/test_helper ydb/services/metadata ydb/core/tx ydb/public/lib/yson_value diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 207f193ea2af..3bc851a02bb4 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -91,8 +91,8 @@ class TWaitCompactionController: public NKikimr::NYDBTest::NColumnShard::TContro namespace { -static const std::vector> testYdbSchema = TTestSchema::YdbSchema(); -static const std::vector> testYdbPk = TTestSchema::YdbPkSchema(); +static const std::vector testYdbSchema = TTestSchema::YdbSchema(); +static const std::vector testYdbPk = TTestSchema::YdbPkSchema(); std::shared_ptr UpdateColumn(std::shared_ptr batch, TString columnName, i64 seconds) { std::string name(columnName.c_str(), columnName.size()); @@ -185,7 +185,7 @@ bool CheckSame(const std::shared_ptr& batch, const ui32 expe } std::vector MakeData(const std::vector& ts, ui32 portionSize, ui32 overlapSize, const TString& ttlColumnName, - const std::vector>& ydbSchema = testYdbSchema) { + const std::vector& ydbSchema = testYdbSchema) { UNIT_ASSERT(ts.size() > 0); ui32 numRows = portionSize + (ts.size() - 1) * (portionSize - overlapSize); @@ -233,16 +233,16 @@ static constexpr ui32 PORTION_ROWS = 80 * 1000; // ts[0] = 1600000000; // date -u --date='@1600000000' Sun Sep 13 12:26:40 UTC 2020 // ts[1] = 1620000000; // date -u --date='@1620000000' Mon May 3 00:00:00 UTC 2021 void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {}, - const std::vector>& ydbSchema = testYdbSchema) + const std::vector& ydbSchema = testYdbSchema) { auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csControllerGuard->SetCompactionEnabled(false); std::vector ts = {1600000000, 1620000000}; ui32 ttlIncSeconds = 1; - for (auto& [name, typeInfo] : ydbSchema) { - if (name == spec.TtlColumn) { - if (typeInfo.GetTypeId() == NTypeIds::Date) { + for (auto& c : ydbSchema) { + if (c.GetName() == spec.TtlColumn) { + if (c.GetType().GetTypeId() == NTypeIds::Date) { ttlIncSeconds = TDuration::Days(1).Seconds(); } break; @@ -1199,13 +1199,12 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { NTypeIds::Datetime }; - auto schema = TTestSchema::YdbSchema({"k0", TTypeInfo(NTypeIds::Timestamp)}); - auto pk = schema; - pk.resize(4); + auto schema = TTestSchema::YdbSchema(NArrow::NTest::TTestColumn("k0", TTypeInfo(NTypeIds::Timestamp))); + auto pk = NArrow::NTest::TTestColumn::CropSchema(schema, 4); for (auto& ydbType : intTypes) { - schema[0].second = TTypeInfo(ydbType); - pk[0].second = TTypeInfo(ydbType); + schema[0].SetType(TTypeInfo(ydbType)); + pk[0].SetType(TTypeInfo(ydbType)); auto txBody = TTestSchema::CreateTableTxBody(tableId, schema, pk); bool ok = TestCreateTable(txBody); UNIT_ASSERT(ok); @@ -1218,8 +1217,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { }; for (auto& ydbType : floatTypes) { - schema[0].second = TTypeInfo(ydbType); - pk[0].second = TTypeInfo(ydbType); + schema[0].SetType(TTypeInfo(ydbType)); + pk[0].SetType(TTypeInfo(ydbType)); auto txBody = TTestSchema::CreateTableTxBody(tableId, schema, pk); bool ok = TestCreateTable(txBody); UNIT_ASSERT(!ok); @@ -1231,8 +1230,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { }; for (auto& ydbType : strTypes) { - schema[0].second = TTypeInfo(ydbType); - pk[0].second = TTypeInfo(ydbType); + schema[0].SetType(TTypeInfo(ydbType)); + pk[0].SetType(TTypeInfo(ydbType)); auto txBody = TTestSchema::CreateTableTxBody(tableId, schema, pk); bool ok = TestCreateTable(txBody); UNIT_ASSERT(ok); @@ -1245,8 +1244,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { }; for (auto& ydbType : xsonTypes) { - schema[0].second = TTypeInfo(ydbType); - pk[0].second = TTypeInfo(ydbType); + schema[0].SetType(TTypeInfo(ydbType)); + pk[0].SetType(TTypeInfo(ydbType)); auto txBody = TTestSchema::CreateTableTxBody(tableId, schema, pk); bool ok = TestCreateTable(txBody); UNIT_ASSERT(!ok); @@ -1260,8 +1259,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { Y_UNIT_TEST(ExternalTTL_Types) { auto ydbSchema = testYdbSchema; for (auto typeId : {NTypeIds::Datetime, NTypeIds::Date, NTypeIds::Uint32, NTypeIds::Uint64}) { - UNIT_ASSERT_EQUAL(ydbSchema[8].first, "saved_at"); - ydbSchema[8].second = TTypeInfo(typeId); + UNIT_ASSERT_EQUAL(ydbSchema[8].GetName(), "saved_at"); + ydbSchema[8].SetType(TTypeInfo(typeId)); TTestSchema::TTableSpecials specs; specs.SetTtlColumn("saved_at"); @@ -1282,8 +1281,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { Y_UNIT_TEST(InternalTTL_Types) { auto ydbSchema = testYdbSchema; for (auto typeId : {NTypeIds::Datetime, NTypeIds::Date, NTypeIds::Uint32, NTypeIds::Uint64}) { - UNIT_ASSERT_EQUAL(ydbSchema[8].first, "saved_at"); - ydbSchema[8].second = TTypeInfo(typeId); + UNIT_ASSERT_EQUAL(ydbSchema[8].GetName(), "saved_at"); + ydbSchema[8].SetType(TTypeInfo(typeId)); TTestSchema::TTableSpecials specs; specs.SetTtlColumn("saved_at"); diff --git a/ydb/core/tx/columnshard/ut_schema/ya.make b/ydb/core/tx/columnshard/ut_schema/ya.make index f94fea4e912b..d3b7fdd5d842 100644 --- a/ydb/core/tx/columnshard/ut_schema/ya.make +++ b/ydb/core/tx/columnshard/ut_schema/ya.make @@ -21,6 +21,7 @@ PEERDIR( ydb/core/testlib/default ydb/core/tx/columnshard/hooks/abstract ydb/core/tx/columnshard/hooks/testing + ydb/core/tx/columnshard/test_helper ydb/services/metadata ydb/core/tx ydb/public/lib/yson_value diff --git a/ydb/core/tx/columnshard/write_actor.cpp b/ydb/core/tx/columnshard/write_actor.cpp index 5d858afe6564..fc57d5ddfafe 100644 --- a/ydb/core/tx/columnshard/write_actor.cpp +++ b/ydb/core/tx/columnshard/write_actor.cpp @@ -41,7 +41,7 @@ class TWriteActor: public TActorBootstrapped, public TMonitoringObj } WriteController->OnBlobWriteResult(*msg); - if (WriteController->IsBlobActionsReady()) { + if (WriteController->IsReady()) { return SendResultAndDie(ctx, NKikimrProto::OK); } } @@ -84,7 +84,7 @@ class TWriteActor: public TActorBootstrapped, public TMonitoringObj writeInfo->GetWriteOperator()->SendWriteBlobRequest(writeInfo->GetData(), writeInfo->GetBlobId()); } - if (WriteController->IsBlobActionsReady()) { + if (WriteController->IsReady()) { return SendResultAndDie(ctx, NKikimrProto::OK); } Become(&TThis::StateWait); diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index d6993c3d65fa..ecb70f40e7e1 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -24,6 +24,7 @@ SRCS( columnshard_view.cpp counters.cpp defs.cpp + inflight_request_tracker.cpp write_actor.cpp tables_manager.cpp ) diff --git a/ydb/core/tx/schemeshard/olap/columns/update.cpp b/ydb/core/tx/schemeshard/olap/columns/update.cpp index a0427414ca75..03e26fecbce5 100644 --- a/ydb/core/tx/schemeshard/olap/columns/update.cpp +++ b/ydb/core/tx/schemeshard/olap/columns/update.cpp @@ -14,6 +14,7 @@ namespace NKikimr::NSchemeShard { Name = columnSchema.GetName(); NotNullFlag = columnSchema.GetNotNull(); TypeName = columnSchema.GetType(); + StorageId = columnSchema.GetStorageId(); if (columnSchema.HasSerializer()) { NArrow::NSerialization::TSerializerContainer serializer; if (!serializer.DeserializeFromProto(columnSchema.GetSerializer())) { @@ -63,6 +64,7 @@ namespace NKikimr::NSchemeShard { void TOlapColumnAdd::ParseFromLocalDB(const NKikimrSchemeOp::TOlapColumnDescription& columnSchema) { Name = columnSchema.GetName(); TypeName = columnSchema.GetType(); + StorageId = columnSchema.GetStorageId(); if (columnSchema.HasTypeInfo()) { Type = NScheme::TypeInfoModFromProtoColumnType( @@ -94,6 +96,7 @@ namespace NKikimr::NSchemeShard { columnSchema.SetName(Name); columnSchema.SetType(TypeName); columnSchema.SetNotNull(NotNullFlag); + columnSchema.SetStorageId(StorageId); if (Serializer) { Serializer->SerializeToProto(*columnSchema.MutableSerializer()); } @@ -110,6 +113,9 @@ namespace NKikimr::NSchemeShard { bool TOlapColumnAdd::ApplyDiff(const TOlapColumnDiff& diffColumn, IErrorCollector& errors) { Y_ABORT_UNLESS(GetName() == diffColumn.GetName()); + if (diffColumn.GetStorageId()) { + StorageId = *diffColumn.GetStorageId(); + } if (diffColumn.GetSerializer()) { Serializer = diffColumn.GetSerializer(); } diff --git a/ydb/core/tx/schemeshard/olap/columns/update.h b/ydb/core/tx/schemeshard/olap/columns/update.h index 26eb18a971af..a9a15823dca6 100644 --- a/ydb/core/tx/schemeshard/olap/columns/update.h +++ b/ydb/core/tx/schemeshard/olap/columns/update.h @@ -14,9 +14,13 @@ class TOlapColumnDiff { YDB_READONLY_DEF(TString, Name); YDB_READONLY_DEF(NArrow::NSerialization::TSerializerContainer, Serializer); YDB_READONLY_DEF(NArrow::NDictionary::TEncodingDiff, DictionaryEncoding); + YDB_READONLY_DEF(std::optional, StorageId); public: bool ParseFromRequest(const NKikimrSchemeOp::TOlapColumnDiff& columnSchema, IErrorCollector& errors) { Name = columnSchema.GetName(); + if (!!columnSchema.GetStorageId()) { + StorageId = columnSchema.GetStorageId(); + } if (!Name) { errors.AddError("empty field name"); return false; @@ -41,6 +45,7 @@ class TOlapColumnAdd { YDB_READONLY_DEF(TString, Name); YDB_READONLY_DEF(TString, TypeName); YDB_READONLY_DEF(NScheme::TTypeInfo, Type); + YDB_READONLY_DEF(TString, StorageId); YDB_FLAG_ACCESSOR(NotNull, false); YDB_READONLY_DEF(std::optional, Serializer); YDB_READONLY_DEF(std::optional, DictionaryEncoding); diff --git a/ydb/core/tx/schemeshard/olap/indexes/schema.cpp b/ydb/core/tx/schemeshard/olap/indexes/schema.cpp index 4275e883711d..0f31bf0e2ede 100644 --- a/ydb/core/tx/schemeshard/olap/indexes/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/indexes/schema.cpp @@ -6,12 +6,14 @@ namespace NKikimr::NSchemeShard { void TOlapIndexSchema::SerializeToProto(NKikimrSchemeOp::TOlapIndexDescription& indexSchema) const { indexSchema.SetId(Id); indexSchema.SetName(Name); + indexSchema.SetStorageId(StorageId); IndexMeta.SerializeToProto(indexSchema); } void TOlapIndexSchema::DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexDescription& indexSchema) { Id = indexSchema.GetId(); Name = indexSchema.GetName(); + StorageId = indexSchema.GetStorageId(); AFL_VERIFY(IndexMeta.DeserializeFromProto(indexSchema))("incorrect_proto", indexSchema.DebugString()); } @@ -22,6 +24,9 @@ bool TOlapIndexSchema::ApplyUpdate(const TOlapSchema& currentSchema, const TOlap errors.AddError("different index classes: " + upsert.GetIndexConstructor().GetClassName() + " vs " + IndexMeta.GetClassName()); return false; } + if (upsert.GetStorageId()) { + StorageId = *upsert.GetStorageId(); + } auto object = upsert.GetIndexConstructor()->CreateIndexMeta(GetId(), GetName(), currentSchema, errors); if (!object) { return false; diff --git a/ydb/core/tx/schemeshard/olap/indexes/schema.h b/ydb/core/tx/schemeshard/olap/indexes/schema.h index 630016fe96a5..1aa302ecb826 100644 --- a/ydb/core/tx/schemeshard/olap/indexes/schema.h +++ b/ydb/core/tx/schemeshard/olap/indexes/schema.h @@ -10,6 +10,7 @@ class TOlapIndexSchema { using TBase = TOlapIndexUpsert; YDB_READONLY(ui32, Id, Max()); YDB_READONLY_DEF(TString, Name); + YDB_READONLY_DEF(TString, StorageId); YDB_READONLY_DEF(NBackgroundTasks::TInterfaceProtoContainer, IndexMeta); public: TOlapIndexSchema() = default; diff --git a/ydb/core/tx/schemeshard/olap/indexes/update.cpp b/ydb/core/tx/schemeshard/olap/indexes/update.cpp index 596cba4b835c..ac2623f24443 100644 --- a/ydb/core/tx/schemeshard/olap/indexes/update.cpp +++ b/ydb/core/tx/schemeshard/olap/indexes/update.cpp @@ -4,11 +4,17 @@ namespace NKikimr::NSchemeShard { void TOlapIndexUpsert::SerializeToProto(NKikimrSchemeOp::TOlapIndexRequested& requestedProto) const { requestedProto.SetName(Name); + if (StorageId && !!*StorageId) { + requestedProto.SetStorageId(*StorageId); + } IndexConstructor.SerializeToProto(requestedProto); } void TOlapIndexUpsert::DeserializeFromProto(const NKikimrSchemeOp::TOlapIndexRequested& indexSchema) { Name = indexSchema.GetName(); + if (!!indexSchema.GetStorageId()) { + StorageId = indexSchema.GetStorageId(); + } AFL_VERIFY(IndexConstructor.DeserializeFromProto(indexSchema))("incorrect_proto", indexSchema.DebugString()); } diff --git a/ydb/core/tx/schemeshard/olap/indexes/update.h b/ydb/core/tx/schemeshard/olap/indexes/update.h index f6d0f88fa312..c9bf0d61628e 100644 --- a/ydb/core/tx/schemeshard/olap/indexes/update.h +++ b/ydb/core/tx/schemeshard/olap/indexes/update.h @@ -11,6 +11,7 @@ namespace NKikimr::NSchemeShard { private: YDB_READONLY_DEF(TString, Name); YDB_READONLY_DEF(TString, TypeName); + YDB_READONLY_DEF(std::optional, StorageId); protected: NBackgroundTasks::TInterfaceProtoContainer IndexConstructor; public: diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index 06c25b6bdd61..d54afe82b33e 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -38,9 +38,9 @@ static const TString defaultTableSchema = R"( } )"; -static const TVector> defaultYdbSchema = { - {"timestamp", TTypeInfo(NTypeIds::Timestamp) }, - {"data", TTypeInfo(NTypeIds::Utf8) } +static const TVector defaultYdbSchema = { + NArrow::NTest::TTestColumn("timestamp", TTypeInfo(NTypeIds::Timestamp) ), + NArrow::NTest::TTestColumn("data", TTypeInfo(NTypeIds::Utf8) ) }; }} diff --git a/ydb/core/tx/schemeshard/ut_olap/ya.make b/ydb/core/tx/schemeshard/ut_olap/ya.make index 8bf46e35dc57..c502f450c32c 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ya.make +++ b/ydb/core/tx/schemeshard/ut_olap/ya.make @@ -21,6 +21,7 @@ PEERDIR( ydb/core/formats ydb/core/tx ydb/core/tx/columnshard + ydb/core/tx/columnshard/test_helper ydb/core/tx/schemeshard/ut_helpers ydb/library/yql/public/udf/service/exception_policy ) diff --git a/ydb/core/wrappers/fake_storage.h b/ydb/core/wrappers/fake_storage.h index 510fb0c2ab4c..b3393c28756e 100644 --- a/ydb/core/wrappers/fake_storage.h +++ b/ydb/core/wrappers/fake_storage.h @@ -40,6 +40,7 @@ class TFakeBucketStorage { return it->second; } void Remove(const TString& objectId) { + TGuard g(Mutex); Data.erase(objectId); } }; @@ -106,12 +107,17 @@ class TFakeExternalStorageOperator: public IExternalStorageOperator { private: const TString Bucket; const TString SecretKey; + std::shared_ptr OwnedStorage; template void ExecuteImpl(TEvent& ev) const { ev->Get()->MutableRequest().WithBucket(Bucket); Y_ABORT_UNLESS(SecretKey == Singleton()->GetSecretKey()); - Singleton()->Execute(ev, ReplyAdapter); + if (OwnedStorage) { + OwnedStorage->Execute(ev, ReplyAdapter); + } else { + Singleton()->Execute(ev, ReplyAdapter); + } } virtual TString DoDebugString() const override { @@ -119,9 +125,10 @@ class TFakeExternalStorageOperator: public IExternalStorageOperator { } public: - TFakeExternalStorageOperator(const TString& bucket, const TString& secretKey) + TFakeExternalStorageOperator(const TString& bucket, const TString& secretKey, const std::shared_ptr storage = {}) : Bucket(bucket) , SecretKey(secretKey) + , OwnedStorage(storage) { } virtual void Execute(TEvCheckObjectExistsRequest::TPtr& ev) const override {