diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index a238fa347459..6268cbe1a641 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -592,11 +592,10 @@ template TPortionInfo::TPreparedBatchData PrepareForAssembleImpl(const TPortionInfo& portion, const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, THashMap& blobsData) { std::vector columns; - auto arrowResultSchema = resultSchema.GetSchema(); - columns.reserve(arrowResultSchema->num_fields()); + columns.reserve(resultSchema.GetColumnIds().size()); const ui32 rowsCount = portion.GetRecordsCount(); - for (auto&& i : arrowResultSchema->fields()) { - columns.emplace_back(rowsCount, dataSchema.GetColumnLoaderOptional(i->name()), resultSchema.GetColumnLoaderVerified(i->name())); + for (auto&& i : resultSchema.GetColumnIds()) { + columns.emplace_back(rowsCount, dataSchema.GetColumnLoaderOptional(i), resultSchema.GetColumnLoaderVerified(i)); } { int skipColumnId = -1; @@ -628,7 +627,7 @@ TPortionInfo::TPreparedBatchData PrepareForAssembleImpl(const TPortionInfo& port preparedColumns.emplace_back(c.Compile()); } - return TPortionInfo::TPreparedBatchData(std::move(preparedColumns), arrowResultSchema, rowsCount); + return TPortionInfo::TPreparedBatchData(std::move(preparedColumns), resultSchema.GetSchema(), rowsCount); } } diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h index 496a7533d428..1a7a4f831f77 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h @@ -109,9 +109,15 @@ class IIndexInfo { [[nodiscard]] static std::vector AddSpecialFieldIds(const std::vector& baseColumnIds) { std::vector result = baseColumnIds; - for (auto&& i : GetSystemColumnIds()) { - result.emplace_back(i); - } + const auto& cIds = GetSystemColumnIds(); + result.insert(result.end(), cIds.begin(), cIds.end()); + return result; + } + + [[nodiscard]] static std::set AddSpecialFieldIds(const std::set& baseColumnIds) { + std::set result = baseColumnIds; + const auto& cIds = GetSystemColumnIds(); + result.insert(cIds.begin(), cIds.end()); return result; } diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index f0ff74084dab..7822060a921a 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -334,6 +334,7 @@ void TIndexInfo::InitializeCaches(const std::shared_ptr& opera } SchemaWithSpecials = IIndexInfo::AddSpecialFields(Schema); SchemaColumnIdsWithSpecials = IIndexInfo::AddSpecialFieldIds(SchemaColumnIds); + SchemaColumnIdsWithSpecialsSet = IIndexInfo::AddSpecialFieldIds(std::set(SchemaColumnIds.begin(), SchemaColumnIds.end())); for (auto&& c : Columns) { AFL_VERIFY(ArrowColumnByColumnIdCache.emplace(c.first, GetColumnFieldVerified(c.first)).second); diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index b2be86d1e964..a53475fe2e70 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -255,6 +255,9 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { std::vector GetColumnNames(const std::vector& ids) const; std::vector GetColumnSTLNames(const std::vector& ids) const; const std::vector& GetColumnIds(const bool withSpecial = true) const; + const std::set& GetColumnIdsSet() const { + return SchemaColumnIdsWithSpecialsSet; + } const std::vector& GetPKColumnIds() const { AFL_VERIFY(PKColumnIds.size()); return PKColumnIds; @@ -324,6 +327,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { TString Name; std::vector SchemaColumnIds; std::vector SchemaColumnIdsWithSpecials; + std::set SchemaColumnIdsWithSpecialsSet; std::vector PKColumnIds; std::shared_ptr Schema; std::shared_ptr SchemaWithSpecials; diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h index 7153866b3387..39146058cf75 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h @@ -27,6 +27,7 @@ class ISnapshotSchema { std::shared_ptr GetColumnLoaderVerified(const std::string& columnName) const; bool IsSpecialColumnId(const ui32 columnId) const; + virtual const std::set& GetColumnIds() const = 0; virtual NArrow::NAccessor::TColumnSaver GetColumnSaver(const ui32 columnId) const = 0; NArrow::NAccessor::TColumnSaver GetColumnSaver(const TString& columnName) const { diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp index 8832e7eb0ec8..85ac212338da 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.cpp @@ -5,8 +5,8 @@ namespace NKikimr::NOlap { TFilteredSnapshotSchema::TFilteredSnapshotSchema(ISnapshotSchema::TPtr originalSnapshot, const std::vector& columnIds) - : TFilteredSnapshotSchema(originalSnapshot, std::set(columnIds.begin(), columnIds.end())) -{} + : TFilteredSnapshotSchema(originalSnapshot, std::set(columnIds.begin(), columnIds.end())) { +} TFilteredSnapshotSchema::TFilteredSnapshotSchema(ISnapshotSchema::TPtr originalSnapshot, const std::set& columnIds) : OriginalSnapshot(originalSnapshot) @@ -48,7 +48,14 @@ std::shared_ptr TFilteredSnapshotSchema::GetColumnLoaderOptional( } std::optional TFilteredSnapshotSchema::GetColumnIdOptional(const std::string& columnName) const { - return OriginalSnapshot->GetColumnIdOptional(columnName); + auto result = OriginalSnapshot->GetColumnIdOptional(columnName); + if (!result) { + return result; + } + if (!ColumnIds.contains(*result)) { + return std::nullopt; + } + return result; } int TFilteredSnapshotSchema::GetFieldIndex(const ui32 columnId) const { diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h index 1b515d5bb9cf..3d54b9aa57c1 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h @@ -9,7 +9,7 @@ namespace NKikimr::NOlap { class TFilteredSnapshotSchema: public ISnapshotSchema { ISnapshotSchema::TPtr OriginalSnapshot; std::shared_ptr Schema; - YDB_READONLY_DEF(std::set, ColumnIds); + std::set ColumnIds; protected: virtual TString DoDebugString() const override; public: @@ -17,6 +17,9 @@ class TFilteredSnapshotSchema: public ISnapshotSchema { TFilteredSnapshotSchema(ISnapshotSchema::TPtr originalSnapshot, const std::set& columnIds); TFilteredSnapshotSchema(ISnapshotSchema::TPtr originalSnapshot, const std::set& columnNames); + virtual const std::set& GetColumnIds() const override { + return ColumnIds; + } TColumnSaver GetColumnSaver(const ui32 columnId) const override; std::shared_ptr GetColumnLoaderOptional(const ui32 columnId) const override; std::optional GetColumnIdOptional(const std::string& columnName) const override; diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h index 539d8f99a02c..2648409d8ca7 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h @@ -23,6 +23,10 @@ class TSnapshotSchema: public ISnapshotSchema { public: TSnapshotSchema(TIndexInfo&& indexInfo, const TSnapshot& snapshot); + virtual const std::set& GetColumnIds() const override { + return IndexInfo.GetColumnIdsSet(); + } + TColumnSaver GetColumnSaver(const ui32 columnId) const override; std::shared_ptr GetColumnLoaderOptional(const ui32 columnId) const override; std::optional GetColumnIdOptional(const std::string& columnName) const override;