From b76781590143d25064f02de794bd557d1d3e4297 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 10:45:07 +0300 Subject: [PATCH 01/12] remove deprecated data from column chunk --- .../engines/portions/portion_info.cpp | 15 ----------- .../engines/portions/portion_info.h | 2 -- .../storage/actualizer/tiering/tiering.cpp | 13 +++++----- .../splitter/abstract/chunk_meta.cpp | 11 +------- .../splitter/abstract/chunk_meta.h | 3 +-- .../tx/schemeshard/olap/schema/schema.cpp | 25 +++++++++++++++++-- 6 files changed, 31 insertions(+), 38 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 3aea7cf48b22..9f529bf682cf 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -17,21 +17,6 @@ namespace NKikimr::NOlap { -std::shared_ptr TPortionInfo::MaxValue(ui32 columnId) const { - std::shared_ptr result; - for (auto&& i : Records) { - if (i.ColumnId == columnId) { - if (!i.GetMeta().GetMax()) { - return nullptr; - } - if (!result || NArrow::ScalarCompare(result, i.GetMeta().GetMax()) < 0) { - result = i.GetMeta().GetMax(); - } - } - } - return result; -} - ui64 TPortionInfo::GetColumnRawBytes(const std::set& entityIds, const bool validation) const { ui64 sum = 0; const auto aggr = [&](const TColumnRecord& r) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 32d6c7d99b28..bd3bfe7c4138 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -496,8 +496,6 @@ class TPortionInfo { return visible; } - std::shared_ptr MaxValue(ui32 columnId) const; - const NArrow::TReplaceKey& IndexKeyStart() const { return Meta.IndexKeyStart; } diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index b8481aab3e88..cc15d01b7ee0 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -28,16 +28,15 @@ std::optional TTieringActualizer::Bu AFL_VERIFY(TieringColumnId); auto indexMeta = portionSchema->GetIndexInfo().GetIndexMetaMax(*TieringColumnId); std::shared_ptr max; - if (!indexMeta) { - max = portion.MaxValue(*TieringColumnId); - if (!max) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "scalar_less_not_max"); - return {}; - } - } else { + if (*TieringColumnId == portionSchema->GetIndexInfo().GetPKColumnIds().front()) { + max = portion.GetMeta().GetFirstLastPK().GetFirst().Column(0).GetScalar(0); + } else if (indexMeta) { NYDBTest::TControllers::GetColumnShardController()->OnStatisticsUsage(NIndexes::TIndexMetaContainer(indexMeta)); const std::vector data = portion.GetIndexInplaceDataVerified(indexMeta->GetIndexId()); max = indexMeta->GetMaxScalarVerified(data, portionSchema->GetIndexInfo().GetColumnFieldVerified(*TieringColumnId)->type()); + } else { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "no data for ttl usage (need to create index or use first pk column)"); + return {}; } auto tieringInfo = Tiering->GetTierToMove(max, now); AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("tiering_info", tieringInfo.DebugString()); diff --git a/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.cpp b/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.cpp index 1e66bfb46e07..6e680b91447b 100644 --- a/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.cpp +++ b/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.cpp @@ -5,20 +5,11 @@ namespace NKikimr::NOlap { TSimpleChunkMeta::TSimpleChunkMeta( - const std::shared_ptr& column, const bool needMax, const bool isSortedColumn) { + const std::shared_ptr& column) { Y_ABORT_UNLESS(column); Y_ABORT_UNLESS(column->GetRecordsCount()); NumRows = column->GetRecordsCount(); RawBytes = column->GetRawSizeVerified(); - - if (needMax) { - if (!isSortedColumn) { - Max = column->GetMaxScalar(); - } else { - Max = column->GetScalar(column->GetRecordsCount() - 1); - } -// AFL_VERIFY(Max); - } } } diff --git a/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h b/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h index 526a2a037967..60414037c924 100644 --- a/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h +++ b/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h @@ -14,12 +14,11 @@ namespace NKikimr::NOlap { class TSimpleChunkMeta { protected: - std::shared_ptr Max; ui32 NumRows = 0; ui32 RawBytes = 0; TSimpleChunkMeta() = default; public: - TSimpleChunkMeta(const std::shared_ptr& column, const bool needMinMax, const bool isSortedColumn); + TSimpleChunkMeta(const std::shared_ptr& column); ui64 GetMetadataSize() const { return sizeof(ui32) + sizeof(ui32) + 8 * 3 * 2; diff --git a/ydb/core/tx/schemeshard/olap/schema/schema.cpp b/ydb/core/tx/schemeshard/olap/schema/schema.cpp index c4a4edf5f96f..84491db2c0e8 100644 --- a/ydb/core/tx/schemeshard/olap/schema/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/schema/schema.cpp @@ -15,7 +15,7 @@ static inline NScheme::TTypeInfo GetType(const TOlapColumnsDescription::TColumn& } -static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, +static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, const THashMap& sourceColumns, const THashMap& alterColumns, const THashMap& colName2Id, @@ -70,6 +70,27 @@ static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle:: errors.AddError(errStr); return false; } + { + bool correct = false; + if (column->GetKeyOrder() && *column->GetKeyOrder() == 0) { + correct = true; + } else { + for (auto&& [_, i] : indexes.GetIndexes()) { + auto meta = std::dynamic_pointer_casti.GetIndexMeta(); + if (!meta) { + continue; + } else if (meta->GetColumnId() == column->GetId()) { + correct = true; + break; + } + } + } + if (!correct) { + errors.AddError("Haven't MAX-index for TTL column and TTL column is not first column in primary key"); + return false; + } + } + return true; } @@ -83,7 +104,7 @@ bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycl errors.AddError("Incorrect ttl column - not found in scheme"); return false; } - return ValidateColumnTableTtl(ttl.GetEnabled(), {}, Columns.GetColumns(), Columns.GetColumnsByName(), errors); + return ValidateColumnTableTtl(ttl.GetEnabled(), {}, Indexes, Columns.GetColumns(), Columns.GetColumnsByName(), errors); } case TTtlProto::kDisabled: default: From b06289246f659976bcd6b473efe979c765992fd9 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 11:01:02 +0300 Subject: [PATCH 02/12] fix --- .../engines/storage/actualizer/tiering/tiering.cpp | 2 +- ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h | 9 +-------- ydb/core/tx/schemeshard/olap/schema/schema.cpp | 5 +++-- ydb/services/bg_tasks/abstract/interface.h | 6 ++++++ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index cc15d01b7ee0..940d36cf6a45 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -29,7 +29,7 @@ std::optional TTieringActualizer::Bu auto indexMeta = portionSchema->GetIndexInfo().GetIndexMetaMax(*TieringColumnId); std::shared_ptr max; if (*TieringColumnId == portionSchema->GetIndexInfo().GetPKColumnIds().front()) { - max = portion.GetMeta().GetFirstLastPK().GetFirst().Column(0).GetScalar(0); + max = NArrow::TStatusValidator::GetValid(portion.GetMeta().GetFirstLastPK().GetFirst().Column(0).GetScalar(0)); } else if (indexMeta) { NYDBTest::TControllers::GetColumnShardController()->OnStatisticsUsage(NIndexes::TIndexMetaContainer(indexMeta)); const std::vector data = portion.GetIndexInplaceDataVerified(indexMeta->GetIndexId()); diff --git a/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h b/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h index 60414037c924..53de4f2b3b61 100644 --- a/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h +++ b/ydb/core/tx/columnshard/splitter/abstract/chunk_meta.h @@ -21,12 +21,9 @@ class TSimpleChunkMeta { TSimpleChunkMeta(const std::shared_ptr& column); ui64 GetMetadataSize() const { - return sizeof(ui32) + sizeof(ui32) + 8 * 3 * 2; + return sizeof(ui32) + sizeof(ui32); } - std::shared_ptr GetMax() const { - return Max; - } ui32 GetNumRows() const { return NumRows; } @@ -37,9 +34,5 @@ class TSimpleChunkMeta { return RawBytes; } - bool HasMax() const noexcept { - return Max.get(); - } - }; } diff --git a/ydb/core/tx/schemeshard/olap/schema/schema.cpp b/ydb/core/tx/schemeshard/olap/schema/schema.cpp index 84491db2c0e8..bfe1add31b65 100644 --- a/ydb/core/tx/schemeshard/olap/schema/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/schema/schema.cpp @@ -1,5 +1,6 @@ #include "schema.h" #include +#include namespace NKikimr::NSchemeShard { @@ -76,7 +77,7 @@ static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle:: correct = true; } else { for (auto&& [_, i] : indexes.GetIndexes()) { - auto meta = std::dynamic_pointer_casti.GetIndexMeta(); + auto meta = i.GetIndexMeta().GetObjectPtrOptionalAs(); if (!meta) { continue; } else if (meta->GetColumnId() == column->GetId()) { @@ -104,7 +105,7 @@ bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycl errors.AddError("Incorrect ttl column - not found in scheme"); return false; } - return ValidateColumnTableTtl(ttl.GetEnabled(), {}, Indexes, Columns.GetColumns(), Columns.GetColumnsByName(), errors); + return ValidateColumnTableTtl(ttl.GetEnabled(), Indexes, {}, Columns.GetColumns(), Columns.GetColumnsByName(), errors); } case TTtlProto::kDisabled: default: diff --git a/ydb/services/bg_tasks/abstract/interface.h b/ydb/services/bg_tasks/abstract/interface.h index 75f6d44c7ce6..b69558e5db13 100644 --- a/ydb/services/bg_tasks/abstract/interface.h +++ b/ydb/services/bg_tasks/abstract/interface.h @@ -163,6 +163,12 @@ class TCommonInterfaceContainer { return result; } + template + std::shared_ptr GetObjectPtrOptionalAs() const { + auto result = std::dynamic_pointer_cast(Object); + return result; + } + const IInterface& GetObjectVerified() const { AFL_VERIFY(Object); return *Object; From 5d5645513ae7a3e33fa210e8dd49e75c4863c2e6 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 11:39:17 +0300 Subject: [PATCH 03/12] fix --- .../engines/portions/column_record.cpp | 46 +++++++------------ .../engines/portions/column_record.h | 29 +++++------- .../engines/portions/constructor.cpp | 2 +- .../engines/portions/portion_info.cpp | 11 +++-- .../engines/portions/portion_info.h | 4 +- 5 files changed, 37 insertions(+), 55 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/portions/column_record.cpp b/ydb/core/tx/columnshard/engines/portions/column_record.cpp index 6127ad439326..26c591be64c3 100644 --- a/ydb/core/tx/columnshard/engines/portions/column_record.cpp +++ b/ydb/core/tx/columnshard/engines/portions/column_record.cpp @@ -1,63 +1,49 @@ #include "column_record.h" #include - -#include +#include #include +#include #include -#include namespace NKikimr::NOlap { -TConclusionStatus TChunkMeta::DeserializeFromProto(const TChunkAddress& address, const NKikimrTxColumnShard::TIndexColumnMeta& proto, const TSimpleColumnInfo& columnInfo) { - auto field = columnInfo.GetArrowField(); +TConclusionStatus TChunkMeta::DeserializeFromProto(const NKikimrTxColumnShard::TIndexColumnMeta& proto) { if (proto.HasNumRows()) { NumRows = proto.GetNumRows(); } if (proto.HasRawBytes()) { RawBytes = proto.GetRawBytes(); } - if (proto.HasMaxValue()) { - AFL_VERIFY(field)("field_id", address.GetColumnId())("field_name", columnInfo.GetColumnName()); - Max = ConstantToScalar(proto.GetMaxValue(), field->type()); - } return TConclusionStatus::Success(); } -TChunkMeta::TChunkMeta(const TColumnChunkLoadContext& context, const TSimpleColumnInfo& columnInfo) { - DeserializeFromProto(context.GetAddress(), context.GetMetaProto(), columnInfo).Validate(); +TChunkMeta::TChunkMeta(const TColumnChunkLoadContext& context) { + DeserializeFromProto(context.GetMetaProto()).Validate(); } -TChunkMeta::TChunkMeta(const std::shared_ptr& column, const TSimpleColumnInfo& columnInfo) - : TBase(column, columnInfo.GetNeedMinMax(), columnInfo.GetIsSorted()) -{ +TChunkMeta::TChunkMeta(const std::shared_ptr& column) + : TBase(column) { } NKikimrTxColumnShard::TIndexColumnMeta TChunkMeta::SerializeToProto() const { NKikimrTxColumnShard::TIndexColumnMeta meta; meta.SetNumRows(NumRows); meta.SetRawBytes(RawBytes); - if (HasMax()) { - ScalarToConstant(*Max, *meta.MutableMaxValue()); - ScalarToConstant(*Max, *meta.MutableMinValue()); - } return meta; } -TColumnRecord::TColumnRecord(const TBlobRangeLink16::TLinkId blobLinkId, const TColumnChunkLoadContext& loadContext, const TSimpleColumnInfo& columnInfo) - : Meta(loadContext, columnInfo) +TColumnRecord::TColumnRecord(const TBlobRangeLink16::TLinkId blobLinkId, const TColumnChunkLoadContext& loadContext) + : Meta(loadContext) , ColumnId(loadContext.GetAddress().GetColumnId()) , Chunk(loadContext.GetAddress().GetChunk()) - , BlobRange(loadContext.GetBlobRange().BuildLink(blobLinkId)) -{ + , BlobRange(loadContext.GetBlobRange().BuildLink(blobLinkId)) { } -TColumnRecord::TColumnRecord( - const TChunkAddress& address, const std::shared_ptr& column, const TSimpleColumnInfo& columnInfo) - : Meta(column, columnInfo) +TColumnRecord::TColumnRecord(const TChunkAddress& address, const std::shared_ptr& column) + : Meta(column) , ColumnId(address.GetColumnId()) - , Chunk(address.GetChunk()) -{ + , Chunk(address.GetChunk()) { } NKikimrColumnShardDataSharingProto::TColumnRecord TColumnRecord::SerializeToProto() const { @@ -69,11 +55,11 @@ NKikimrColumnShardDataSharingProto::TColumnRecord TColumnRecord::SerializeToProt return result; } -NKikimr::TConclusionStatus TColumnRecord::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TColumnRecord& proto, const TSimpleColumnInfo& columnInfo) { +NKikimr::TConclusionStatus TColumnRecord::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TColumnRecord& proto) { ColumnId = proto.GetColumnId(); Chunk = proto.GetChunkIdx(); { - auto parse = Meta.DeserializeFromProto(GetAddress(), proto.GetMeta(), columnInfo); + auto parse = Meta.DeserializeFromProto(proto.GetMeta()); if (!parse) { return parse; } @@ -88,4 +74,4 @@ NKikimr::TConclusionStatus TColumnRecord::DeserializeFromProto(const NKikimrColu return TConclusionStatus::Success(); } -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/column_record.h b/ydb/core/tx/columnshard/engines/portions/column_record.h index 18fd0984d61b..fd2efc97e9b9 100644 --- a/ydb/core/tx/columnshard/engines/portions/column_record.h +++ b/ydb/core/tx/columnshard/engines/portions/column_record.h @@ -2,8 +2,6 @@ #include "common.h" -#include -#include #include #include #include @@ -11,6 +9,8 @@ #include #include +#include +#include #include #include @@ -30,8 +30,7 @@ struct TChunkMeta: public TSimpleChunkMeta { private: using TBase = TSimpleChunkMeta; TChunkMeta() = default; - [[nodiscard]] TConclusionStatus DeserializeFromProto( - const TChunkAddress& address, const NKikimrTxColumnShard::TIndexColumnMeta& proto, const TSimpleColumnInfo& columnInfo); + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrTxColumnShard::TIndexColumnMeta& proto); friend class TColumnRecord; public: @@ -39,10 +38,9 @@ struct TChunkMeta: public TSimpleChunkMeta { : TBase(baseMeta) { } - [[nodiscard]] static TConclusion BuildFromProto( - const TChunkAddress& address, const NKikimrTxColumnShard::TIndexColumnMeta& proto, const TSimpleColumnInfo& columnInfo) { + [[nodiscard]] static TConclusion BuildFromProto(const NKikimrTxColumnShard::TIndexColumnMeta& proto) { TChunkMeta result; - auto parse = result.DeserializeFromProto(address, proto, columnInfo); + auto parse = result.DeserializeFromProto(proto); if (!parse) { return parse; } @@ -61,9 +59,9 @@ struct TChunkMeta: public TSimpleChunkMeta { } }; - TChunkMeta(const TColumnChunkLoadContext& context, const TSimpleColumnInfo& columnInfo); + TChunkMeta(const TColumnChunkLoadContext& context); - TChunkMeta(const std::shared_ptr& column, const TSimpleColumnInfo& columnInfo); + TChunkMeta(const std::shared_ptr& column); }; class TColumnRecord { @@ -74,7 +72,7 @@ class TColumnRecord { } TColumnRecord() = default; - TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TColumnRecord& proto, const TSimpleColumnInfo& columnInfo); + TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TColumnRecord& proto); public: ui32 ColumnId = 0; @@ -124,10 +122,9 @@ class TColumnRecord { } NKikimrColumnShardDataSharingProto::TColumnRecord SerializeToProto() const; - static TConclusion BuildFromProto( - const NKikimrColumnShardDataSharingProto::TColumnRecord& proto, const TSimpleColumnInfo& columnInfo) { + static TConclusion BuildFromProto(const NKikimrColumnShardDataSharingProto::TColumnRecord& proto) { TColumnRecord result; - auto parse = result.DeserializeFromProto(proto, columnInfo); + auto parse = result.DeserializeFromProto(proto); if (!parse) { return parse; } @@ -166,10 +163,8 @@ class TColumnRecord { << "blob_range:" << BlobRange.ToString() << ";"; } - TColumnRecord( - const TChunkAddress& address, const std::shared_ptr& column, const TSimpleColumnInfo& columnInfo); - - TColumnRecord(const TBlobRangeLink16::TLinkId blobLinkId, const TColumnChunkLoadContext& loadContext, const TSimpleColumnInfo& columnInfo); + TColumnRecord(const TChunkAddress& address, const std::shared_ptr& column); + TColumnRecord(const TBlobRangeLink16::TLinkId blobLinkId, const TColumnChunkLoadContext& loadContext); friend IOutputStream& operator<<(IOutputStream& out, const TColumnRecord& rec) { out << '{'; diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.cpp b/ydb/core/tx/columnshard/engines/portions/constructor.cpp index 5125d60f292c..304b38a6bf87 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor.cpp @@ -100,7 +100,7 @@ ISnapshotSchema::TPtr TPortionInfoConstructor::GetSchema(const TVersionedIndex& } void TPortionInfoConstructor::LoadRecord(const TIndexInfo& indexInfo, const TColumnChunkLoadContext& loadContext) { - TColumnRecord rec(RegisterBlobId(loadContext.GetBlobRange().GetBlobId()), loadContext, indexInfo.GetColumnFeaturesVerified(loadContext.GetAddress().GetColumnId())); + TColumnRecord rec(RegisterBlobId(loadContext.GetBlobRange().GetBlobId()), loadContext); Records.push_back(std::move(rec)); if (loadContext.GetPortionMeta()) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 9f529bf682cf..e450981e3db5 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -234,7 +234,7 @@ void TPortionInfo::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortion } } -TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& info) { +TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto) { PathId = proto.GetPathId(); Portion = proto.GetPortionId(); SchemaVersion = proto.GetSchemaVersion(); @@ -258,7 +258,7 @@ TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDat } } for (auto&& i : proto.GetRecords()) { - auto parse = TColumnRecord::BuildFromProto(i, info.GetColumnFeaturesVerified(i.GetColumnId())); + auto parse = TColumnRecord::BuildFromProto(i); if (!parse) { return parse; } @@ -275,13 +275,14 @@ TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDat return TConclusionStatus::Success(); } -TConclusion TPortionInfo::BuildFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& info) { +TConclusion TPortionInfo::BuildFromProto( + const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& indexInfo) { TPortionMetaConstructor constructor; - if (!constructor.LoadMetadata(proto.GetMeta(), info)) { + if (!constructor.LoadMetadata(proto.GetMeta(), indexInfo)) { return TConclusionStatus::Fail("cannot parse meta"); } TPortionInfo result(constructor.Build()); - auto parse = result.DeserializeFromProto(proto, info); + auto parse = result.DeserializeFromProto(proto); if (!parse) { return parse; } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index bd3bfe7c4138..02b0fdd7ec63 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -88,7 +88,7 @@ class TPortionInfo { YDB_READONLY_DEF(std::vector, Indexes); YDB_READONLY(TRuntimeFeatures, RuntimeFeatures, 0); std::vector BlobIds; - TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& info); + TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto); template static void CheckChunksOrder(const std::vector& chunks) { @@ -294,7 +294,7 @@ class TPortionInfo { return defaultTierName; } - static TConclusion BuildFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& info); + static TConclusion BuildFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto, const TIndexInfo& indexInfo); void SerializeToProto(NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const; std::vector BuildPages() const; From 37c7266545b64ea797ba37c8698b7f82164f97ad Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 11:39:58 +0300 Subject: [PATCH 04/12] fix --- ydb/core/tx/columnshard/engines/storage/chunks/column.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/storage/chunks/column.h b/ydb/core/tx/columnshard/engines/storage/chunks/column.h index 9de818c49fb6..55aa9481ad61 100644 --- a/ydb/core/tx/columnshard/engines/storage/chunks/column.h +++ b/ydb/core/tx/columnshard/engines/storage/chunks/column.h @@ -59,7 +59,7 @@ class TChunkPreparation: public IPortionColumnChunk { TChunkPreparation(const TString& data, const std::shared_ptr& column, const TChunkAddress& address, const TSimpleColumnInfo& columnInfo) : TBase(address.GetColumnId()) , Data(data) - , Record(address, column, columnInfo) + , Record(address, column) , ColumnInfo(columnInfo) { Y_ABORT_UNLESS(column->GetRecordsCount()); First = column->GetScalar(0); From 340406b4446f5760606b8b38bf4b517f5f603e84 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 12:44:43 +0300 Subject: [PATCH 05/12] add test and fix --- ydb/core/kqp/ut/olap/statistics_ut.cpp | 57 +++++++++++++++++-- .../olap/operations/alter_store.cpp | 12 ++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/ydb/core/kqp/ut/olap/statistics_ut.cpp b/ydb/core/kqp/ut/olap/statistics_ut.cpp index ece5e454bacb..094fefbd4028 100644 --- a/ydb/core/kqp/ut/olap/statistics_ut.cpp +++ b/ydb/core/kqp/ut/olap/statistics_ut.cpp @@ -1,4 +1,5 @@ #include "helpers/typed_local.h" + #include namespace NKikimr::NKqp { @@ -14,25 +15,30 @@ Y_UNIT_TEST_SUITE(KqpOlapStatistics) { helper.CreateTestOlapTable(); auto tableClient = kikimr.GetTableClient(); { - auto alterQuery = TStringBuilder() << R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"pk_int\"}`))"; + auto alterQuery = + TStringBuilder() + << R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"pk_int\"}`))"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); } { - auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"field\"}`);"; + auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, " + "NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"field\"}`);"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_UNEQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); } { - auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"pk_int\"}`);"; + auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, " + "NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"pk_int\"}`);"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_UNEQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); } { - auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=DROP_INDEX, NAME=max_pk_int);"; + auto alterQuery = TStringBuilder() + << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=DROP_INDEX, NAME=max_pk_int);"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); @@ -40,6 +46,44 @@ Y_UNIT_TEST_SUITE(KqpOlapStatistics) { } } + Y_UNIT_TEST(StatsUsageNotPK) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + { + auto settings = TKikimrSettings().SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + Tests::NCommon::TLoggerInit(kikimr).Initialize(); + TTypedLocalHelper helper("Utf8", kikimr); + helper.CreateTestOlapTable(); + auto tableClient = kikimr.GetTableClient(); + { + auto alterQuery = TStringBuilder() << "ALTER TABLE `/Root/olapStore/olapTable` SET (TTL = Interval(\"P1D\") ON ts);"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_UNEQUAL(alterResult.GetStatus(), NYdb::EStatus::SUCCESS); + } + { + auto alterQuery = + TStringBuilder() + << R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=max_ts, TYPE=MAX, FEATURES=`{\"column_name\": \"ts\"}`))"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + { + auto alterQuery = TStringBuilder() << "ALTER TABLE `/Root/olapStore/olapTable` SET (TTL = Interval(\"P1D\") ON ts);"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(alterResult.GetStatus(), NYdb::EStatus::SUCCESS); + } + { + auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=DROP_INDEX, NAME=max_ts);"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_UNEQUAL(alterResult.GetStatus(), NYdb::EStatus::SUCCESS); + } + } + } + Y_UNIT_TEST(StatsUsageWithTTL) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); { @@ -50,7 +94,8 @@ Y_UNIT_TEST_SUITE(KqpOlapStatistics) { helper.CreateTestOlapTable(); auto tableClient = kikimr.GetTableClient(); { - auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, TYPE=MAX, NAME=max_ts, FEATURES=`{\"column_name\": \"ts\"}`);"; + auto alterQuery = TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, TYPE=MAX, " + "NAME=max_ts, FEATURES=`{\"column_name\": \"ts\"}`);"; auto session = tableClient.CreateSession().GetValueSync().GetSession(); auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); @@ -71,4 +116,4 @@ Y_UNIT_TEST_SUITE(KqpOlapStatistics) { } } -} +} // namespace NKikimr::NKqp diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp index 57f05068b162..1f4c83a40101 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter_store.cpp @@ -526,6 +526,18 @@ class TAlterOlapStore: public TSubOperation { return result; } + for (auto&& tPathId: alterData->ColumnTables) { + auto table = context.SS->ColumnTables.GetVerifiedPtr(tPathId); + if (!table->Description.HasTtlSettings()) { + continue; + } + auto it = alterData->SchemaPresets.find(table->Description.GetSchemaPresetId()); + AFL_VERIFY(it != alterData->SchemaPresets.end())("preset_info", table->Description.DebugString()); + if (!it->second.ValidateTtlSettings(table->Description.GetTtlSettings(), errors)) { + return result; + } + } + if (!AppData()->FeatureFlags.GetEnableSparsedColumns()) { for (auto& [_, preset]: alterData->SchemaPresets) { for (auto& [_, column]: preset.GetColumns().GetColumns()) { From bfd05e74f8b9568560004f501545826c4d522072 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 13:24:07 +0300 Subject: [PATCH 06/12] fix --- ydb/core/tx/schemeshard/olap/schema/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/core/tx/schemeshard/olap/schema/ya.make b/ydb/core/tx/schemeshard/olap/schema/ya.make index 76b2d2d1c801..32d9eeb6fa1e 100644 --- a/ydb/core/tx/schemeshard/olap/schema/ya.make +++ b/ydb/core/tx/schemeshard/olap/schema/ya.make @@ -9,6 +9,7 @@ PEERDIR( ydb/core/tx/schemeshard/olap/columns ydb/core/tx/schemeshard/olap/indexes ydb/core/tx/schemeshard/olap/options + ydb/core/tx/columnshard/engines/storage/indexes/max ydb/core/tx/schemeshard/common ) From 7e9c97b59ebfb7e71632271d3c19098d4622d5d7 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 13:39:03 +0300 Subject: [PATCH 07/12] correction --- ydb/core/tx/columnshard/engines/storage/indexes/max/meta.cpp | 1 - ydb/core/tx/columnshard/engines/storage/indexes/max/meta.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.cpp index b672f278e017..ac004a9ebabe 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.h b/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.h index ef58ede92956..8d760e184283 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.h +++ b/ydb/core/tx/columnshard/engines/storage/indexes/max/meta.h @@ -1,5 +1,6 @@ #pragma once #include + namespace NKikimr::NOlap::NIndexes::NMax { class TIndexMeta: public TIndexByColumns { From fd73f0507bc7faf5eeff8f76a2dbdfcaeb0b62a6 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 13:43:25 +0300 Subject: [PATCH 08/12] fix --- .../tx/schemeshard/olap/schema/schema.cpp | 94 +------------------ ydb/core/tx/schemeshard/olap/schema/ya.make | 1 - .../tx/schemeshard/olap/ttl/validator.cpp | 92 ++++++++++++++++++ ydb/core/tx/schemeshard/olap/ttl/validator.h | 15 +++ ydb/core/tx/schemeshard/olap/ttl/ya.make | 2 + 5 files changed, 111 insertions(+), 93 deletions(-) create mode 100644 ydb/core/tx/schemeshard/olap/ttl/validator.cpp create mode 100644 ydb/core/tx/schemeshard/olap/ttl/validator.h diff --git a/ydb/core/tx/schemeshard/olap/schema/schema.cpp b/ydb/core/tx/schemeshard/olap/schema/schema.cpp index bfe1add31b65..d491b7948d02 100644 --- a/ydb/core/tx/schemeshard/olap/schema/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/schema/schema.cpp @@ -1,100 +1,10 @@ #include "schema.h" #include #include +#include namespace NKikimr::NSchemeShard { -namespace { -static inline bool IsDropped(const TOlapColumnsDescription::TColumn& col) { - Y_UNUSED(col); - return false; -} - -static inline NScheme::TTypeInfo GetType(const TOlapColumnsDescription::TColumn& col) { - return col.GetType(); -} - -} - -static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, - const THashMap& sourceColumns, - const THashMap& alterColumns, - const THashMap& colName2Id, - IErrorCollector& errors) { - const TString colName = ttl.GetColumnName(); - - auto it = colName2Id.find(colName); - if (it == colName2Id.end()) { - errors.AddError(Sprintf("Cannot enable TTL on unknown column: '%s'", colName.data())); - return false; - } - - const TOlapColumnsDescription::TColumn* column = nullptr; - const ui32 colId = it->second; - if (alterColumns.contains(colId)) { - column = &alterColumns.at(colId); - } else if (sourceColumns.contains(colId)) { - column = &sourceColumns.at(colId); - } else { - Y_ABORT_UNLESS("Unknown column"); - } - - if (IsDropped(*column)) { - errors.AddError(Sprintf("Cannot enable TTL on dropped column: '%s'", colName.data())); - return false; - } - - if (ttl.HasExpireAfterBytes()) { - errors.AddError("TTL with eviction by size is not supported yet"); - return false; - } - - if (!ttl.HasExpireAfterSeconds()) { - errors.AddError("TTL without eviction time"); - return false; - } - - auto unit = ttl.GetColumnUnit(); - - const auto& columnType = GetType(*column); - switch (columnType.GetTypeId()) { - case NScheme::NTypeIds::DyNumber: - case NScheme::NTypeIds::Pg: - errors.AddError("Unsupported column type for TTL in column tables"); - return false; - default: - break; - } - - TString errStr; - if (!NValidation::TTTLValidator::ValidateUnit(columnType, unit, errStr)) { - errors.AddError(errStr); - return false; - } - { - bool correct = false; - if (column->GetKeyOrder() && *column->GetKeyOrder() == 0) { - correct = true; - } else { - for (auto&& [_, i] : indexes.GetIndexes()) { - auto meta = i.GetIndexMeta().GetObjectPtrOptionalAs(); - if (!meta) { - continue; - } else if (meta->GetColumnId() == column->GetId()) { - correct = true; - break; - } - } - } - if (!correct) { - errors.AddError("Haven't MAX-index for TTL column and TTL column is not first column in primary key"); - return false; - } - } - - return true; -} - bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycle& ttl, IErrorCollector& errors) const { using TTtlProto = NKikimrSchemeOp::TColumnDataLifeCycle; switch (ttl.GetStatusCase()) { @@ -105,7 +15,7 @@ bool TOlapSchema::ValidateTtlSettings(const NKikimrSchemeOp::TColumnDataLifeCycl errors.AddError("Incorrect ttl column - not found in scheme"); return false; } - return ValidateColumnTableTtl(ttl.GetEnabled(), Indexes, {}, Columns.GetColumns(), Columns.GetColumnsByName(), errors); + return TTTLValidator::ValidateColumnTableTtl(ttl.GetEnabled(), Indexes, {}, Columns.GetColumns(), Columns.GetColumnsByName(), errors); } case TTtlProto::kDisabled: default: diff --git a/ydb/core/tx/schemeshard/olap/schema/ya.make b/ydb/core/tx/schemeshard/olap/schema/ya.make index 32d9eeb6fa1e..76b2d2d1c801 100644 --- a/ydb/core/tx/schemeshard/olap/schema/ya.make +++ b/ydb/core/tx/schemeshard/olap/schema/ya.make @@ -9,7 +9,6 @@ PEERDIR( ydb/core/tx/schemeshard/olap/columns ydb/core/tx/schemeshard/olap/indexes ydb/core/tx/schemeshard/olap/options - ydb/core/tx/columnshard/engines/storage/indexes/max ydb/core/tx/schemeshard/common ) diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp new file mode 100644 index 000000000000..71a194b7b032 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -0,0 +1,92 @@ +#include "validator.h" + +namespace NKikimr::NSchemeShard { + +namespace { +static inline bool IsDropped(const TOlapColumnsDescription::TColumn& col) { + Y_UNUSED(col); + return false; +} + +static inline NScheme::TTypeInfo GetType(const TOlapColumnsDescription::TColumn& col) { + return col.GetType(); +} + +} + +bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, const THashMap& sourceColumns, const THashMap& alterColumns, const THashMap& colName2Id, IErrorCollector& errors) { + const TString colName = ttl.GetColumnName(); + + auto it = colName2Id.find(colName); + if (it == colName2Id.end()) { + errors.AddError(Sprintf("Cannot enable TTL on unknown column: '%s'", colName.data())); + return false; + } + + const TOlapColumnsDescription::TColumn* column = nullptr; + const ui32 colId = it->second; + if (alterColumns.contains(colId)) { + column = &alterColumns.at(colId); + } else if (sourceColumns.contains(colId)) { + column = &sourceColumns.at(colId); + } else { + Y_ABORT_UNLESS("Unknown column"); + } + + if (IsDropped(*column)) { + errors.AddError(Sprintf("Cannot enable TTL on dropped column: '%s'", colName.data())); + return false; + } + + if (ttl.HasExpireAfterBytes()) { + errors.AddError("TTL with eviction by size is not supported yet"); + return false; + } + + if (!ttl.HasExpireAfterSeconds()) { + errors.AddError("TTL without eviction time"); + return false; + } + + auto unit = ttl.GetColumnUnit(); + + const auto& columnType = GetType(*column); + switch (columnType.GetTypeId()) { + case NScheme::NTypeIds::DyNumber: + case NScheme::NTypeIds::Pg: + errors.AddError("Unsupported column type for TTL in column tables"); + return false; + default: + break; + } + + TString errStr; + if (!NValidation::TTTLValidator::ValidateUnit(columnType, unit, errStr)) { + errors.AddError(errStr); + return false; + } + { + bool correct = false; + if (column->GetKeyOrder() && *column->GetKeyOrder() == 0) { + correct = true; + } else { + for (auto&& [_, i] : indexes.GetIndexes()) { + auto meta = i.GetIndexMeta().GetObjectPtrOptionalAs(); + if (!meta) { + continue; + } else if (meta->GetColumnId() == column->GetId()) { + correct = true; + break; + } + } + } + if (!correct) { + errors.AddError("Haven't MAX-index for TTL column and TTL column is not first column in primary key"); + return false; + } + } + + return true; +} + +} diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.h b/ydb/core/tx/schemeshard/olap/ttl/validator.h new file mode 100644 index 000000000000..72b9a975e835 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.h @@ -0,0 +1,15 @@ +#pragma once +#include +#include + +namespace NKikimr::NSchemeShard { + class TTTLValidator { + public: + static bool ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLifeCycle::TTtl& ttl, const TOlapIndexesDescription& indexes, + const THashMap& sourceColumns, + const THashMap& alterColumns, + const THashMap& colName2Id, + IErrorCollector& errors); + + }; +} diff --git a/ydb/core/tx/schemeshard/olap/ttl/ya.make b/ydb/core/tx/schemeshard/olap/ttl/ya.make index 8aea246ebddf..0c54465bf9dc 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/ya.make +++ b/ydb/core/tx/schemeshard/olap/ttl/ya.make @@ -3,11 +3,13 @@ LIBRARY() SRCS( schema.cpp update.cpp + validator.cpp ) PEERDIR( ydb/core/base ydb/core/protos + ydb/core/tx/columnshard/engines/storage/indexes/max ) YQL_LAST_ABI_VERSION() From 8654a254c5e5e61341532e9e1d11a70def5003a2 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 13:50:40 +0300 Subject: [PATCH 09/12] fix --- ydb/core/tx/schemeshard/olap/schema/schema.cpp | 1 - ydb/core/tx/schemeshard/olap/ttl/validator.cpp | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/schemeshard/olap/schema/schema.cpp b/ydb/core/tx/schemeshard/olap/schema/schema.cpp index d491b7948d02..10be48bb334a 100644 --- a/ydb/core/tx/schemeshard/olap/schema/schema.cpp +++ b/ydb/core/tx/schemeshard/olap/schema/schema.cpp @@ -1,6 +1,5 @@ #include "schema.h" #include -#include #include namespace NKikimr::NSchemeShard { diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp index 71a194b7b032..68c9ab466034 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -1,4 +1,6 @@ #include "validator.h" +#include +#include namespace NKikimr::NSchemeShard { From 6837248577c5e550326d5905340660ec27e441f3 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 17:11:57 +0300 Subject: [PATCH 10/12] fix --- ydb/core/tx/schemeshard/olap/ttl/validator.cpp | 7 ++----- ydb/core/tx/schemeshard/olap/ttl/ya.make | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp index 68c9ab466034..8cae2a0b4fdb 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/validator.cpp +++ b/ydb/core/tx/schemeshard/olap/ttl/validator.cpp @@ -1,6 +1,5 @@ #include "validator.h" #include -#include namespace NKikimr::NSchemeShard { @@ -73,10 +72,8 @@ bool TTTLValidator::ValidateColumnTableTtl(const NKikimrSchemeOp::TColumnDataLif correct = true; } else { for (auto&& [_, i] : indexes.GetIndexes()) { - auto meta = i.GetIndexMeta().GetObjectPtrOptionalAs(); - if (!meta) { - continue; - } else if (meta->GetColumnId() == column->GetId()) { + auto proto = i.GetIndexMeta().SerializeToProto(); + if (proto.HasMaxIndex() && proto.GetMaxIndex().GetColumnId() == column->GetId()) { correct = true; break; } diff --git a/ydb/core/tx/schemeshard/olap/ttl/ya.make b/ydb/core/tx/schemeshard/olap/ttl/ya.make index 0c54465bf9dc..f6c57de62a9f 100644 --- a/ydb/core/tx/schemeshard/olap/ttl/ya.make +++ b/ydb/core/tx/schemeshard/olap/ttl/ya.make @@ -9,7 +9,6 @@ SRCS( PEERDIR( ydb/core/base ydb/core/protos - ydb/core/tx/columnshard/engines/storage/indexes/max ) YQL_LAST_ABI_VERSION() From 742a2db914bb4e17c223d6613ee3026983c52c8b Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 20:25:15 +0300 Subject: [PATCH 11/12] fix tests --- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 40 ++++++++++++++++++- .../storage/actualizer/tiering/tiering.cpp | 7 ++-- ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp | 20 ++-------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index a32d75fc45df..216c0bba67a2 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5082,6 +5082,24 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + { + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + ALTER OBJECT `)" << tableName << R"(` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, + NAME=max_value1, TYPE=MAX, FEATURES=`{\"column_name\": \"Value1\"}`))"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + auto query2 = TStringBuilder() << R"( + --!syntax_v1 + ALTER OBJECT `)" << tableName << R"(` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, + NAME=max_value2, TYPE=MAX, FEATURES=`{\"column_name\": \"Value2\"}`))"; + result = session.ExecuteSchemeQuery(query2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + auto query2 = TStringBuilder() << R"( --!syntax_v1 ALTER TABLE `)" << tableName << R"(` SET(TTL = Interval("P1D") ON Key);)"; @@ -7654,7 +7672,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { }; TTestHelper::TColumnTable testTable; - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id", "id_second"}).SetSharding({"id"}).SetSchema(schema).SetTTL("created_at", "Interval(\"PT1H\")"); + testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"created_at", "id_second"}).SetSharding({"created_at"}).SetSchema(schema).SetTTL("created_at", "Interval(\"PT1H\")"); testHelper.CreateTable(testTable); { @@ -7716,7 +7734,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { }; TTestHelper::TColumnTable testTable; - testTable.SetName(tableName).SetPrimaryKey({"id", "id_second"}).SetSharding({"id"}).SetSchema(schema).SetTTL("created_at", "Interval(\"PT1H\")"); + testTable.SetName(tableName).SetPrimaryKey({"created_at", "id_second"}).SetSharding({"created_at"}).SetSchema(schema).SetTTL("created_at", "Interval(\"PT1H\")"); testHelper.CreateTable(testTable); testHelper.CreateTier("tier1"); @@ -7755,6 +7773,16 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id", "id_second"}).SetSharding({"id"}).SetSchema(schema); testHelper.CreateTable(testTable); + { + auto alterQuery = TStringBuilder() << R"( + --!syntax_v1 + ALTER OBJECT `)" << testTable.GetName() << R"(` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, + NAME=max_value1, TYPE=MAX, FEATURES=`{\"column_name\": \"created_at\"}`))"; + auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); + + } + { auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`SET (TTL = Interval(\"PT1H\") ON created_at);"; auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); @@ -8412,6 +8440,14 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TColumnTable testTable; testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema); testHelper.CreateTable(testTable); + { + auto alterQuery = TStringBuilder() << R"( + --!syntax_v1 + ALTER OBJECT `)" << testTable.GetName() << R"(` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, + NAME=max_pk_int, TYPE=MAX, FEATURES=`{\"column_name\": \"created_at\"}`))"; + auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } { auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`SET (TTL = Interval(\"PT1H\") ON created_at);"; diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index 940d36cf6a45..b137e29311b3 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -28,12 +28,13 @@ std::optional TTieringActualizer::Bu AFL_VERIFY(TieringColumnId); auto indexMeta = portionSchema->GetIndexInfo().GetIndexMetaMax(*TieringColumnId); std::shared_ptr max; - if (*TieringColumnId == portionSchema->GetIndexInfo().GetPKColumnIds().front()) { - max = NArrow::TStatusValidator::GetValid(portion.GetMeta().GetFirstLastPK().GetFirst().Column(0).GetScalar(0)); - } else if (indexMeta) { + if (indexMeta) { NYDBTest::TControllers::GetColumnShardController()->OnStatisticsUsage(NIndexes::TIndexMetaContainer(indexMeta)); const std::vector data = portion.GetIndexInplaceDataVerified(indexMeta->GetIndexId()); max = indexMeta->GetMaxScalarVerified(data, portionSchema->GetIndexInfo().GetColumnFieldVerified(*TieringColumnId)->type()); + } else if (*TieringColumnId == portionSchema->GetIndexInfo().GetPKColumnIds().front()) { + NYDBTest::TControllers::GetColumnShardController()->OnMaxValueUsage(); + max = NArrow::TStatusValidator::GetValid(portion.GetMeta().GetFirstLastPK().GetFirst().Column(0).GetScalar(0)); } else { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "no data for ttl usage (need to create index or use first pk column)"); return {}; diff --git a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp index 3e2c665eb704..206c378f4f55 100644 --- a/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp +++ b/ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp @@ -1081,8 +1081,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { Name: "%s" Schema { Columns { Name: "key" Type: "Uint64" NotNull: true } - Columns { Name: "modified_at" Type: "%s" } - KeyColumnNames: ["key"] + Columns { Name: "modified_at" Type: "%s" NotNull: true } + KeyColumnNames: ["modified_at"] } TtlSettings { Enabled { @@ -1160,10 +1160,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { Name: "TTLEnabledTable" Schema { Columns { Name: "key" Type: "Uint64" NotNull: true } - Columns { Name: "modified_at" Type: "Timestamp" } + Columns { Name: "modified_at" Type: "Timestamp" NotNull: true } Columns { Name: "saved_at" Type: "Datetime" } Columns { Name: "data" Type: "Utf8" } - KeyColumnNames: ["key"] + KeyColumnNames: ["modified_at"] } )"); env.TestWaitNotification(runtime, txId); @@ -1189,18 +1189,6 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) { env.TestWaitNotification(runtime, txId); CheckTtlSettings(runtime, OlapTtlChecker()); - TestAlterColumnTable(runtime, ++txId, "/MyRoot", R"( - Name: "TTLEnabledTable" - AlterTtlSettings { - Enabled { - ColumnName: "saved_at" - ExpireAfterSeconds: 3600 - } - } - )"); - env.TestWaitNotification(runtime, txId); - CheckTtlSettings(runtime, OlapTtlChecker("saved_at")); - TestAlterColumnTable(runtime, ++txId, "/MyRoot", R"( Name: "TTLEnabledTable" AlterTtlSettings { From 945ca8c1a8aca06203936f5619ca8a282dec986c Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 24 Oct 2024 21:12:41 +0300 Subject: [PATCH 12/12] mute unsupported --- .github/config/muted_ya.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 5c8deb65272e..d0d847284233 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -35,6 +35,7 @@ ydb/core/kqp/ut/scheme [*/*]+chunk+chunk ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad ydb/core/kqp/ut/service [*/*] chunk chunk ydb/core/kqp/ut/service [*/*]+chunk+chunk +ydb/services/ydb/ut YdbLogStore.AlterLogTable ydb/core/mind/hive/ut THiveTest.DrainWithHiveRestart ydb/core/persqueue/ut [*/*] chunk chunk ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient