From 456c37b57c8c7c3559a2a4e51f3b43335651a1cb Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Tue, 17 Sep 2024 11:03:46 +0300 Subject: [PATCH 1/2] fix addressing and additional logging --- .../engines/changes/compaction/sparsed/logic.h | 6 ++++-- ydb/library/formats/arrow/accessor/abstract/accessor.cpp | 2 +- ydb/library/formats/arrow/accessor/abstract/accessor.h | 9 +++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h b/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h index 35a3649d61f3..f9a96d5bf974 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h @@ -58,7 +58,9 @@ class TSparsedMerger: public IColumnMerger { void InitArrays(const ui32 position) { AFL_VERIFY(!ChunkAddress || ChunkFinishPosition <= position); - ChunkAddress = CurrentChunkedArray->GetChunk(ChunkAddress, position); + AFL_VERIFY(CurrentOwnedArray->GetAddress().GetGlobalStartPosition() <= position)("pos", position)( + "global", CurrentOwnedArray->GetAddress().GetGlobalStartPosition()); + ChunkAddress = CurrentChunkedArray->GetChunk(ChunkAddress, position - CurrentOwnedArray->GetAddress().GetGlobalStartPosition()); AFL_VERIFY(ChunkAddress); ChunkStartPosition = CurrentOwnedArray->GetAddress().GetGlobalStartPosition() + ChunkAddress->GetAddress().GetGlobalStartPosition(); ChunkFinishPosition = @@ -83,7 +85,7 @@ class TSparsedMerger: public IColumnMerger { return {}; } if (ChunkFinishPosition <= currentIndex) { - InitArrays(currentGlobalPosition); + InitArrays(currentIndex); continue; } for (; currentIndex < ChunkFinishPosition; ++currentIndex) { diff --git a/ydb/library/formats/arrow/accessor/abstract/accessor.cpp b/ydb/library/formats/arrow/accessor/abstract/accessor.cpp index 9d4c33efa42e..c58ed0e28373 100644 --- a/ydb/library/formats/arrow/accessor/abstract/accessor.cpp +++ b/ydb/library/formats/arrow/accessor/abstract/accessor.cpp @@ -45,7 +45,7 @@ std::shared_ptr IChunkedArray::Slice(const ui32 offset, con NKikimr::NArrow::NAccessor::IChunkedArray::TFullDataAddress IChunkedArray::GetChunk( const std::optional& chunkCurrent, const ui64 position) const { - AFL_VERIFY(position < GetRecordsCount()); + AFL_VERIFY(position < GetRecordsCount())("pos", position)("records", GetRecordsCount())("current", chunkCurrent ? chunkCurrent->DebugString() : Default()); std::optional address; if (IsDataOwner()) { diff --git a/ydb/library/formats/arrow/accessor/abstract/accessor.h b/ydb/library/formats/arrow/accessor/abstract/accessor.h index c8e7979618dd..934ca2fddea4 100644 --- a/ydb/library/formats/arrow/accessor/abstract/accessor.h +++ b/ydb/library/formats/arrow/accessor/abstract/accessor.h @@ -104,8 +104,13 @@ class IChunkedArray { } TString DebugString() const { - return TStringBuilder() << "start=" << GlobalStartPosition << ";finish=" << GlobalFinishPosition - << ";addresses_count=" << Addresses.size() << ";"; + TStringBuilder sb; + sb << "start=" << GlobalStartPosition << ";finish=" << GlobalFinishPosition + << ";addresses_count=" << Addresses.size() << ";"; + for (auto&& i : Addresses) { + sb << "addresses=" << i.DebugString() << ";"; + } + return sb; } }; From b6e59a2cd89010ee4a57c036592ef86fdd710289 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Tue, 17 Sep 2024 16:33:33 +0300 Subject: [PATCH 2/2] fixes --- .../engines/changes/compaction/sparsed/logic.cpp | 4 +++- .../columnshard/engines/changes/compaction/sparsed/logic.h | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.cpp index fb682bf79e5b..d2c4e14f0664 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.cpp @@ -134,7 +134,7 @@ bool TSparsedMerger::TPlainChunkCursor::AddIndexTo(const ui32 index, TWriter& wr bool TSparsedMerger::TSparsedChunkCursor::AddIndexTo(const ui32 index, TWriter& writer) { AFL_VERIFY(ChunkStartGlobalPosition <= index); - AFL_VERIFY(index == NextGlobalPosition); + AFL_VERIFY(index == NextGlobalPosition)("index", index)("next", NextGlobalPosition); writer.AddRealData(Chunk->GetColValue(), NextLocalPosition); return true; } @@ -163,7 +163,9 @@ void TSparsedMerger::TCursor::InitArrays(const ui32 position) { PlainCursor = make_shared(CurrentOwnedArray->GetArray(), &*CurrentOwnedArray); SparsedCursor = nullptr; } + AFL_VERIFY(CurrentOwnedArray->GetAddress().GetGlobalStartPosition() <= position); FinishGlobalPosition = CurrentOwnedArray->GetAddress().GetGlobalStartPosition() + CurrentOwnedArray->GetArray()->GetRecordsCount(); + AFL_VERIFY(position < FinishGlobalPosition); } } // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h b/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h index f9a96d5bf974..9fc64606a092 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.h @@ -65,6 +65,8 @@ class TSparsedMerger: public IColumnMerger { ChunkStartPosition = CurrentOwnedArray->GetAddress().GetGlobalStartPosition() + ChunkAddress->GetAddress().GetGlobalStartPosition(); ChunkFinishPosition = CurrentOwnedArray->GetAddress().GetGlobalStartPosition() + ChunkAddress->GetAddress().GetGlobalFinishPosition(); + AFL_VERIFY(position < ChunkFinishPosition)("finish", ChunkFinishPosition)("pos", position); + AFL_VERIFY(ChunkStartPosition <= position)("start", ChunkStartPosition)("pos", position); } public: @@ -78,7 +80,8 @@ class TSparsedMerger: public IColumnMerger { } bool AddIndexTo(const ui32 index, TWriter& writer); std::optional MoveToSignificant(const ui32 currentGlobalPosition, const TColumnMergeContext& context) { - AFL_VERIFY(ChunkStartPosition <= currentGlobalPosition); + AFL_VERIFY(ChunkStartPosition <= currentGlobalPosition)("start", ChunkStartPosition)("pos", currentGlobalPosition)( + "global_start", CurrentOwnedArray->GetAddress().GetGlobalStartPosition()); ui32 currentIndex = currentGlobalPosition; while (true) { if (CurrentOwnedArray->GetAddress().GetGlobalFinishPosition() <= currentIndex) { @@ -192,6 +195,7 @@ class TSparsedMerger: public IColumnMerger { if (FinishGlobalPosition == Array->GetRecordsCount()) { return FinishGlobalPosition; } else { + currentPosition = FinishGlobalPosition; InitArrays(FinishGlobalPosition); } }