From 390afc66a3aa6d140f3b71ed3f13d179eabfd321 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Fri, 26 Jul 2024 10:16:10 +0300 Subject: [PATCH] add stats test for internal column table from tables store (#6684) --- .../scheme/versions/abstract_scheme.cpp | 3 +- .../operations/batch_builder/builder.cpp | 1 + .../test_helper/columnshard_ut_common.cpp | 16 ++++--- .../test_helper/columnshard_ut_common.h | 6 ++- ydb/core/tx/columnshard/test_helper/helper.h | 1 + .../schemeshard_path_describer.cpp | 2 + ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 46 ++++++++++++++----- 7 files changed, 54 insertions(+), 21 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp index d0fd1151d223..b33a012d8222 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp @@ -87,7 +87,8 @@ TConclusion> ISnapshotSchema::PrepareForModi for (auto&& i : batch->schema()->fields()) { AFL_VERIFY(GetIndexInfo().HasColumnName(i->name())); if (!dstSchema->GetFieldByName(i->name())->Equals(i)) { - return TConclusionStatus::Fail("not equal field types for column '" + i->name() + "'"); + return TConclusionStatus::Fail("not equal field types for column '" + i->name() + "': " + i->ToString() + " vs " + + dstSchema->GetFieldByName(i->name())->ToString()); } if (GetIndexInfo().IsNullableVerified(i->name())) { continue; diff --git a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp index 232d3b556e95..88d61a5f3934 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp @@ -11,6 +11,7 @@ namespace NKikimr::NOlap { void TBuildBatchesTask::ReplyError(const TString& message) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("problem", "cannot build batch for insert")("reason", message)("data", WriteData.GetWriteMeta().GetLongTxIdOptional()); auto writeDataPtr = std::make_shared(std::move(WriteData)); TWritingBuffer buffer(writeDataPtr->GetBlobsAction(), { std::make_shared(*writeDataPtr) }); auto result = NColumnShard::TEvPrivate::TEvWriteBlobsResult::Error(NKikimrProto::EReplyStatus::CORRUPTED, std::move(buffer), message); diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index 4f8cfe4ce4b9..bad9129b80be 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -125,22 +125,26 @@ bool WriteDataImpl(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shar } bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector& ydbSchema, std::vector* writeIds, const NEvWrite::EModificationType mType) { + const std::vector& ydbSchema, std::vector* writeIds, const NEvWrite::EModificationType mType, + const std::set& notNullColumns) { NLongTxService::TLongTxId longTxId; UNIT_ASSERT(longTxId.ParseString("ydb://long-tx/01ezvvxjdk2hd4vdgjs68knvp8?node_id=1")); - return WriteDataImpl(runtime, sender, shardId, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), writeIds, mType); - + return WriteDataImpl( + runtime, sender, shardId, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema, notNullColumns), writeIds, mType); } bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector& ydbSchema, bool waitResult, std::vector* writeIds, const NEvWrite::EModificationType mType) { + const std::vector& ydbSchema, bool waitResult, std::vector* writeIds, + const NEvWrite::EModificationType mType, const std::set& notNullColumns) { NLongTxService::TLongTxId longTxId; UNIT_ASSERT(longTxId.ParseString("ydb://long-tx/01ezvvxjdk2hd4vdgjs68knvp8?node_id=1")); if (writeIds) { - return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), writeIds, mType); + return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data, + NArrow::MakeArrowSchema(ydbSchema, notNullColumns), writeIds, mType); } std::vector ids; - return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data, NArrow::MakeArrowSchema(ydbSchema), waitResult ? &ids : nullptr, mType); + return WriteDataImpl(runtime, sender, TTestTxConfig::TxTablet0, tableId, longTxId, writeId, data, + NArrow::MakeArrowSchema(ydbSchema, notNullColumns), waitResult ? &ids : nullptr, mType); } std::optional WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 4306babbb1ed..b24b05e4ffdd 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -406,10 +406,12 @@ void PlanSchemaTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot void PlanWriteTx(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot snap, bool waitResult = true); bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector& ydbSchema, std::vector* writeIds, const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert); + const std::vector& ydbSchema, std::vector* writeIds, + const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert, const std::set& notNullColumns = {}); bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const ui64 tableId, const TString& data, - const std::vector& ydbSchema, bool waitResult = true, std::vector* writeIds = nullptr, const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert); + const std::vector& ydbSchema, bool waitResult = true, std::vector* writeIds = nullptr, + const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert, const std::set& notNullColumns = {}); std::optional WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, ui64 tableId, const ui64 writePartId, const TString& data, diff --git a/ydb/core/tx/columnshard/test_helper/helper.h b/ydb/core/tx/columnshard/test_helper/helper.h index cf71657cf97c..956d962f0bd1 100644 --- a/ydb/core/tx/columnshard/test_helper/helper.h +++ b/ydb/core/tx/columnshard/test_helper/helper.h @@ -51,6 +51,7 @@ class TTestColumn { YDB_ACCESSOR_DEF(TString, Name); YDB_ACCESSOR_DEF(NScheme::TTypeInfo, Type); YDB_ACCESSOR_DEF(TString, StorageId); + public: explicit TTestColumn(const TString& name, const NScheme::TTypeInfo& type) : Name(name) diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 34ecf178b3bf..c0dc45860f5e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -480,6 +480,8 @@ void TPathDescriber::DescribeColumnTable(TPathId pathId, TPathElement::TPtr path } if (tableInfo->GetStats().TableStats.contains(pathId)) { FillTableStats(*pathDescription, tableInfo->GetStats().TableStats.at(pathId)); + } else { + FillTableStats(*pathDescription, TPartitionStats()); } } } diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index c53fdeb58425..dc12e13f7b31 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -634,17 +635,17 @@ Y_UNIT_TEST_SUITE(TOlap) { env.TestWaitNotification(runtime, txId); } - // TODO: AlterTiers - // negatives for store: disallow alters - // negatives for table: wrong tiers count, wrong tiers, wrong eviction column, wrong eviction values, - // different TTL columns in tiers -#if 0 Y_UNIT_TEST(StoreStats) { TTestBasicRuntime runtime; TTestEnv env(runtime); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); runtime.UpdateCurrentTime(TInstant::Now() - TDuration::Seconds(600)); + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); + // disable stats batching auto& appData = runtime.GetAppData(); appData.SchemeShardConfig.SetStatsBatchTimeoutMs(0); @@ -690,6 +691,16 @@ Y_UNIT_TEST_SUITE(TOlap) { UNIT_ASSERT(shardId); UNIT_ASSERT(pathId); UNIT_ASSERT(planStep); + { + auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore/ColumnTable", true, true); + Cerr << description.DebugString() << Endl; + auto& tabletStats = description.GetPathDescription().GetTableStats(); + + UNIT_ASSERT(description.GetPathDescription().HasTableStats()); + UNIT_ASSERT_EQUAL(tabletStats.GetRowCount(), 0); + UNIT_ASSERT_EQUAL(tabletStats.GetDataSize(), 0); + } + ui32 rowsInBatch = 100000; @@ -702,7 +713,7 @@ Y_UNIT_TEST_SUITE(TOlap) { TSet txIds; for (ui32 i = 0; i < 10; ++i) { std::vector writeIds; - NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds); + NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds, NEvWrite::EModificationType::Upsert, { "timestamp" }); NTxUT::ProposeCommit(runtime, sender, shardId, ++txId, writeIds); txIds.insert(txId); } @@ -714,16 +725,28 @@ Y_UNIT_TEST_SUITE(TOlap) { // trigger periodic stats at shard (after timeout) std::vector writeIds; - NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds); + NTxUT::WriteData(runtime, sender, shardId, ++writeId, pathId, data, defaultYdbSchema, &writeIds, NEvWrite::EModificationType::Upsert, { "timestamp" }); NTxUT::ProposeCommit(runtime, sender, shardId, ++txId, writeIds); NTxUT::PlanCommit(runtime, sender, shardId, ++planStep, {txId}); } + csController->WaitIndexation(TDuration::Seconds(5)); + { + auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore", true, true); + auto& tabletStats = description.GetPathDescription().GetTableStats(); - auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore", true, true); - auto& tabletStats = description.GetPathDescription().GetTableStats(); + UNIT_ASSERT_GT(tabletStats.GetRowCount(), 0); + UNIT_ASSERT_GT(tabletStats.GetDataSize(), 0); + } + + { + auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore/ColumnTable", true, true); + Cerr << description.DebugString() << Endl; + auto& tabletStats = description.GetPathDescription().GetTableStats(); + + UNIT_ASSERT_GT(tabletStats.GetRowCount(), 0); + UNIT_ASSERT_GT(tabletStats.GetDataSize(), 0); + } - UNIT_ASSERT_GT(tabletStats.GetRowCount(), 0); - UNIT_ASSERT_GT(tabletStats.GetDataSize(), 0); #if 0 TestDropColumnTable(runtime, ++txId, "/MyRoot/OlapStore", "ColumnTable"); env.TestWaitNotification(runtime, txId); @@ -738,5 +761,4 @@ Y_UNIT_TEST_SUITE(TOlap) { TestLsPathId(runtime, 2, NLs::PathStringEqual("")); #endif } -#endif }