From 5d533138238467016fc368982670b36450f82382 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Sun, 18 Aug 2024 18:07:29 +0000 Subject: [PATCH 1/4] Add unit-test for data eviction in tiering --- ydb/core/kqp/ut/common/columnshard.cpp | 2 +- ydb/core/kqp/ut/olap/tiering_ut.cpp | 87 ++++++++++++++++++++++++++ ydb/core/kqp/ut/olap/ya.make | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 ydb/core/kqp/ut/olap/tiering_ut.cpp diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 267c4103adef..dd9314d90643 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -22,7 +22,7 @@ namespace NKqp { } SecretableSecretKey: { Value: { - Data: "secretSecretKey" + Data: "fakeSecret" } } } diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp new file mode 100644 index 000000000000..beee2afdb05e --- /dev/null +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -0,0 +1,87 @@ +#include "helpers/get_value.h" +#include "helpers/local.h" +#include "helpers/query_executor.h" +#include "helpers/typed_local.h" +#include "helpers/writer.h" + +#include +#include +#include +#include +#include + +namespace NKikimr::NKqp { + +Y_UNIT_TEST_SUITE(KqpOlapTiering) { + Y_UNIT_TEST(Eviction) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + + TKikimrSettings runnerSettings; + runnerSettings.WithSampleTables = false; + TTestHelper testHelper(runnerSettings); + TLocalHelper localHelper(testHelper.GetKikimr()); + NYdb::NTable::TTableClient tableClient = testHelper.GetKikimr().GetTableClient(); + Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize(); + Singleton()->SetSecretKey("fakeSecret"); + + localHelper.CreateTestOlapTable(); + testHelper.CreateTier("tier1"); + const TString tiering_rule = testHelper.CreateTieringRule("tier1", "timestamp"); + + for (ui64 i = 0; i < 100; ++i) { + WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000); + } + + csController->WaitCompactions(TDuration::Seconds(5)); + + { + auto selectQuery = TString(R"( + SELECT + TierName + FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` + WHERE Activity == 1 + GROUP BY TierName + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); + } + + testHelper.SetTiering("/Root/olapStore/olapTable", tiering_rule); + csController->WaitCompactions(TDuration::Seconds(5)); + + { + auto selectQuery = TString(R"( + SELECT + TierName + FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` + WHERE Activity == 1 + GROUP BY TierName + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); + } + + testHelper.ResetTiering("/Root/olapStore/olapTable"); + csController->WaitCompactions(TDuration::Seconds(5)); + + { + auto selectQuery = TString(R"( + SELECT + TierName + FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` + WHERE Activity == 1 + GROUP BY TierName + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); + } + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/olap/ya.make b/ydb/core/kqp/ut/olap/ya.make index 06deb96569b9..acf719f7cba2 100644 --- a/ydb/core/kqp/ut/olap/ya.make +++ b/ydb/core/kqp/ut/olap/ya.make @@ -24,6 +24,7 @@ SRCS( aggregations_ut.cpp write_ut.cpp sparsed_ut.cpp + tiering_ut.cpp ) PEERDIR( From 0b8ebd721ca5cb779a189667ba738af17c85d3ef Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Mon, 19 Aug 2024 08:16:00 +0000 Subject: [PATCH 2/4] fix style, add rawBytes --- ydb/core/kqp/ut/olap/tiering_ut.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index beee2afdb05e..5d30fc778e73 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -26,18 +26,19 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { localHelper.CreateTestOlapTable(); testHelper.CreateTier("tier1"); - const TString tiering_rule = testHelper.CreateTieringRule("tier1", "timestamp"); + const TString tieringRule = testHelper.CreateTieringRule("tier1", "timestamp"); for (ui64 i = 0; i < 100; ++i) { WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000); } - csController->WaitCompactions(TDuration::Seconds(5)); + csController->WaitActualization(TDuration::Seconds(5)); + ui64 columnRawBytes = 0; { auto selectQuery = TString(R"( SELECT - TierName + TierName, SUM(ColumnRawBytes) As RawBytes FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` WHERE Activity == 1 GROUP BY TierName @@ -46,15 +47,17 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto rows = ExecuteScanQuery(tableClient, selectQuery); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); + + columnRawBytes = GetUint64(rows[0].at("RawBytes")); } - testHelper.SetTiering("/Root/olapStore/olapTable", tiering_rule); - csController->WaitCompactions(TDuration::Seconds(5)); + testHelper.SetTiering("/Root/olapStore/olapTable", tieringRule); + csController->WaitActualization(TDuration::Seconds(5)); { auto selectQuery = TString(R"( SELECT - TierName + TierName, SUM(ColumnRawBytes) As RawBytes FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` WHERE Activity == 1 GROUP BY TierName @@ -63,6 +66,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto rows = ExecuteScanQuery(tableClient, selectQuery); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); + UNIT_ASSERT_VALUES_EQUAL(GetUint64(rows[0].at("RawBytes")), columnRawBytes); } testHelper.ResetTiering("/Root/olapStore/olapTable"); @@ -71,7 +75,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { { auto selectQuery = TString(R"( SELECT - TierName + TierName, SUM(ColumnRawBytes) As RawBytes FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` WHERE Activity == 1 GROUP BY TierName @@ -80,6 +84,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto rows = ExecuteScanQuery(tableClient, selectQuery); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); + UNIT_ASSERT_VALUES_EQUAL(GetUint64(rows[0].at("RawBytes")), columnRawBytes); } } } From 057375c2e36e0ff81a4ceb5d6fdb55431bcd25e9 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Mon, 19 Aug 2024 10:43:55 +0000 Subject: [PATCH 3/4] add logs --- ydb/core/kqp/ut/olap/tiering_ut.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index 5d30fc778e73..8c2513c81e3b 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -49,6 +49,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); columnRawBytes = GetUint64(rows[0].at("RawBytes")); + UNIT_ASSERT_GT(columnRawBytes, 0); } testHelper.SetTiering("/Root/olapStore/olapTable", tieringRule); @@ -66,7 +67,8 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto rows = ExecuteScanQuery(tableClient, selectQuery); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(rows[0].at("RawBytes")), columnRawBytes); + UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, + TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes << " after=" << rows[0].at("RawBytes")); } testHelper.ResetTiering("/Root/olapStore/olapTable"); @@ -84,7 +86,9 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto rows = ExecuteScanQuery(tableClient, selectQuery); UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); - UNIT_ASSERT_VALUES_EQUAL(GetUint64(rows[0].at("RawBytes")), columnRawBytes); + UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, + TStringBuilder() << "RawBytes changed after resetting tiering: before=" << columnRawBytes + << " after=" << rows[0].at("RawBytes")); } } } From fc8c8fa7540a726cd64a22608a0fa91c1b217d34 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Tue, 20 Aug 2024 08:50:15 +0000 Subject: [PATCH 4/4] fix build --- ydb/core/kqp/ut/olap/tiering_ut.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index 8c2513c81e3b..b9e3b9f79da1 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -68,7 +68,8 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, - TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes << " after=" << rows[0].at("RawBytes")); + TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes + << " after=" << GetUint64(rows[0].at("RawBytes"))); } testHelper.ResetTiering("/Root/olapStore/olapTable");