From 292de6992b101e09bf1edba45efb83561bdac194 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Thu, 29 Aug 2024 09:26:53 +0000 Subject: [PATCH 1/2] Add unit test: drop and and column column with different type --- .github/config/muted_ya.txt | 1 + ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 60 +++++++++++++++++++ .../engines/changes/compaction/merger.cpp | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index a3c51ad0abee..2ef78a0f7979 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -12,6 +12,7 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.* ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata ydb/core/kqp/ut/olap KqpOlapBlobsSharing.* ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL +ydb/core/kqp/ut/olap KqpOlapScheme.DropColumnAfterAdd ydb/core/kqp/ut/pg KqpPg.CreateIndex ydb/core/kqp/ut/query KqpLimits.QueryReplySize ydb/core/kqp/ut/query KqpQuery.QueryTimeout diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 98517c19d739..4769d092bc42 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -7848,6 +7849,65 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); } + Y_UNIT_TEST(DropThenAddColumn) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); + csController->DisableBackground(NYDBTest::ICSController::EBackground::Compaction); + + TKikimrSettings runnerSettings; + runnerSettings.WithSampleTables = false; + TTestHelper testHelper(runnerSettings); + + TVector schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), + TTestHelper::TColumnSchema().SetName("value").SetType(NScheme::NTypeIds::Utf8), + }; + + TTestHelper::TColumnTable testTable; + testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + testHelper.CreateTable(testTable); + + { + TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); + tableInserter.AddRow().Add(1).Add("test_res_1"); + tableInserter.AddRow().Add(2).Add("test_res_2"); + testHelper.BulkUpsert(testTable, tableInserter); + } + + csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); + csController->EnableBackground(NYDBTest::ICSController::EBackground::Compaction); + csController->WaitIndexation(TDuration::Seconds(5)); + csController->WaitCompactions(TDuration::Seconds(5)); + csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); + csController->DisableBackground(NYDBTest::ICSController::EBackground::Compaction); + + { + auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "` DROP COLUMN value;"; + auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + { + auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "` ADD COLUMN value Uint64;"; + auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } + schema.back().SetType(NScheme::NTypeIds::Uint64); + + { + TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); + tableInserter.AddRow().Add(3).Add(42); + tableInserter.AddRow().Add(4).Add(43); + testHelper.BulkUpsert(testTable, tableInserter); + } + + csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); + csController->EnableBackground(NYDBTest::ICSController::EBackground::Compaction); + csController->WaitIndexation(TDuration::Seconds(5)); + csController->WaitCompactions(TDuration::Seconds(5)); + + testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest`", "[[4;#;[\"test_res_1\"]]]"); + } + Y_UNIT_TEST(DropTtlColumn) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp index 72ca7d2019ed..aad9533a97b4 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp @@ -59,7 +59,7 @@ std::vector TMerger::Execute(const std::shared for (auto&& p : Batches) { ui32 columnIdx = 0; for (auto&& i : p->GetSchema()->GetFields()) { - const std::optional columnId = resultFiltered->GetIndexInfo().GetColumnIdOptional(i->name()); + const std::optional columnId = resultFiltered->GetColumnIdOptional(i->name()); if (columnId) { auto it = columnsData.find(*columnId); if (it == columnsData.end()) { From b1bac05d1a13ad4f7bea2c8f703d800b5591882a Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Thu, 29 Aug 2024 10:49:00 +0000 Subject: [PATCH 2/2] correct muted_ya.txt --- .github/config/muted_ya.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 2ef78a0f7979..b43f05b1e198 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -12,12 +12,12 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.* ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata ydb/core/kqp/ut/olap KqpOlapBlobsSharing.* ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL -ydb/core/kqp/ut/olap KqpOlapScheme.DropColumnAfterAdd ydb/core/kqp/ut/pg KqpPg.CreateIndex ydb/core/kqp/ut/query KqpLimits.QueryReplySize ydb/core/kqp/ut/query KqpQuery.QueryTimeout ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage ydb/core/kqp/ut/scheme [*/*]* +ydb/core/kqp/ut/scheme KqpOlapScheme.DropThenAddColumn ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter