Skip to content

Commit

Permalink
Transfer scheme history to new partitions (ydb-platform#9959)
Browse files Browse the repository at this point in the history 
Conflicts:
	.github/config/muted_ya.txt
  • Loading branch information
@fexolm @zverevgeny
fexolm authored and zverevgeny committed Jan 1, 2025
1 parent 6862456 commit 84e4a06
Show file tree
Hide file tree
Showing 33 changed files with 508 additions and 202 deletions.
42 changes: 16 additions & 26 deletions .github/config/muted_ya.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,22 @@ ydb/core/blobstorage/dsproxy/ut_fat TBlobStorageProxyTest.TestBatchedPutRequestD
ydb/core/client/ut TClientTest.PromoteFollower
ydb/core/client/ut TClientTest.ReadFromFollower
ydb/core/client/ut TFlatTest.AutoSplitMergeQueue
ydb/core/cms/ut_sentinel_unstable *
ydb/core/external_sources *
ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
ydb/core/kqp/ut/join KqpJoinOrder.Chain65Nodes
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
ydb/core/cms/ut_sentinel_unstable TSentinelUnstableTests.BSControllerCantChangeStatus
ydb/core/cms/ut_sentinel_unstable sole chunk chunk
ydb/core/cms/ut_sentinel_unstable sole+chunk+chunk
ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadHuge
ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadSmall
ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteHuge
ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteSmall
ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.Select
ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWrite
ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertViaLegacyScripting-Streaming
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_1_clean_with_restarts
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingConsistency64
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.TableReshardingModuloN
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.UpsertWhileSplitTest
ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
ydb/core/kqp/ut/olap KqpOlapSysView.StatsSysViewBytesDictStatActualization
ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_SumL_GroupL_OrderL
Expand All @@ -34,24 +43,6 @@ ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
<<<<<<< HEAD
ydb/core/kqp/ut/service [38/50]*
ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
ydb/core/persqueue/ut [37/40] chunk chunk
ydb/core/persqueue/ut [38/40] chunk chunk
ydb/core/persqueue/ut TPQTest.*DirectRead*
ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.PartitionSplit_ManySession_AutoscaleAwareSDK
ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.PartitionSplit_ManySession_existed_AutoscaleAwareSDK
ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.PartitionSplit_ReadNotEmptyPartitions_AutoscaleAwareSDK
ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.PartitionSplit_ReadNotEmptyPartitions_BeforeAutoscaleAwareSDK
ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.ReadingAfterSplitTest_PreferedPartition_AutoscaleAwareSDK
ydb/core/persqueue/ut/ut_with_sdk [4/10] chunk chunk
ydb/core/persqueue/ut/ut_with_sdk [6/10] chunk chunk
ydb/core/persqueue/ut/ut_with_sdk [7/10] chunk chunk
ydb/core/persqueue/ut/ut_with_sdk [8/10] chunk chunk
ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
ydb/core/tx/datashard/ut_change_exchange Cdc.InitialScanDebezium
=======
ydb/core/kqp/ut/service [*/*] chunk chunk
ydb/core/kqp/ut/service [*/*]+chunk+chunk
ydb/services/ydb/ut YdbLogStore.AlterLogTable
Expand All @@ -61,7 +52,6 @@ ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithData
ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithDataAndPersistentPartitionStats
ydb/core/tx/schemeshard/ut_pq_reboots TPqGroupTestReboots.AlterWithReboots-PQConfigTransactionsAtSchemeShard-false
>>>>>>> e2ee1b2b1a (Clean max scalar (#10826))
ydb/core/tx/schemeshard/ut_restore TImportTests.ShouldSucceedOnManyTables
ydb/core/tx/schemeshard/ut_split_merge TSchemeShardSplitBySizeTest.Merge1KShards
ydb/core/tx/tiering/ut ColumnShardTiers.TTLUsage
Expand Down
77 changes: 69 additions & 8 deletions ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,9 +321,6 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {

void Execute() {
TLocalHelper(Kikimr).SetShardingMethod(ShardingType).CreateTestOlapTable("olapTable", "olapStore", 24, 4);

Tests::NCommon::TLoggerInit(Kikimr).SetComponents({ NKikimrServices::TX_COLUMNSHARD, NKikimrServices::TX_COLUMNSHARD_SCAN }, "CS").SetPriority(NActors::NLog::PRI_DEBUG).Initialize();

{
WriteTestData(Kikimr, "/Root/olapStore/olapTable", 1000000, 300000000, 10000);
WriteTestData(Kikimr, "/Root/olapStore/olapTable", 1100000, 300100000, 10000);
Expand Down Expand Up @@ -403,7 +400,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
}

Y_UNIT_TEST(TableReshardingModuloN) {
TShardingTypeTest().SetShardingType("HASH_FUNCTION_CONSISTENCY_64").Execute();
TShardingTypeTest().SetShardingType("HASH_FUNCTION_MODULO_N").Execute();
}

class TAsyncReshardingTest: public TReshardingTest {
Expand Down Expand Up @@ -435,11 +432,36 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
TReshardingTest::CheckCount(NumRows);
}

void AddManyColumns() {
auto alterQuery = TStringBuilder() << "ALTER TABLESTORE `/Root/olapStore` ";
for (int i = 0; i < 10000; i++) {
alterQuery << " ADD COLUMN col_" << i << " Int8";
if (i < 10000 - 1) {
alterQuery << ", ";
}
}

auto session = TableClient.CreateSession().GetValueSync().GetSession();
auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync();

UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString());
}

void RestartAllShards() {
for (i64 id : CSController->GetShardActualIds()) {
Kikimr.GetTestServer().GetRuntime()->Send(MakePipePerNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward(new TEvents::TEvPoisonPill(), id, false));
}
}

void ChangeSchema() {
auto alterQuery =
"ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=level, "
"`SERIALIZER.CLASS_NAME`=`ARROW_SERIALIZER`, "
"`COMPRESSION.TYPE`=`zstd`);";
const char* alterQuery;
if (HasNewCol) {
alterQuery = "ALTER TABLESTORE `/Root/olapStore` DROP COLUMN new_col";
} else {
alterQuery = "ALTER TABLESTORE `/Root/olapStore` ADD COLUMN new_col Int8";
}
HasNewCol = !HasNewCol;

auto session = TableClient.CreateSession().GetValueSync().GetSession();
auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync();

Expand All @@ -454,6 +476,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
ui64 LastPathId = 1000000;
ui64 LastTs = 300000000;
ui64 NumRows = 0;
ui64 HasNewCol = false;
};

Y_UNIT_TEST(UpsertWhileSplitTest) {
Expand Down Expand Up @@ -498,6 +521,44 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
tester.StartResharding("SPLIT");
tester.WaitResharding();

tester.RestartAllShards();

tester.CheckCount();
}

Y_UNIT_TEST(MultipleSchemaVersions) {
TAsyncReshardingTest tester;
tester.DisableCompaction();

for (int i = 0; i < 3; i++) {
tester.AddBatch(1);
tester.ChangeSchema();
}

tester.StartResharding("SPLIT");
tester.WaitResharding();

tester.RestartAllShards();

tester.CheckCount();
}

Y_UNIT_TEST(HugeSchemeHistory) {
TAsyncReshardingTest tester;
tester.DisableCompaction();

tester.AddManyColumns();

for (int i = 0; i < 100; i++) {
tester.AddBatch(1);
tester.ChangeSchema();
}

tester.StartResharding("SPLIT");
tester.WaitResharding();

tester.RestartAllShards();

tester.CheckCount();
}
}
Expand Down
1 change: 1 addition & 0 deletions ydb/core/protos/counters_columnshard.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,4 +201,5 @@ enum ETxTypes {
TXTYPE_GC_START = 34 [(TxTypeOpts) = {Name: "TxGarbageCollectionStart"}];
TXTYPE_APPLY_NORMALIZER = 35 [(TxTypeOpts) = {Name: "TxApplyNormalizer"}];
TXTYPE_START_INTERNAL_SCAN = 36 [(TxTypeOpts) = {Name: "TxStartInternalScan"}];
TXTYPE_DATA_SHARING_START_SOURCE_CURSOR = 37 [(TxTypeOpts) = {Name: "TxDataSharingStartSourceCursor"}];
}
Loading

0 comments on commit 84e4a06

Please sign in to comment.