diff --git a/src/mds/schedule/copySetScheduler.cpp b/src/mds/schedule/copySetScheduler.cpp index 9a758c60a2..1ffa1e551b 100644 --- a/src/mds/schedule/copySetScheduler.cpp +++ b/src/mds/schedule/copySetScheduler.cpp @@ -32,33 +32,98 @@ namespace curve { namespace mds { namespace schedule { int CopySetScheduler::Schedule() { - LOG(INFO) << "copysetScheduler begin"; + LOG(INFO) << "schedule: copysetScheduler begin"; - int res = 0; + int oneRoundGenOp = 0; for (auto lid : topo_->GetLogicalpools()) { - res = DoCopySetSchedule(lid); + oneRoundGenOp += DoCopySetSchedule(lid); } - return res; + + LOG(INFO) << "schedule: copysetScheduler end, generate operator num " + << oneRoundGenOp; + return oneRoundGenOp; } -int CopySetScheduler::DoCopySetSchedule(PoolIdType lid) { - // 1. collect the chunkserver list and copyset list of the cluster, then - // collect copyset on every online chunkserver - auto copysetList = topo_->GetCopySetInfosInLogicalPool(lid); - auto chunkserverList = topo_->GetChunkServersInLogicalPool(lid); - std::map> distribute; - SchedulerHelper::CopySetDistributionInOnlineChunkServer( - copysetList, chunkserverList, &distribute); - if (distribute.empty()) { - LOG(WARNING) << "no not-retired chunkserver in topology"; - return UNINTIALIZE_ID; +int CopySetScheduler::PenddingCopySetSchedule(const std::map> &distribute) { + int oneRoundGenOp = 0; + // for every chunkserver, find one copy set to migrate out + for (auto it = distribute.begin(); it != distribute.end(); it++) { + ChunkServerIdType source = it->first; + int copysetNum = it->second.size(); + if (copysetNum == 0) { + continue; + } + + // find one copy set to migrate out from source chunkserver + for (auto info : it->second) { + // does not meet the basic conditions + if (!CopySetSatisfiyBasicMigrationCond(info)) { + continue; + } + + auto target = SelectBestPlacementChunkServer(info, source); + if (target == UNINTIALIZE_ID) { + LOG(WARNING) << "copysetScheduler can not select chunkServer " + "to migrate " << info.CopySetInfoStr() + << ", which replica: " << source << " is pendding"; + continue; + } + + Operator op = operatorFactory.CreateChangePeerOperator( + info, source, target, OperatorPriority::HighPriority); + op.timeLimit = std::chrono::seconds(changeTimeSec_); + + if (AddOperatorAndCreateCopyset(op, info, target)) { + oneRoundGenOp++; + break; + } + } + } + + if (oneRoundGenOp != 0) { + LOG(INFO) << "pendding copyset scheduler migrate " << oneRoundGenOp + << " copyset at this round"; } + return oneRoundGenOp; +} + +bool CopySetScheduler::AddOperatorAndCreateCopyset(const Operator &op, + const CopySetInfo &choose, + const ChunkServerIdType &target) { + // add operator + if (!opController_->AddOperator(op)) { + LOG(INFO) << "copysetSchduler add op " << op.OpToString() + << " fail, copyset has already has operator" + << " or operator num exceeds the limit."; + return false; + } + + // create copyset + if (!topo_->CreateCopySetAtChunkServer(choose.id, target)) { + LOG(ERROR) << "copysetScheduler create " << choose.CopySetInfoStr() + << " on chunkServer: " << target + << " error, delete operator" << op.OpToString(); + opController_->RemoveOperator(choose.id); + return false; + } + + LOG(INFO) << "copysetScheduler create " << choose.CopySetInfoStr() + << "on chunkserver:" << target + << " success. generator op: " + << op.OpToString() << "success"; + return true; +} + +int CopySetScheduler::NormalCopySetSchedule(const std::map> &distribute) { // 2. measure the average, range and standard deviation of number of copyset // on chunkservers float avg; int range; float stdvariance; + int oneRoundGenOp = 0; StatsCopysetDistribute(distribute, &avg, &range, &stdvariance); /** * 3. Set migration condition @@ -83,7 +148,7 @@ int CopySetScheduler::DoCopySetSchedule(PoolIdType lid) { **/ ChunkServerIdType source = UNINTIALIZE_ID; if (range <= avg * copysetNumRangePercent_) { - return source; + return oneRoundGenOp; } Operator op; @@ -91,29 +156,37 @@ int CopySetScheduler::DoCopySetSchedule(PoolIdType lid) { CopySetInfo choose; // this function call will select the source, target and the copyset if (CopySetMigration(distribute, &op, &source, &target, &choose)) { - // add operator - if (!opController_->AddOperator(op)) { - LOG(INFO) << "copysetSchduler add op " << op.OpToString() - << " fail, copyset has already has operator"; + if (AddOperatorAndCreateCopyset(op, choose, target)) { + oneRoundGenOp++; } + } - // create copyset - if (!topo_->CreateCopySetAtChunkServer(choose.id, target)) { - LOG(ERROR) << "copysetScheduler create " << choose.CopySetInfoStr() - << " on chunkServer: " << target - << " error, delete operator" << op.OpToString(); - opController_->RemoveOperator(choose.id); - } else { - LOG(INFO) << "copysetScheduler create " << choose.CopySetInfoStr() - << "on chunkserver:" << target - << " success. generator op: " - << op.OpToString() << "success"; - } + return oneRoundGenOp; +} + +int CopySetScheduler::DoCopySetSchedule(PoolIdType lid) { + // 1. collect the chunkserver list and copyset list of the cluster, then + // collect copyset on every online chunkserver + auto copysetList = topo_->GetCopySetInfosInLogicalPool(lid); + auto chunkserverList = topo_->GetChunkServersInLogicalPool(lid); + + std::map> penddingDistribute; + SchedulerHelper::CopySetDistributionInOnlineChunkServer( + ChunkServerStatus::PENDDING, + copysetList, chunkserverList, &penddingDistribute); + if (!penddingDistribute.empty()) { + return PenddingCopySetSchedule(penddingDistribute); } - LOG_EVERY_N(INFO, 20) << "copysetScheduler is continually adjusting"; - LOG(INFO) << "copysetScheduler end."; - return static_cast(source); + std::map> normalDistribute; + SchedulerHelper::CopySetDistributionInOnlineChunkServer( + ChunkServerStatus::READWRITE, copysetList, + chunkserverList, &normalDistribute); + if (normalDistribute.empty()) { + LOG(WARNING) << "no not-retired chunkserver in topology"; + return 0; + } + return NormalCopySetSchedule(normalDistribute); } void CopySetScheduler::StatsCopysetDistribute( diff --git a/src/mds/schedule/leaderScheduler.cpp b/src/mds/schedule/leaderScheduler.cpp index fd15c162a8..4d7cdc455d 100644 --- a/src/mds/schedule/leaderScheduler.cpp +++ b/src/mds/schedule/leaderScheduler.cpp @@ -34,12 +34,15 @@ namespace curve { namespace mds { namespace schedule { int LeaderScheduler::Schedule() { - LOG(INFO) << "leaderScheduler begin."; - + LOG(INFO) << "schedule: leaderScheduler begin."; + int oneRoundGenOp = 0; for (auto lid : topo_->GetLogicalpools()) { - DoLeaderSchedule(lid); + oneRoundGenOp += DoLeaderSchedule(lid); } - return 1; + + LOG(INFO) << "schedule: leaderScheduler end, generate operator num " + << oneRoundGenOp; + return oneRoundGenOp; } int LeaderScheduler::DoLeaderSchedule(PoolIdType lid) { @@ -57,7 +60,8 @@ int LeaderScheduler::DoLeaderSchedule(PoolIdType lid) { std::shuffle(csInfos.begin(), csInfos.end(), g); for (auto csInfo : csInfos) { - if (csInfo.IsOffline()) { + // skip offline chunkserver or pendding chunkserver + if (csInfo.IsOffline() || csInfo.IsPendding()) { continue; } @@ -177,6 +181,12 @@ bool LeaderScheduler::transferLeaderOut(ChunkServerIdType source, int count, break; } + // can not transfer to pendding chunkserver + if (csInfo.IsPendding()) { + continue; + } + + // can not transfer to myself if (source == peerInfo.id) { continue; } diff --git a/src/mds/schedule/rapidLeaderScheduler.cpp b/src/mds/schedule/rapidLeaderScheduler.cpp index 1768634738..cbc72bfa63 100644 --- a/src/mds/schedule/rapidLeaderScheduler.cpp +++ b/src/mds/schedule/rapidLeaderScheduler.cpp @@ -121,6 +121,7 @@ bool RapidLeaderScheduler::LeaderStatInSpecifiedLogicalPool( // get copyset info list for every chunkserver SchedulerHelper::CopySetDistributionInOnlineChunkServer( + ChunkServerStatus::READWRITE, copysetVec, chunkserverVec, &stat->distribute); // calculate average leader number for every chunkserver diff --git a/src/mds/schedule/scheduler.cpp b/src/mds/schedule/scheduler.cpp index 1303f39d7f..949387256b 100644 --- a/src/mds/schedule/scheduler.cpp +++ b/src/mds/schedule/scheduler.cpp @@ -256,7 +256,8 @@ ChunkServerIdType Scheduler::SelectRedundantReplicaToRemove( // the one to remove should be selected from BCDE // 2.2 greater // any chunkserver to remove will satisfy the requirement, thus we first - // consider the one not online, then consider the scatter-width + // consider the one not online, then the one is pendding , + // then consider the scatter-width // for example, if the replicas are: // A(zone1) B(zone2) C(zone3) D(zone4) E(zone4) // we can remove anyone of it @@ -273,7 +274,7 @@ ChunkServerIdType Scheduler::SelectRedundantReplicaToRemove( } } - // consider offline chunkservers first + // consider offline or pendding chunkservers first for (auto cs : candidateChunkServer) { ChunkServerInfo csInfo; if (!topo_->GetChunkServerInfo(cs, &csInfo)) { @@ -283,15 +284,22 @@ ChunkServerIdType Scheduler::SelectRedundantReplicaToRemove( return UNINTIALIZE_ID; } - // chunkserver is not offline + // chunkserver is offline if (csInfo.IsOffline()) { LOG(WARNING) << "scheduler choose to remove offline chunkServer " << cs << " from " << copySetInfo.CopySetInfoStr(); return cs; } + + // chunkserver is pendding + if (csInfo.IsPendding()) { + LOG(WARNING) << "scheduler choose to remove pendding chunkServer " + << cs << " from " << copySetInfo.CopySetInfoStr(); + return cs; + } } - // if all chunkserver are online, select according to the impact on scatter-width //NOLINT + // if all chunkserver are online and not pendding, select according to the impact on scatter-width //NOLINT // first rank chunkser by the number of copyset on it std::map> distribute; std::vector>> desc; diff --git a/src/mds/schedule/scheduler.h b/src/mds/schedule/scheduler.h index f1757c7eaf..278a0f2d3c 100644 --- a/src/mds/schedule/scheduler.h +++ b/src/mds/schedule/scheduler.h @@ -171,7 +171,7 @@ class CopySetScheduler : public Scheduler { * @brief Schedule Generating operator according to * the condition of the cluster * - * @return chunkserver to add, a value for POC + * @return operator num generated */ int Schedule() override; @@ -245,6 +245,36 @@ class CopySetScheduler : public Scheduler { */ bool CopySetSatisfiyBasicMigrationCond(const CopySetInfo &info); + /** + * @brief migrate one copyset between online & no pendding chunkservers + * + * @param[in] distribute Copyset on every online && no pendding chunkserver + * + * @return Source node of the migration + */ + int NormalCopySetSchedule(const std::map> &distribute); + + /** + * @brief migrate one copyset from online && pendding chunkserver to + * other healty chunkserver + * @param[in] distribute Copyset on every online && pendding chunkserver + * @return migrate copyset num + */ + int PenddingCopySetSchedule(const std::map> &distribute); + + /** + * @brief add op to opController and create copyset at chunkserver + * @param[in] op the oprator to add to opController + * @param[in] choose the copyset need to schedule + * @param[in] target the chunkserver want to add to this copyset + * @return return true if sucess, retrun false if not + */ + bool AddOperatorAndCreateCopyset(const Operator &op, + const CopySetInfo &choose, + const ChunkServerIdType &target); + private: // Running interval of CopySetScheduler int64_t runInterval_; diff --git a/src/mds/schedule/scheduler_helper.cpp b/src/mds/schedule/scheduler_helper.cpp index 3ce07d9fda..b862ab7c11 100644 --- a/src/mds/schedule/scheduler_helper.cpp +++ b/src/mds/schedule/scheduler_helper.cpp @@ -379,6 +379,43 @@ void SchedulerHelper::CopySetDistributionInOnlineChunkServer( } } } + +void SchedulerHelper::CopySetDistributionInOnlineChunkServer( + const ChunkServerStatus status, + const std::vector ©setList, + const std::vector &chunkserverList, + std::map> *out) { + // calculate the copysetlist by traversing the copyset list + for (auto item : copysetList) { + for (auto peer : item.peers) { + if (out->find(peer.id) == out->end()) { + (*out)[peer.id] = std::vector{item}; + } else { + (*out)[peer.id].emplace_back(item); + } + } + } + + for (auto item : chunkserverList) { + // remove offline chunkserver + if (item.IsOffline()) { + out->erase(item.info.id); + continue; + } + + if (item.status != status) { + out->erase(item.info.id); + continue; + } + + // report empty list for empty and healthy chunkserver + if (item.status == ChunkServerStatus::READWRITE + && out->find(item.info.id) == out->end()) { + (*out)[item.info.id] = std::vector{}; + } + } +} + } // namespace schedule } // namespace mds } // namespace curve diff --git a/src/mds/schedule/scheduler_helper.h b/src/mds/schedule/scheduler_helper.h index a91d833f42..7a81af78df 100644 --- a/src/mds/schedule/scheduler_helper.h +++ b/src/mds/schedule/scheduler_helper.h @@ -172,6 +172,21 @@ class SchedulerHelper { const std::vector ©setList, const std::vector &chunkserverList, std::map> *out); + + /** + * @brief CopySetDistribution Calculate the copyset number on + * chunkserver in online status + * + * @param[in] status filter the chunkserver in status + * @param[in] copysetList Copyset list in topo info + * @param[in] chunkserverList Chunkserver list in topo info + * @param[out] out List of copyset on chunkserver + */ + static void CopySetDistributionInOnlineChunkServer( + const ChunkServerStatus status, + const std::vector ©setList, + const std::vector &chunkserverList, + std::map> *out); }; } // namespace schedule } // namespace mds diff --git a/src/tools/status_tool.cpp b/src/tools/status_tool.cpp index 8941db7020..c16c39631a 100644 --- a/src/tools/status_tool.cpp +++ b/src/tools/status_tool.cpp @@ -206,13 +206,25 @@ int StatusTool::ChunkServerListCmd() { std::cout << "ListChunkserversInCluster fail!" << std::endl; return -1; } + std::cout << "curve chunkserver list: " << std::endl; uint64_t total = 0; uint64_t online = 0; uint64_t offline = 0; uint64_t unstable = 0; + uint64_t pendding = 0; + uint64_t retired = 0; + uint64_t penddingCopyset = 0; for (auto& chunkserver : chunkservers) { auto csId = chunkserver.chunkserverid(); + std::vector copysets; + int ret = mdsClient_->GetCopySetsInChunkServer(csId, ©sets); + if (ret != 0) { + std::cout << "GetCopySetsInChunkServer fail, chunkserver id = " + << csId; + return -1; + } + double unhealthyRatio; if (FLAGS_checkCSAlive) { // 发RPC重置online状态 @@ -249,6 +261,13 @@ int StatusTool::ChunkServerListCmd() { } online++; } + if (chunkserver.status() == ChunkServerStatus::PENDDING) { + pendding++; + penddingCopyset += copysets.size(); + } + if (chunkserver.status() == ChunkServerStatus::RETIRED) { + retired++; + } total++; std::cout << "chunkServerID = " << csId << ", diskType = " << chunkserver.disktype() @@ -260,6 +279,7 @@ int StatusTool::ChunkServerListCmd() { << DiskState_Name(chunkserver.diskstatus()) << ", onlineState = " << OnlineState_Name(chunkserver.onlinestate()) + << ", copysetNum = " << copysets.size() << ", mountPoint = " << chunkserver.mountpoint() << ", diskCapacity = " << chunkserver.diskcapacity() / curve::mds::kGB << " GB" @@ -279,6 +299,10 @@ int StatusTool::ChunkServerListCmd() { std::cout <<", unstable: " << unstable; } std::cout << ", offline: " << offline << std::endl; + + std::cout << "pendding: " << pendding + << ", penddingCopyset: " << penddingCopyset + << ", retired:" << retired << std::endl; return 0; } diff --git a/test/mds/schedule/leaderScheduler_test.cpp b/test/mds/schedule/leaderScheduler_test.cpp index 2c0d9d6641..e109647809 100644 --- a/test/mds/schedule/leaderScheduler_test.cpp +++ b/test/mds/schedule/leaderScheduler_test.cpp @@ -350,6 +350,97 @@ TEST_F(TestLeaderSchedule, test_tranferLeaderout_normal) { ASSERT_EQ(csInfo6.info.id, res->GetTargetPeer()); } +TEST_F(TestLeaderSchedule, test_tranferLeaderout_pendding) { + // chunkserver1 chunkserver2 chunkserver3 + // leaderCount 1 2 0 + // copyset 1 1 1 + PeerInfo peer1(1, 1, 1, "192.168.10.1", 9000); + PeerInfo peer2(2, 2, 2, "192.168.10.2", 9000); + PeerInfo peer3(3, 3, 3, "192.168.10.3", 9000); + PeerInfo peer4(4, 4, 4, "192.168.10.4", 9000); + PeerInfo peer5(5, 5, 5, "192.168.10.5", 9000); + PeerInfo peer6(6, 6, 6, "192.168.10.6", 9000); + auto onlineState = ::curve::mds::topology::OnlineState::ONLINE; + auto offlineState = ::curve::mds::topology::OnlineState::OFFLINE; + auto diskState = ::curve::mds::topology::DiskState::DISKNORMAL; + auto statInfo = ::curve::mds::heartbeat::ChunkServerStatisticInfo(); + ChunkServerInfo csInfo1( + peer1, onlineState, diskState, ChunkServerStatus::PENDDING, + 0, 100, 10, statInfo); + ChunkServerInfo csInfo2( + peer2, onlineState, diskState, ChunkServerStatus::READWRITE, + 5, 100, 10, statInfo); + ChunkServerInfo csInfo3( + peer3, onlineState, diskState, ChunkServerStatus::READWRITE, + 0, 100, 10, statInfo); + + ChunkServerInfo csInfo4( + peer4, onlineState, diskState, ChunkServerStatus::READWRITE, + 4, 100, 10, statInfo); + ChunkServerInfo csInfo5( + peer5, onlineState, diskState, ChunkServerStatus::READWRITE, + 5, 100, 10, statInfo); + ChunkServerInfo csInfo6( + peer6, onlineState, diskState, ChunkServerStatus::PENDDING, + 0, 100, 10, statInfo); + struct timeval tm; + gettimeofday(&tm, NULL); + csInfo3.startUpTime = tm.tv_sec - 2; + csInfo6.startUpTime = tm.tv_sec - 2; + std::vector csInfos1({csInfo1, csInfo2, csInfo3}); + std::vector csInfos2({csInfo4, csInfo5, csInfo6}); + + PoolIdType poolId = 1; + CopySetIdType copysetId = 1; + CopySetKey copySetKey; + copySetKey.first = poolId; + copySetKey.second = copysetId; + EpochType epoch = 1; + ChunkServerIdType leader = 2; + CopySetInfo copySet1(copySetKey, epoch, leader, + std::vector({peer1, peer2, peer3}), + ConfigChangeInfo{}, CopysetStatistics{}); + CopySetInfo copySet2(CopySetKey{2, 1}, epoch, 5, + std::vector({peer4, peer5, peer6}), + ConfigChangeInfo{}, CopysetStatistics{}); + std::vector copySetInfos1({copySet1}); + std::vector copySetInfos2({copySet2}); + + EXPECT_CALL(*topoAdapter_, GetLogicalpools()) + .WillOnce(Return(std::vector({1, 2}))); + EXPECT_CALL(*topoAdapter_, GetChunkServersInLogicalPool(1)) + .WillOnce(Return(csInfos1)); + EXPECT_CALL(*topoAdapter_, GetChunkServersInLogicalPool(2)) + .WillOnce(Return(csInfos2)); + EXPECT_CALL(*topoAdapter_, GetCopySetInfosInLogicalPool(1)) + .WillOnce(Return(copySetInfos1)); + EXPECT_CALL(*topoAdapter_, GetCopySetInfosInLogicalPool(2)) + .WillOnce(Return(copySetInfos2)); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo1), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(2, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo2), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(3, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo3), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(4, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(csInfo4), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(5, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(csInfo5), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(6, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(csInfo6), Return(true))); + + leaderScheduler_->Schedule(); + Operator op; + ASSERT_TRUE(opController_->GetOperatorById(copySet1.id, &op)); + ASSERT_EQ(OperatorPriority::NormalPriority, op.priority); + ASSERT_EQ(std::chrono::seconds(10), op.timeLimit); + TransferLeader *res = dynamic_cast(op.step.get()); + ASSERT_TRUE(res != nullptr); + ASSERT_EQ(csInfo3.info.id, res->GetTargetPeer()); + + ASSERT_FALSE(opController_->GetOperatorById(copySet2.id, &op)); +} + TEST_F(TestLeaderSchedule, test_transferLeaderIn_normal) { // chunkserver1 chunkserver2 chunkserver3 chunkserver4 // leaderCount 0 3 2 1 @@ -433,6 +524,89 @@ TEST_F(TestLeaderSchedule, test_transferLeaderIn_normal) { ASSERT_EQ(1, res->GetTargetPeer()); } +TEST_F(TestLeaderSchedule, test_transferLeaderIn_pendding) { + // chunkserver1 chunkserver2 chunkserver3 chunkserver4 + // leaderCount 0 3 2 1 + // copyset 1 1 1(有operator) + // 2 2 2 + // 3 3 3 + PeerInfo peer1(1, 1, 1, "192.168.10.1", 9000); + PeerInfo peer2(2, 2, 2, "192.168.10.2", 9000); + PeerInfo peer3(3, 3, 3, "192.168.10.3", 9000); + PeerInfo peer4(3, 4, 4, "192.168.10.4", 9000); + auto onlineState = ::curve::mds::topology::OnlineState::ONLINE; + auto offlineState = ::curve::mds::topology::OnlineState::OFFLINE; + auto diskState = ::curve::mds::topology::DiskState::DISKNORMAL; + auto statInfo = ::curve::mds::heartbeat::ChunkServerStatisticInfo(); + ChunkServerInfo csInfo1( + peer1, onlineState, diskState, ChunkServerStatus::READWRITE, + 0, 100, 10, statInfo); + csInfo1.startUpTime = ::curve::common::TimeUtility::GetTimeofDaySec() - 4; + ChunkServerInfo csInfo2( + peer2, onlineState, diskState, ChunkServerStatus::READWRITE, + 3, 100, 10, statInfo); + ChunkServerInfo csInfo3( + peer3, onlineState, diskState, ChunkServerStatus::PENDDING, + 2, 100, 10, statInfo); + ChunkServerInfo csInfo4( + peer4, onlineState, diskState, ChunkServerStatus::READWRITE, + 1, 100, 10, statInfo); + std::vector csInfos({csInfo1, csInfo2, csInfo3, csInfo4}); + + PoolIdType poolId = 1; + CopySetIdType copysetId = 1; + CopySetKey copySetKey; + copySetKey.first = poolId; + copySetKey.second = copysetId; + EpochType epoch = 1; + ChunkServerIdType leader = 2; + CopySetInfo copySet1(copySetKey, epoch, leader, + std::vector({peer1, peer2, peer3}), + ConfigChangeInfo{}, CopysetStatistics{}); + copySetKey.second = 2; + leader = 3; + CopySetInfo copySet2(copySetKey, epoch, leader, + std::vector({peer1, peer2, peer3}), + ConfigChangeInfo{}, CopysetStatistics{}); + copySetKey.second = 3; + leader = 4; + CopySetInfo copySet3(copySetKey, epoch, leader, + std::vector({peer2, peer3, peer4}), + ConfigChangeInfo{}, CopysetStatistics{}); + + copySetKey.second = 1; + Operator testOperator(1, copySetKey, OperatorPriority::NormalPriority, + steady_clock::now(), std::make_shared(1)); + ASSERT_TRUE(opController_->AddOperator(testOperator)); + + EXPECT_CALL(*topoAdapter_, GetLogicalpools()) + .WillOnce(Return(std::vector({1}))); + EXPECT_CALL(*topoAdapter_, GetChunkServersInLogicalPool(1)) + .WillOnce(Return(csInfos)); + EXPECT_CALL(*topoAdapter_, GetCopySetInfosInLogicalPool(1)) + .Times(2) + .WillOnce(Return(std::vector({copySet1}))) + .WillOnce(Return(std::vector({copySet3, copySet2}))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(1, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<1>(csInfo1), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(3, _)) + .Times(3) + .WillRepeatedly(DoAll(SetArgPointee<1>(csInfo3), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(2, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<1>(csInfo2), Return(true))); + + ASSERT_EQ(1, leaderScheduler_->Schedule()); + Operator op; + ASSERT_TRUE(opController_->GetOperatorById(copySet2.id, &op)); + ASSERT_EQ(OperatorPriority::NormalPriority, op.priority); + ASSERT_EQ(std::chrono::seconds(10), op.timeLimit); + TransferLeader *res = dynamic_cast(op.step.get()); + ASSERT_TRUE(res != nullptr); + ASSERT_EQ(1, res->GetTargetPeer()); +} + } // namespace schedule } // namespace mds } // namespace curve diff --git a/test/mds/schedule/rapidLeaderSheduler_test.cpp b/test/mds/schedule/rapidLeaderSheduler_test.cpp index bf22891114..3caecf7111 100644 --- a/test/mds/schedule/rapidLeaderSheduler_test.cpp +++ b/test/mds/schedule/rapidLeaderSheduler_test.cpp @@ -216,6 +216,38 @@ TEST_F(TestRapidLeaderSchedule, test_rapid_schedule_success) { ASSERT_EQ(3, operators[2].copysetID.second); } +TEST_F(TestRapidLeaderSchedule, test_rapid_schedule_pendding) { + // 快速均衡成功 + // chunkserver-1 chunkserver-2 chunkserver-3 + // copyset-1(leader) copyset-1 copyset-1 + // copyset-2(leader) copyset-2 copyset-2 + // copyset-3(leader) copyset-3 copyset-3 + std::shared_ptr rapidLeaderScheduler; + rapidLeaderScheduler = std::make_shared( + opt_, topoAdapter_, opController_, 1); + + EXPECT_CALL(*topoAdapter_, GetLogicalpools()) + .WillOnce(Return(std::vector{1})); + auto chunkserverInfosBak = chunkServerInfos_; + chunkserverInfosBak[0].leaderCount = 3; + chunkserverInfosBak[0].status = ChunkServerStatus::PENDDING; + EXPECT_CALL(*topoAdapter_, GetChunkServersInLogicalPool(1)) + .WillOnce(Return(chunkserverInfosBak)); + + auto copyset1 = GetCopySetInfoForTest(); + auto copyset2 = GetCopySetInfoForTest(); + copyset2.id = CopySetKey{1, 2}; + auto copyset3 = GetCopySetInfoForTest(); + copyset3.id = CopySetKey{1, 3}; + EXPECT_CALL(*topoAdapter_, GetCopySetInfosInLogicalPool(1)) + .WillOnce(Return( + std::vector{copyset1, copyset2, copyset3})); + + ASSERT_EQ(kScheduleErrCodeSuccess, rapidLeaderScheduler->Schedule()); + auto operators = opController_->GetOperators(); + ASSERT_EQ(0, operators.size()); +} + } // namespace schedule } // namespace mds } // namespace curve diff --git a/test/mds/schedule/replicaScheduler_test.cpp b/test/mds/schedule/replicaScheduler_test.cpp index 871c5bccad..fb169c8c6a 100644 --- a/test/mds/schedule/replicaScheduler_test.cpp +++ b/test/mds/schedule/replicaScheduler_test.cpp @@ -333,6 +333,68 @@ TEST_F(TestReplicaSchedule, test_copySet_has_larger_replicaNum_selectCorrect) { RemovePeer *res = dynamic_cast(op.step.get()); ASSERT_FALSE(res == nullptr); } + +TEST_F(TestReplicaSchedule, test_copySet_has_larger_replicaNum_selectpendding) { + auto testCopySetInfo = GetCopySetInfoForTest(); + PeerInfo peer1(1, 1, 1, "192.168.10.1", 9000); + PeerInfo peer2(2, 2, 2, "192.168.10.2", 9000); + PeerInfo peer3(3, 3, 3, "192.168.10.3", 9000); + PeerInfo peer4(4, 4, 4, "192.168.10.4", 9000); + testCopySetInfo.peers = std::vector({peer1, peer2, peer3, peer4}); + ChunkServerInfo csInfo1(testCopySetInfo.peers[0], OnlineState::ONLINE, + DiskState::DISKNORMAL, ChunkServerStatus::READWRITE, + 2, 100, 100, ChunkServerStatisticInfo{}); + ChunkServerInfo csInfo2(testCopySetInfo.peers[1], OnlineState::ONLINE, + DiskState::DISKNORMAL, ChunkServerStatus::READWRITE, + 2, 100, 100, ChunkServerStatisticInfo{}); + ChunkServerInfo csInfo3(testCopySetInfo.peers[2], OnlineState::ONLINE, + DiskState::DISKNORMAL, ChunkServerStatus::READWRITE, + 2, 100, 100, ChunkServerStatisticInfo{}); + ChunkServerInfo csInfo4(testCopySetInfo.peers[3], OnlineState::ONLINE, + DiskState::DISKNORMAL, ChunkServerStatus::PENDDING, + 2, 100, 100, ChunkServerStatisticInfo{}); + EXPECT_CALL(*topoAdapter_, GetStandardReplicaNumInLogicalPool(_)) + .Times(2).WillRepeatedly(Return(3)); + EXPECT_CALL(*topoAdapter_, GetAvgScatterWidthInLogicalPool(_)) + .WillRepeatedly(Return(90)); + EXPECT_CALL(*topoAdapter_, GetCopySetInfos()) + .WillOnce(Return(std::vector({testCopySetInfo}))); + EXPECT_CALL(*topoAdapter_, GetStandardZoneNumInLogicalPool(_)) + .WillOnce(Return(3)); + + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo1), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(2, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo2), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(3, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo3), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerInfo(4, _)) + .WillOnce(DoAll(SetArgPointee<1>(csInfo4), Return(true))); + EXPECT_CALL(*topoAdapter_, GetChunkServerScatterMap(1, _)) + .WillRepeatedly(SetArgPointee<1>( + std::map{{2, 1}, {3, 1}, {4, 1}})); + EXPECT_CALL(*topoAdapter_, GetChunkServerScatterMap(2, _)) + .WillRepeatedly(SetArgPointee<1>( + std::map{{1, 1}, {3, 1}, {4, 1}})); + EXPECT_CALL(*topoAdapter_, GetChunkServerScatterMap(3, _)) + .WillRepeatedly(SetArgPointee<1>( + std::map{{1, 1}, {2, 1}, {4, 1}})); + EXPECT_CALL(*topoAdapter_, GetChunkServerScatterMap(4, _)) + .WillRepeatedly(SetArgPointee<1>( + std::map{{2, 1}, {3, 1}, {1, 1}})); + + replicaScheduler_->Schedule(); + Operator op; + ASSERT_TRUE(opController_->GetOperatorById(testCopySetInfo.id, &op)); + ASSERT_EQ(testCopySetInfo.id, op.copysetID); + ASSERT_EQ(testCopySetInfo.epoch, op.startEpoch); + ASSERT_EQ(OperatorPriority::HighPriority, op.priority); + ASSERT_EQ(std::chrono::seconds(100), op.timeLimit); + RemovePeer *res = dynamic_cast(op.step.get()); + ASSERT_FALSE(res == nullptr); + ASSERT_EQ(res->GetTargetPeer(), 4); +} + } // namespace schedule } // namespace mds } // namespace curve diff --git a/test/mds/schedule/schedulerPOC/scheduler_poc.cpp b/test/mds/schedule/schedulerPOC/scheduler_poc.cpp index 3bc9bd8bbf..34dead4272 100644 --- a/test/mds/schedule/schedulerPOC/scheduler_poc.cpp +++ b/test/mds/schedule/schedulerPOC/scheduler_poc.cpp @@ -398,6 +398,10 @@ class CopysetSchedulerPOC : public testing::Test { LOG(INFO) << "chunkserver " << it << "is offline"; continue; } + if (chunkserver.GetStatus() == ChunkServerStatus::PENDDING) { + LOG(INFO) << "chunkserver " << it << "is pendding"; + continue; + } int factor = GetChunkServerScatterwith(it); sumFactor += factor; @@ -486,6 +490,9 @@ class CopysetSchedulerPOC : public testing::Test { if (chunkserver.GetOnlineState() == OnlineState::OFFLINE) { continue; } + if (chunkserver.GetStatus() == ChunkServerStatus::PENDDING) { + continue; + } int number = topo_->GetCopySetsInChunkServer(it).size(); sumNumber += number; numberMap[it] = number; @@ -1054,11 +1061,14 @@ TEST_F(CopysetSchedulerPOC, test_scatterwith_after_copysetRebalance_1) { //NOLIN // 2. chunkserver-choose恢复成online状态 SetChunkServerOnline(choose); BuildCopySetScheduler(1); - int removeOne = 0; + std::vector csList; + csList = topo_->GetChunkServerInCluster(); + std::set csSet(csList.begin(), csList.end()); + int operatorCount = 0; do { - removeOne = copySetScheduler_->Schedule(); - ApplyOperatorsInOpController(std::set{removeOne}); - } while (removeOne > 0); + operatorCount = copySetScheduler_->Schedule(); + ApplyOperatorsInOpController(csSet); + } while (operatorCount > 0); PrintScatterWithInLogicalPool(); PrintCopySetNumInLogicalPool(); LOG(INFO) << "offline one:" << choose; @@ -1192,14 +1202,16 @@ TEST_F(CopysetSchedulerPOC, test_scatterwith_after_copysetRebalance_3) { //NOLIN // 2. chunkserver恢复成online状态 SetChunkServerOnline(idlist); BuildCopySetScheduler(1); - int removeOne = 0; + std::vector csList; + csList = topo_->GetChunkServerInCluster(); + std::set csSet(csList.begin(), csList.end()); + int operatorCount = 0; do { - removeOne = copySetScheduler_->Schedule(); - if (removeOne > 0) { - ApplyOperatorsInOpController( - std::set{removeOne}); + operatorCount = copySetScheduler_->Schedule(); + if (operatorCount > 0) { + ApplyOperatorsInOpController(csSet); } - } while (removeOne > 0); + } while (operatorCount > 0); PrintScatterWithInLogicalPool(); PrintCopySetNumInLogicalPool(); @@ -1217,6 +1229,63 @@ TEST_F(CopysetSchedulerPOC, test_scatterwith_after_copysetRebalance_3) { //NOLIN // 均值：100, 方差：1, 标准差： 1, 最大值： 101, 最小值：91 } +TEST_F(CopysetSchedulerPOC, test_scatterwith_after_copysetRebalance_4) { //NOLINT + // set one chunkserver status from online to pendding, and the copyset on it will schedule out //NOLINT + + // set one chunkserver status to pendding + auto chunkserverlist = topo_->GetChunkServerInServer(1); + ChunkServerIdType target = *chunkserverlist.begin(); + topo_->UpdateChunkServerRwState(ChunkServerStatus::PENDDING, target); + + int opNum = 0; + int targetOpNum = topo_->GetCopySetsInChunkServer(target).size(); + BuilRecoverScheduler(1); + BuildCopySetScheduler(1); + int removeOne = 0; + do { + removeOne = copySetScheduler_->Schedule(); + opNum += removeOne; + if (removeOne > 0) { + ApplyOperatorsInOpController( + std::set{target}); + } + } while (removeOne > 0); + ASSERT_EQ(opNum, targetOpNum); + ASSERT_EQ(0, topo_->GetCopySetsInChunkServer(target).size()); + PrintScatterWithInOnlineChunkServer(); + PrintCopySetNumInOnlineChunkServer(); +} + +TEST_F(CopysetSchedulerPOC, test_scatterwith_after_copysetRebalance_5) { //NOLINT + // set two chunkserver status from online to pendding, and the copyset on it will schedule out //NOLINT + + // set two chunkserver status to pendding + auto chunkserverlist = topo_->GetChunkServerInServer(1); + ChunkServerIdType target = *chunkserverlist.begin(); + topo_->UpdateChunkServerRwState(ChunkServerStatus::PENDDING, target); + topo_->UpdateChunkServerRwState(ChunkServerStatus::PENDDING, target + 1); + + int opNum = 0; + int targetOpNum = topo_->GetCopySetsInChunkServer(target).size(); + targetOpNum += topo_->GetCopySetsInChunkServer(target + 1).size(); + BuilRecoverScheduler(1); + BuildCopySetScheduler(1); + int removeOne = 0; + do { + removeOne = copySetScheduler_->Schedule(); + opNum += removeOne; + if (removeOne > 0) { + ASSERT_EQ(removeOne, 2); + ApplyOperatorsInOpController( + std::set{target, target + 1}); + } + } while (removeOne > 0); + ASSERT_EQ(opNum, targetOpNum); + ASSERT_EQ(0, topo_->GetCopySetsInChunkServer(target).size()); + PrintScatterWithInOnlineChunkServer(); + PrintCopySetNumInOnlineChunkServer(); +} + TEST_F(CopysetSchedulerPOC, DISABLED_test_leader_rebalance) { leaderCountOn = true; BuildLeaderScheduler(4);