Skip to content

Commit

Permalink
Support intersecting hosts in MergeBoxes (#12282)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexvru
alexvru committed Dec 5, 2024
1 parent 2f9a0e3 commit 06d6e2f
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 8 deletions.
45 changes: 37 additions & 8 deletions ydb/core/mind/bscontroller/cmds_box.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,17 +108,46 @@ namespace NKikimr::NBsController {
TBoxInfo& origin = getBox(cmd.GetOriginBoxId(), cmd.GetOriginBoxGeneration(), "origin");
TBoxInfo& target = getBox(cmd.GetTargetBoxId(), cmd.GetTargetBoxGeneration(), "target");

while (origin.Hosts) {
auto node = origin.Hosts.extract(origin.Hosts.begin());
node.key().BoxId = cmd.GetTargetBoxId();
if (!target.Hosts.insert(std::move(node)).inserted) {
throw TExError() << "duplicate hosts in merged box";
THashMap<std::tuple<THostConfigId, THostConfigId>, THostConfigId> mergedHostConfigs;

for (auto& [originKey, originValue] : origin.Hosts) {
const TBoxInfo::THostKey targetKey = originKey.ChangeBox(cmd.GetTargetBoxId()); // make new key for target set
if (const auto [it, inserted] = target.Hosts.try_emplace(targetKey, std::move(originValue)); !inserted) {
TBoxInfo::THostInfo& targetValue = it->second;
if (targetValue.EnforcedNodeId != originValue.EnforcedNodeId) {
throw TExError() << "different EnforcedNodeId for the same HostId# " << THostId(originKey);
} else {
const auto sourceConfigs = std::make_tuple(std::min(originValue.HostConfigId, targetValue.HostConfigId),
std::max(originValue.HostConfigId, targetValue.HostConfigId));

const auto [jt, inserted] = mergedHostConfigs.try_emplace(sourceConfigs);
if (inserted) {
auto& hostConfigs = HostConfigs.Unshare();
jt->second = hostConfigs.empty() ? 1 : std::prev(hostConfigs.end())->first + 1;
Y_ABORT_UNLESS(!hostConfigs.contains(jt->second));
THostConfigInfo& newHostConfig = hostConfigs[jt->second];
newHostConfig.Generation = 1;

auto addDrivesFrom = [&](THostConfigId hostConfigId) {
const auto it = hostConfigs.find(hostConfigId);
if (it == hostConfigs.end()) {
throw TExError() << "missing HostConfig for origin/target box";
}
for (const auto& [key, value] : it->second.Drives) {
const THostConfigInfo::TDriveKey newKey{{jt->second, key.Path}};
if (const auto [it, inserted] = newHostConfig.Drives.emplace(newKey, value); !inserted) {
throw TExError() << "duplicate drives in origin/target host configs";
}
}
};
addDrivesFrom(originValue.HostConfigId);
addDrivesFrom(targetValue.HostConfigId);
}
targetValue.HostConfigId = jt->second;
}
}
}

// spin the generation
target.Generation = target.Generation.GetOrElse(1) + 1;

auto& storagePools = StoragePools.Unshare();
auto& storagePoolGroups = StoragePoolGroups.Unshare();

Expand Down
4 changes: 4 additions & 0 deletions ydb/core/mind/bscontroller/impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -996,6 +996,10 @@ class TBlobStorageController : public TActor<TBlobStorageController>, public TTa
std::tie(BoxId, Fqdn, IcPort) = std::move(key);
}

THostKey ChangeBox(Schema::BoxHostV2::BoxId::Type newBoxId) const {
return {{newBoxId, Fqdn, IcPort}};
}

auto GetKey() const {
return std::tie(BoxId, Fqdn, IcPort);
}
Expand Down
127 changes: 127 additions & 0 deletions ydb/core/mind/bscontroller/ut_bscontroller/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,133 @@ Y_UNIT_TEST_SUITE(BsControllerConfig) {
});
}

Y_UNIT_TEST(MergeIntersectingBoxes) {
const ui32 numNodes = 50;
const ui32 numNodes1 = 20;
const ui32 numGroups1 = numNodes * 3;
const ui32 numGroups2 = numNodes1 * 4;
TEnvironmentSetup env(numNodes, 1);
RunTestWithReboots(env.TabletIds, [&] { return env.PrepareInitialEventsFilter(); }, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& outActiveZone) {
TFinalizer finalizer(env);
env.Prepare(dispatchName, setup, outActiveZone);

TVector<TEnvironmentSetup::TNodeRecord> nodes1, nodes2;
for (const auto& node : env.GetNodes()) {
nodes1.push_back(node);
if (nodes2.size() < numNodes1) {
nodes2.push_back(node);
}
}

TSet<ui32> nodeIds1, nodeIds2;
for (const auto& item : nodes1) {
nodeIds1.insert(std::get<2>(item));
}
for (const auto& item : nodes2) {
nodeIds2.insert(std::get<2>(item));
}

NKikimrBlobStorage::TConfigRequest request;
env.DefineBox(1, "first box", {
{"/dev/disk1", NKikimrBlobStorage::ROT, false, false, 0},
{"/dev/disk2", NKikimrBlobStorage::ROT, true, false, 0},
{"/dev/disk3", NKikimrBlobStorage::SSD, false, false, 0},
}, nodes1, request);
env.DefineStoragePool(1, 1, "first storage pool", numGroups1, NKikimrBlobStorage::ROT, {}, request);

NKikimrBlobStorage::TConfigResponse response = env.Invoke(request);
UNIT_ASSERT(response.GetSuccess());

request.Clear();
env.DefineBox(2, "second box", {
{"/dev/disk4", NKikimrBlobStorage::ROT, false, false, 0},
{"/dev/disk5", NKikimrBlobStorage::ROT, true, false, 0},
{"/dev/disk6", NKikimrBlobStorage::SSD, false, false, 0},
}, nodes2, request);
env.DefineStoragePool(2, 1, "second storage pool", numGroups2, NKikimrBlobStorage::ROT, {}, request);

response = env.Invoke(request);
UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription());

////////////////////////////////////////////////////////////////////////////////////////////////////////
// merge boxes

request.Clear();
auto& cmd = *request.AddCommand()->MutableMergeBoxes();
cmd.SetOriginBoxId(2);
cmd.SetOriginBoxGeneration(1);
cmd.SetTargetBoxId(1);
cmd.SetTargetBoxGeneration(1);
auto& item = *cmd.AddStoragePoolIdMap();
item.SetOriginStoragePoolId(1);
item.SetTargetStoragePoolId(2);

response = env.Invoke(request);
UNIT_ASSERT(response.GetSuccess());

////////////////////////////////////////////////////////////////////////////////////////////////////////
// validate result

request.Clear();
request.AddCommand()->MutableReadBox();
request.AddCommand()->MutableReadHostConfig();
request.AddCommand()->MutableQueryBaseConfig();
response = env.Invoke(request);
UNIT_ASSERT(response.GetSuccess());

THashMap<std::tuple<TString, i32>, ui32> nodeMap;
for (const auto& node : env.GetNodes()) {
nodeMap.emplace(std::make_tuple(std::get<0>(node), std::get<1>(node)), std::get<2>(node));
}

// check box nodes
using TDrives = std::set<TString>;
THashMap<ui64, TDrives> hostConfigs;
for (const auto& hostConfig : response.GetStatus(1).GetHostConfig()) {
std::set<TString>& paths = hostConfigs[hostConfig.GetHostConfigId()];
for (const auto& drive : hostConfig.GetDrive()) {
const auto [it, inserted] = paths.insert(drive.GetPath());
UNIT_ASSERT(inserted);
}
}
const auto& boxes = response.GetStatus(0).GetBox();
UNIT_ASSERT_VALUES_EQUAL(boxes.size(), 1);
const auto& box = boxes.at(0);
UNIT_ASSERT_VALUES_EQUAL(box.HostSize(), numNodes);
for (const auto& item : box.GetHost()) {
UNIT_ASSERT(hostConfigs.contains(item.GetHostConfigId()));
const auto& hostConfig = hostConfigs[item.GetHostConfigId()];
const auto& key = item.GetKey();
const auto keyTuple = std::make_tuple(key.GetFqdn(), key.GetIcPort());
UNIT_ASSERT(nodeMap.contains(keyTuple));
const ui32 nodeId = nodeMap[keyTuple];
if (nodeIds2.contains(nodeId)) { // 6 drives
UNIT_ASSERT_EQUAL(hostConfig, (TDrives{"/dev/disk1", "/dev/disk2", "/dev/disk3", "/dev/disk4", "/dev/disk5", "/dev/disk6"}));
} else { // 3 drives
UNIT_ASSERT_EQUAL(hostConfig, (TDrives{"/dev/disk1", "/dev/disk2", "/dev/disk3"}));
}
}

// validate base config
ui32 num1 = 0, num2 = 0;
{
const auto& baseConfig = response.GetStatus(2).GetBaseConfig();
for (const auto& pdisk : baseConfig.GetPDisk()) {
UNIT_ASSERT_VALUES_EQUAL(pdisk.GetBoxId(), 1);
}
for (const auto& group : baseConfig.GetGroup()) {
UNIT_ASSERT_VALUES_EQUAL(group.GetBoxId(), 1);
const ui64 storagePoolId = group.GetStoragePoolId();
UNIT_ASSERT(storagePoolId == 1 || storagePoolId == 2);
++(storagePoolId == 1 ? num1 : num2);
}
UNIT_ASSERT_VALUES_EQUAL(num1, numGroups1);
UNIT_ASSERT_VALUES_EQUAL(num2, numGroups2);
}

});
}

Y_UNIT_TEST(MoveGroups) {
const ui32 numNodes = 50;
const ui32 numGroups1 = 100;
Expand Down

0 comments on commit 06d6e2f

Please sign in to comment.