Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

curvefs metaserver supports external services #974

Closed
wants to merge 1 commit into from
Closed

curvefs metaserver supports external services #974

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions curvefs/conf/metaserver.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ s3compactwq.s3infocache_size=100
global.ip=127.0.0.1 # __CURVEADM_TEMPLATE__ ${service_addr} __CURVEADM_TEMPLATE__ __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_host }} __ANSIBLE_TEMPLATE__
global.port=16701 # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__ __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_port }} __ANSIBLE_TEMPLATE__
global.external_ip=127.0.0.1 # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__ __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_host }} __ANSIBLE_TEMPLATE__
global.external_port=16701 # __CURVEADM_TEMPLATE__ ${service_external_port} __CURVEADM_TEMPLATE__
global.enable_external_server=false

# metaserver log directory
# this config item can be replaced by start up option `-log_dir`
Expand Down
2 changes: 1 addition & 1 deletion curvefs/conf/tools.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ rpcTimeoutMs=10000
rpcRetryTimes=5
# topo file path
topoFilePath=curvefs/test/tools/topo_example.json # __CURVEADM_TEMPLATE__ /curvefs/tools/conf/topology.json __CURVEADM_TEMPLATE__ __ANSIBLE_TEMPLATE__ {{ project_root_dest }}/conf/topology.json __ANSIBLE_TEMPLATE__
# metaserver
# metaserver external address
metaserverAddr=127.0.0.1:6701 # __CURVEADM_TEMPLATE__ ${cluster_metaserver_addr} __CURVEADM_TEMPLATE__ __ANSIBLE_TEMPLATE__ {{ groups.metaserver | join_peer(hostvars, "metaserver_listen_port") }} __ANSIBLE_TEMPLATE__
# etcd
etcdAddr=127.0.0.1:12379 # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__ __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_peer(hostvars, "etcd_listen_client_port") }} __ANSIBLE_TEMPLATE__
Expand Down
21 changes: 11 additions & 10 deletions curvefs/proto/topology.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ message ZoneData {
message ServerData {
required uint32 serverId = 1;
required string hostName = 2;
required string internalHostIP = 3;
required string internalIP = 3;
required uint32 internalPort = 4;
required string externalHostIP = 5;
required string externalIP = 5;
required uint32 externalPort = 6;
required uint32 zoneId = 7;
required uint32 PoolId = 8;
Expand All @@ -85,9 +85,9 @@ message MetaServerData {
required uint32 MetaServerId = 1;
required string hostName = 2;
required string token = 3;
required string internalHostIP = 4;
required string internalIP = 4;
required uint32 internalPort = 5;
required string externalHostIP = 6;
required string externalIP = 6;
required uint32 externalPort = 7;
required uint32 serverId = 8;
required uint64 diskCapacity = 9;
Expand Down Expand Up @@ -115,8 +115,8 @@ message Copyset {
message MetaServerInfo {
required uint32 metaServerID = 1;
required string hostname = 2;
required string hostIp = 3;
required uint32 port = 4;
required string internalIp = 3;
required uint32 internalPort = 4;
required string externalIp = 5;
required uint32 externalPort = 6;
required OnlineState onlineState = 7;
Expand All @@ -126,8 +126,8 @@ message MetaServerInfo {
// MetaServer message
message MetaServerRegistRequest {
required string hostName = 1;
required string hostIp = 2;
required uint32 port = 3;
required string internalIp = 2;
required uint32 internalPort = 3;
required string externalIp = 4;
required uint32 externalPort = 5;
};
Expand Down Expand Up @@ -325,9 +325,10 @@ message GetMetaServerListInCopySetsRequest {

message MetaServerLocation {
required uint32 metaServerID = 1;
required string hostIp = 2;
required uint32 port = 3;
required string internalIp = 2;
required uint32 internalport = 3;
optional string externalIp = 4;
optional uint32 externalPort = 5;
}

message CopySetServerInfo {
Expand Down
44 changes: 27 additions & 17 deletions curvefs/src/client/rpcclient/mds_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,31 @@ MdsClientImpl::CommitTx(const std::vector<PartitionTxId> &txIds) {
return static_cast<TopoStatusCode>(rc);
}

template<typename T>
void GetEndPoint(const T &info, butil::EndPoint *internal,
butil::EndPoint *external) {
const std::string &internalIp = info.internalip();
const std::string &externalIp = [&info]() {
if (info.has_externalip()) {
return info.externalip();
} else {
return info.internalip();
}
}();

const uint32_t internalPort = info.internalport();
const uint32_t externalPort = [&info]() {
if (info.has_externalport()) {
return info.externalport();
} else {
return info.internalport();
}
}();

butil::str2endpoint(internalIp.c_str(), internalPort, internal);
butil::str2endpoint(externalIp.c_str(), externalPort, external);
}

bool MdsClientImpl::GetMetaServerInfo(
const PeerAddr &addr, CopysetPeerInfo<MetaserverID> *metaserverInfo) {
std::vector<std::string> strs;
Expand Down Expand Up @@ -283,16 +308,9 @@ bool MdsClientImpl::GetMetaServerInfo(
} else {
const auto &info = response.metaserverinfo();
MetaserverID metaserverID = info.metaserverid();
std::string internalIp = info.hostip();
std::string externalIp = internalIp;
if (info.has_externalip()) {
externalIp = info.externalip();
}
uint32_t port = info.port();
butil::EndPoint internal;
butil::str2endpoint(internalIp.c_str(), port, &internal);
butil::EndPoint external;
butil::str2endpoint(externalIp.c_str(), port, &external);
GetEndPoint(info, &internal, &external);
*metaserverInfo = CopysetPeerInfo<MetaserverID>(
metaserverID, PeerAddr(internal), PeerAddr(external));
}
Expand Down Expand Up @@ -329,18 +347,10 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
CopysetPeerInfo<MetaserverID> csinfo;
::curvefs::mds::topology::MetaServerLocation csl =
info.cslocs(j);
uint16_t port = csl.port();
std::string internalIp = csl.hostip();
csinfo.peerID = csl.metaserverid();
std::string externalIp = internalIp;
if (csl.has_externalip()) {
externalIp = csl.externalip();
}

butil::EndPoint internal;
butil::str2endpoint(internalIp.c_str(), port, &internal);
butil::EndPoint external;
butil::str2endpoint(externalIp.c_str(), port, &external);
GetEndPoint(csl, &internal, &external);
csinfo.internalAddr = PeerAddr(internal);
csinfo.externalAddr = PeerAddr(external);
copysetseverl.AddCopysetPeerInfo(csinfo);
Expand Down
2 changes: 1 addition & 1 deletion curvefs/src/mds/heartbeat/copyset_conf_generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ std::string CopysetConfGenerator::BuildPeerByMetaserverId(
}

return ::curvefs::mds::topology::BuildPeerIdWithIpPort(
metaServer.GetInternalHostIp(), metaServer.GetInternalPort(), 0);
metaServer.GetInternalIp(), metaServer.GetInternalPort(), 0);
}
} // namespace heartbeat
} // namespace mds
Expand Down
4 changes: 2 additions & 2 deletions curvefs/src/mds/heartbeat/heartbeat_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,14 +181,14 @@ HeartbeatStatusCode HeartbeatManager::CheckRequest(
}

// mismatch ip address reported by metaserver and mds record
if (request.ip() != metaServer.GetInternalHostIp() ||
if (request.ip() != metaServer.GetInternalIp() ||
request.port() != metaServer.GetInternalPort()) {
LOG(ERROR) << "heartbeatManager receive heartbeat from metaServer: "
<< request.metaserverid()
<< ", but find report ip:" << request.ip()
<< ", report port:" << request.port()
<< " do not consistent with topo record ip:"
<< metaServer.GetInternalHostIp()
<< metaServer.GetInternalIp()
<< ", record port:" << metaServer.GetInternalPort();
return HeartbeatStatusCode::hbMetaServerIpPortNotMatch;
}
Expand Down
6 changes: 3 additions & 3 deletions curvefs/src/mds/schedule/topoAdapter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ bool TopoAdapterImpl::GetPeerInfo(MetaServerIdType id, PeerInfo *peerInfo) {
if ((canGetMetaServer = topo_->GetMetaServer(id, &ms)) &&
(canGetServer = topo_->GetServer(ms.GetServerId(), &server))) {
*peerInfo = PeerInfo(ms.GetId(), server.GetZoneId(), server.GetId(),
ms.GetInternalHostIp(), ms.GetInternalPort());
ms.GetInternalIp(), ms.GetInternalPort());
} else {
LOG(ERROR) << "topoAdapter can not find metaServer(" << id
<< ", res:" << canGetMetaServer
Expand Down Expand Up @@ -276,10 +276,10 @@ bool TopoAdapterImpl::MetaServerFromTopoToSchedule(
if (topo_->GetServer(origin.GetServerId(), &server)) {
out->info =
PeerInfo{origin.GetId(), server.GetZoneId(), server.GetId(),
origin.GetInternalHostIp(), origin.GetInternalPort()};
origin.GetInternalIp(), origin.GetInternalPort()};
} else {
LOG(ERROR) << "can not get server:" << origin.GetId()
<< ", ip:" << origin.GetInternalHostIp()
<< ", ip:" << origin.GetInternalIp()
<< ", port:" << origin.GetInternalPort() << " from topology";

return false;
Expand Down
8 changes: 4 additions & 4 deletions curvefs/src/mds/topology/topology.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,13 +445,13 @@ ServerIdType TopologyImpl::FindServerByHostIpPort(const std::string &hostIp,
uint32_t port) const {
ReadLockGuard rlockServer(serverMutex_);
for (auto it = serverMap_.begin(); it != serverMap_.end(); it++) {
if (it->second.GetInternalHostIp() == hostIp) {
if (it->second.GetInternalIp() == hostIp) {
if (0 == it->second.GetInternalPort()) {
return it->first;
} else if (port == it->second.GetInternalPort()) {
return it->first;
}
} else if (it->second.GetExternalHostIp() == hostIp) {
} else if (it->second.GetExternalIp() == hostIp) {
if (0 == it->second.GetExternalPort()) {
return it->first;
} else if (port == it->second.GetExternalPort()) {
Expand Down Expand Up @@ -509,7 +509,7 @@ bool TopologyImpl::GetMetaServer(const std::string &hostIp, uint32_t port,
ReadLockGuard rlockMetaServerMap(metaServerMutex_);
for (auto it = metaServerMap_.begin(); it != metaServerMap_.end(); it++) {
ReadLockGuard rlockMetaServer(it->second.GetRWLockRef());
if (it->second.GetInternalHostIp() == hostIp &&
if (it->second.GetInternalIp() == hostIp &&
it->second.GetInternalPort() == port) {
*out = it->second;
return true;
Expand Down Expand Up @@ -1331,7 +1331,7 @@ void TopologyImpl::GetMetaServersSpace(
ReadLockGuard rlockMetaServer(i.second.GetRWLockRef());
auto metaServerUsage = new curvefs::mds::topology::MetadataUsage();
metaServerUsage->set_metaserveraddr(
i.second.GetInternalHostIp() + ":" +
i.second.GetInternalIp() + ":" +
std::to_string(i.second.GetInternalPort()));
auto const& space = i.second.GetMetaServerSpace();
metaServerUsage->set_total(space.GetDiskCapacity());
Expand Down
16 changes: 8 additions & 8 deletions curvefs/src/mds/topology/topology_item.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,9 @@ bool Server::SerializeToString(std::string *value) const {
ServerData data;
data.set_serverid(id_);
data.set_hostname(hostName_);
data.set_internalhostip(internalHostIp_);
data.set_internalip(internalIp_);
data.set_internalport(internalPort_);
data.set_externalhostip(externalHostIp_);
data.set_externalip(externalIp_);
data.set_externalport(externalPort_);
data.set_zoneid(zoneId_);
data.set_poolid(poolId_);
Expand All @@ -144,9 +144,9 @@ bool Server::ParseFromString(const std::string &value) {
bool ret = data.ParseFromString(value);
id_ = data.serverid();
hostName_ = data.hostname();
internalHostIp_ = data.internalhostip();
internalIp_ = data.internalip();
internalPort_ = data.internalport();
externalHostIp_ = data.externalhostip();
externalIp_ = data.externalip();
externalPort_ = data.externalport();
zoneId_ = data.zoneid();
poolId_ = data.poolid();
Expand All @@ -158,9 +158,9 @@ bool MetaServer::SerializeToString(std::string *value) const {
data.set_metaserverid(id_);
data.set_hostname(hostName_);
data.set_token(token_);
data.set_internalhostip(internalHostIp_);
data.set_internalip(internalIp_);
data.set_internalport(internalPort_);
data.set_externalhostip(externalHostIp_);
data.set_externalip(externalIp_);
data.set_externalport(externalPort_);
data.set_serverid(serverId_);
data.set_diskcapacity(space_.GetDiskCapacity());
Expand All @@ -176,9 +176,9 @@ bool MetaServer::ParseFromString(const std::string &value) {
hostName_ = data.hostname();
token_ = data.token();
serverId_ = data.serverid();
internalHostIp_ = data.internalhostip();
internalIp_ = data.internalip();
internalPort_ = data.internalport();
externalHostIp_ = data.externalhostip();
externalIp_ = data.externalip();
externalPort_ = data.externalport();
onlineState_ = OnlineState::UNSTABLE;
space_.SetDiskCapacity(data.diskcapacity());
Expand Down
Loading