curvefs: The curvefs cluster has some internal service which support …

…cluster communicate itself and some external services support for client and management tools.

If all the services start on the same ip+port will affect the communication within the cluster when the heavy stress from client.
Start the services needed by outsied on another ip+port as a external server.

curvefs/conf/metaserver.conf

@@ -49,6 +49,8 @@ s3compactwq.s3infocache_size=100
 global.ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_host }} __ANSIBLE_TEMPLATE__
 global.port=16701  # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_port }} __ANSIBLE_TEMPLATE__
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_host }} __ANSIBLE_TEMPLATE__
+global.external_port=16701 # __CURVEADM_TEMPLATE__ ${service_external_port} __CURVEADM_TEMPLATE__
+global.enable_external_server=false
 
 # metaserver log directory
 # this config item can be replaced by start up option `-log_dir`

curvefs/conf/tools.conf

@@ -6,7 +6,7 @@ rpcTimeoutMs=10000
 rpcRetryTimes=5
 # topo file path
 topoFilePath=curvefs/test/tools/topo_example.json  # __CURVEADM_TEMPLATE__ /curvefs/tools/conf/topology.json __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ project_root_dest }}/conf/topology.json __ANSIBLE_TEMPLATE__
-# metaserver
+# metaserver external address
 metaserverAddr=127.0.0.1:6701  # __CURVEADM_TEMPLATE__ ${cluster_metaserver_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.metaserver | join_peer(hostvars, "metaserver_listen_port") }} __ANSIBLE_TEMPLATE__
 # etcd
 etcdAddr=127.0.0.1:12379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_peer(hostvars, "etcd_listen_client_port") }} __ANSIBLE_TEMPLATE__

curvefs/proto/topology.proto

@@ -73,9 +73,9 @@ message ZoneData {
 message ServerData {
     required uint32 serverId = 1;
     required string hostName = 2;
-    required string internalHostIP = 3;
+    required string internalIP = 3;
     required uint32 internalPort = 4;
-    required string externalHostIP = 5;
+    required string externalIP = 5;
     required uint32 externalPort = 6;
     required uint32 zoneId = 7;
     required uint32 PoolId = 8;
@@ -85,9 +85,9 @@ message MetaServerData {
     required uint32 MetaServerId = 1;
     required string hostName = 2;
     required string token = 3;
-    required string internalHostIP = 4;
+    required string internalIP = 4;
     required uint32 internalPort = 5;
-    required string externalHostIP = 6;
+    required string externalIP = 6;
     required uint32 externalPort = 7;
     required uint32 serverId = 8;
     required uint64 diskCapacity = 9;
@@ -115,8 +115,8 @@ message Copyset {
 message MetaServerInfo {
     required uint32 metaServerID = 1;
     required string hostname = 2;
-    required string hostIp = 3;
-    required uint32 port = 4;
+    required string internalIp = 3;
+    required uint32 internalPort = 4;
     required string externalIp = 5;
     required uint32 externalPort = 6;
     required OnlineState onlineState = 7;
@@ -126,8 +126,8 @@ message MetaServerInfo {
 // MetaServer message
 message MetaServerRegistRequest {
     required string hostName = 1;
-    required string hostIp = 2;
-    required uint32 port = 3;
+    required string internalIp = 2;
+    required uint32 internalPort = 3;
     required string externalIp = 4;
     required uint32 externalPort = 5;
 };
@@ -325,9 +325,10 @@ message GetMetaServerListInCopySetsRequest {
 
 message MetaServerLocation {
     required uint32 metaServerID = 1;
-    required string hostIp = 2;
-    required uint32 port = 3;
+    required string internalIp = 2;
+    required uint32 internalport = 3;
     optional string externalIp = 4;
+    optional uint32 externalPort = 5;
 }
 
 message CopySetServerInfo {

curvefs/src/client/rpcclient/mds_client.cpp

@@ -256,6 +256,31 @@ MdsClientImpl::CommitTx(const std::vector<PartitionTxId> &txIds) {
     return static_cast<TopoStatusCode>(rc);
 }
 
+template<typename T>
+void GetEndPoint(const T &info, butil::EndPoint *internal,
+    butil::EndPoint *external) {
+    const std::string &internalIp = info.internalip();
+    const std::string &externalIp = [&info]() {
+        if (info.has_externalip()) {
+            return info.externalip();
+        } else {
+            return info.internalip();
+        }
+    }();
+
+    const uint32_t internalPort = info.internalport();
+    const uint32_t externalPort = [&info]() {
+        if (info.has_externalport()) {
+            return info.externalport();
+        } else {
+            return info.internalport();
+        }
+    }();
+
+    butil::str2endpoint(internalIp.c_str(), internalPort, internal);
+    butil::str2endpoint(externalIp.c_str(), externalPort, external);
+}
+
 bool MdsClientImpl::GetMetaServerInfo(
     const PeerAddr &addr, CopysetPeerInfo<MetaserverID> *metaserverInfo) {
     std::vector<std::string> strs;
@@ -283,16 +308,9 @@ bool MdsClientImpl::GetMetaServerInfo(
         } else {
             const auto &info = response.metaserverinfo();
             MetaserverID metaserverID = info.metaserverid();
-            std::string internalIp = info.hostip();
-            std::string externalIp = internalIp;
-            if (info.has_externalip()) {
-                externalIp = info.externalip();
-            }
-            uint32_t port = info.port();
             butil::EndPoint internal;
-            butil::str2endpoint(internalIp.c_str(), port, &internal);
             butil::EndPoint external;
-            butil::str2endpoint(externalIp.c_str(), port, &external);
+            GetEndPoint(info, &internal, &external);
             *metaserverInfo = CopysetPeerInfo<MetaserverID>(
                 metaserverID, PeerAddr(internal), PeerAddr(external));
         }
@@ -329,18 +347,10 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
                 CopysetPeerInfo<MetaserverID> csinfo;
                 ::curvefs::mds::topology::MetaServerLocation csl =
                     info.cslocs(j);
-                uint16_t port = csl.port();
-                std::string internalIp = csl.hostip();
                 csinfo.peerID = csl.metaserverid();
-                std::string externalIp = internalIp;
-                if (csl.has_externalip()) {
-                    externalIp = csl.externalip();
-                }
-
                 butil::EndPoint internal;
-                butil::str2endpoint(internalIp.c_str(), port, &internal);
                 butil::EndPoint external;
-                butil::str2endpoint(externalIp.c_str(), port, &external);
+                GetEndPoint(csl, &internal, &external);
                 csinfo.internalAddr = PeerAddr(internal);
                 csinfo.externalAddr = PeerAddr(external);
                 copysetseverl.AddCopysetPeerInfo(csinfo);

curvefs/src/mds/heartbeat/copyset_conf_generator.cpp

@@ -165,7 +165,7 @@ std::string CopysetConfGenerator::BuildPeerByMetaserverId(
     }
 
     return ::curvefs::mds::topology::BuildPeerIdWithIpPort(
-        metaServer.GetInternalHostIp(), metaServer.GetInternalPort(), 0);
+        metaServer.GetInternalIp(), metaServer.GetInternalPort(), 0);
 }
 }  // namespace heartbeat
 }  // namespace mds

curvefs/src/mds/heartbeat/heartbeat_manager.cpp

@@ -181,14 +181,14 @@ HeartbeatStatusCode HeartbeatManager::CheckRequest(
     }
 
     // mismatch ip address reported by metaserver and mds record
-    if (request.ip() != metaServer.GetInternalHostIp() ||
+    if (request.ip() != metaServer.GetInternalIp() ||
         request.port() != metaServer.GetInternalPort()) {
         LOG(ERROR) << "heartbeatManager receive heartbeat from metaServer: "
                    << request.metaserverid()
                    << ", but find report ip:" << request.ip()
                    << ", report port:" << request.port()
                    << " do not consistent with topo record ip:"
-                   << metaServer.GetInternalHostIp()
+                   << metaServer.GetInternalIp()
                    << ", record port:" << metaServer.GetInternalPort();
         return HeartbeatStatusCode::hbMetaServerIpPortNotMatch;
     }

curvefs/src/mds/schedule/topoAdapter.cpp

@@ -227,7 +227,7 @@ bool TopoAdapterImpl::GetPeerInfo(MetaServerIdType id, PeerInfo *peerInfo) {
     if ((canGetMetaServer = topo_->GetMetaServer(id, &ms)) &&
         (canGetServer = topo_->GetServer(ms.GetServerId(), &server))) {
         *peerInfo = PeerInfo(ms.GetId(), server.GetZoneId(), server.GetId(),
-                             ms.GetInternalHostIp(), ms.GetInternalPort());
+                             ms.GetInternalIp(), ms.GetInternalPort());
     } else {
         LOG(ERROR) << "topoAdapter can not find metaServer(" << id
                    << ", res:" << canGetMetaServer
@@ -276,10 +276,10 @@ bool TopoAdapterImpl::MetaServerFromTopoToSchedule(
     if (topo_->GetServer(origin.GetServerId(), &server)) {
         out->info =
             PeerInfo{origin.GetId(), server.GetZoneId(), server.GetId(),
-                     origin.GetInternalHostIp(), origin.GetInternalPort()};
+                     origin.GetInternalIp(), origin.GetInternalPort()};
     } else {
         LOG(ERROR) << "can not get server:" << origin.GetId()
-                   << ", ip:" << origin.GetInternalHostIp()
+                   << ", ip:" << origin.GetInternalIp()
                    << ", port:" << origin.GetInternalPort() << " from topology";
 
         return false;

curvefs/src/mds/topology/topology.cpp

@@ -445,13 +445,13 @@ ServerIdType TopologyImpl::FindServerByHostIpPort(const std::string &hostIp,
                                                   uint32_t port) const {
     ReadLockGuard rlockServer(serverMutex_);
     for (auto it = serverMap_.begin(); it != serverMap_.end(); it++) {
-        if (it->second.GetInternalHostIp() == hostIp) {
+        if (it->second.GetInternalIp() == hostIp) {
             if (0 == it->second.GetInternalPort()) {
                 return it->first;
             } else if (port == it->second.GetInternalPort()) {
                 return it->first;
             }
-        } else if (it->second.GetExternalHostIp() == hostIp) {
+        } else if (it->second.GetExternalIp() == hostIp) {
             if (0 == it->second.GetExternalPort()) {
                 return it->first;
             } else if (port == it->second.GetExternalPort()) {
@@ -509,7 +509,7 @@ bool TopologyImpl::GetMetaServer(const std::string &hostIp, uint32_t port,
     ReadLockGuard rlockMetaServerMap(metaServerMutex_);
     for (auto it = metaServerMap_.begin(); it != metaServerMap_.end(); it++) {
         ReadLockGuard rlockMetaServer(it->second.GetRWLockRef());
-        if (it->second.GetInternalHostIp() == hostIp &&
+        if (it->second.GetInternalIp() == hostIp &&
             it->second.GetInternalPort() == port) {
             *out = it->second;
             return true;
@@ -1331,7 +1331,7 @@ void TopologyImpl::GetMetaServersSpace(
         ReadLockGuard rlockMetaServer(i.second.GetRWLockRef());
         auto metaServerUsage = new curvefs::mds::topology::MetadataUsage();
         metaServerUsage->set_metaserveraddr(
-            i.second.GetInternalHostIp() + ":" +
+            i.second.GetInternalIp() + ":" +
             std::to_string(i.second.GetInternalPort()));
         auto const& space = i.second.GetMetaServerSpace();
         metaServerUsage->set_total(space.GetDiskCapacity());

curvefs/src/mds/topology/topology_item.cpp

@@ -130,9 +130,9 @@ bool Server::SerializeToString(std::string *value) const {
     ServerData data;
     data.set_serverid(id_);
     data.set_hostname(hostName_);
-    data.set_internalhostip(internalHostIp_);
+    data.set_internalip(internalIp_);
     data.set_internalport(internalPort_);
-    data.set_externalhostip(externalHostIp_);
+    data.set_externalip(externalIp_);
     data.set_externalport(externalPort_);
     data.set_zoneid(zoneId_);
     data.set_poolid(poolId_);
@@ -144,9 +144,9 @@ bool Server::ParseFromString(const std::string &value) {
     bool ret = data.ParseFromString(value);
     id_ = data.serverid();
     hostName_ = data.hostname();
-    internalHostIp_ = data.internalhostip();
+    internalIp_ = data.internalip();
     internalPort_ = data.internalport();
-    externalHostIp_ = data.externalhostip();
+    externalIp_ = data.externalip();
     externalPort_ = data.externalport();
     zoneId_ = data.zoneid();
     poolId_ = data.poolid();
@@ -158,9 +158,9 @@ bool MetaServer::SerializeToString(std::string *value) const {
     data.set_metaserverid(id_);
     data.set_hostname(hostName_);
     data.set_token(token_);
-    data.set_internalhostip(internalHostIp_);
+    data.set_internalip(internalIp_);
     data.set_internalport(internalPort_);
-    data.set_externalhostip(externalHostIp_);
+    data.set_externalip(externalIp_);
     data.set_externalport(externalPort_);
     data.set_serverid(serverId_);
     data.set_diskcapacity(space_.GetDiskCapacity());
@@ -176,9 +176,9 @@ bool MetaServer::ParseFromString(const std::string &value) {
     hostName_ = data.hostname();
     token_ = data.token();
     serverId_ = data.serverid();
-    internalHostIp_ = data.internalhostip();
+    internalIp_ = data.internalip();
     internalPort_ = data.internalport();
-    externalHostIp_ = data.externalhostip();
+    externalIp_ = data.externalip();
     externalPort_ = data.externalport();
     onlineState_ = OnlineState::UNSTABLE;
     space_.SetDiskCapacity(data.diskcapacity());