Include signed ip address in TIER2 handshake. #8902 (#9100)

* Algorithm Design: Sign, Verify Ip Address With Interface SignedIpAddress
* Implemented Protocol Buffer Serialization Deserialization For std::net::IpAddr and SignedIpAddress
* Implemented serialization deserialization of Handshake message containing SignedIpAddress for Protocol Buffer
* Implemented and unit tested verifying ip address is properly signed and properly handles errors otherwise
* Tested backwards compatibility with lack of signature for ip address

chain/network/src/network_protocol/borsh_conv.rs

@@ -14,6 +14,7 @@ impl From<&net::Handshake> for mem::Handshake {
             sender_chain_info: x.sender_chain_info.clone(),
             partial_edge_info: x.partial_edge_info.clone(),
             owned_account: None,
+            signed_ip_address: None, // borsh isn't backwards compatible. Hence, no support for signing ip address
         }
     }
 }

chain/network/src/network_protocol/mod.rs

@@ -90,6 +90,36 @@ impl std::str::FromStr for PeerAddr {
     }
 }
 
+/// Proof that a given peer_id owns an ip address, included in Handshake message
+#[derive(Clone, PartialEq, Eq, Debug, Hash)]
+pub struct SignedIpAddress {
+    pub ip_address: std::net::IpAddr,
+    pub(crate) signature: near_crypto::Signature, // signature for signed ip_address
+}
+
+impl SignedIpAddress {
+    pub fn new(ip_address: std::net::IpAddr, secret_key: &near_crypto::SecretKey) -> Self {
+        let signature = secret_key.sign(&SignedIpAddress::ip_bytes_helper(&ip_address));
+        Self { ip_address: ip_address, signature: signature }
+    }
+
+    pub fn verify(&self, public_key: &PublicKey) -> bool {
+        self.signature.verify(&self.ip_bytes(), &public_key)
+    }
+
+    fn ip_bytes_helper(ip_address: &std::net::IpAddr) -> Vec<u8> {
+        let ip_bytes: Vec<u8> = match ip_address {
+            std::net::IpAddr::V4(ip) => ip.octets().to_vec(),
+            std::net::IpAddr::V6(ip) => ip.octets().to_vec(),
+        };
+        return ip_bytes;
+    }
+
+    fn ip_bytes(&self) -> Vec<u8> {
+        return SignedIpAddress::ip_bytes_helper(&self.ip_address);
+    }
+}
+
 /// AccountData is a piece of global state that a validator
 /// signs and broadcasts to the network. It is essentially
 /// the data that a validator wants to share with the network.
@@ -307,6 +337,8 @@ pub struct Handshake {
     pub(crate) partial_edge_info: PartialEdgeInfo,
     /// Account owned by the sender.
     pub(crate) owned_account: Option<SignedOwnedAccount>,
+    /// Signed Ip Address of the sender
+    pub(crate) signed_ip_address: Option<SignedIpAddress>,
 }
 
 #[derive(PartialEq, Eq, Clone, Debug, strum::IntoStaticStr)]

chain/network/src/network_protocol/network.proto

@@ -168,6 +168,8 @@ message Handshake {
   PartialEdgeInfo partial_edge_info = 7;
   // See description of OwnedAccount.
   AccountKeySignedPayload owned_account = 8; // optional
+  // Sender's signature to verify ip address of the sender
+  SignedIpAddr signed_ip_addr = 9; // required, but marked as optional for protocol buffer backward compatibility
 }
 
 // Response to Handshake, in case the Handshake was rejected.
@@ -199,6 +201,16 @@ message LastEdge {
   Edge edge = 1;
 }
 
+message IpAddr {
+  // IPv4 (4 bytes) or IPv6 (16 bytes) in network byte order.
+  bytes ip = 1;
+}
+
+message SignedIpAddr {
+  IpAddr ip_addr = 1;
+  Signature signature = 2;
+}
+
 message SocketAddr {
   // IPv4 (4 bytes) or IPv6 (16 bytes) in network byte order.
   bytes ip = 1;

chain/network/src/network_protocol/proto_conv/handshake.rs

@@ -77,6 +77,8 @@ pub enum ParseHandshakeError {
     PartialEdgeInfo(ParseRequiredError<ParsePartialEdgeInfoError>),
     #[error("owned_account {0}")]
     OwnedAccount(ParseSignedOwnedAccountError),
+    #[error("signed_ip_address {0}")]
+    SignedIpAddress(ParseSignedIpAddrError),
 }
 
 impl From<&Handshake> for proto::Handshake {
@@ -90,6 +92,7 @@ impl From<&Handshake> for proto::Handshake {
             sender_chain_info: MF::some((&x.sender_chain_info).into()),
             partial_edge_info: MF::some((&x.partial_edge_info).into()),
             owned_account: x.owned_account.as_ref().map(Into::into).into(),
+            signed_ip_addr: x.signed_ip_address.as_ref().map(Into::into).into(),
             ..Self::default()
         }
     }
@@ -120,6 +123,8 @@ impl TryFrom<&proto::Handshake> for Handshake {
                 .map_err(Self::Error::PartialEdgeInfo)?,
             owned_account: try_from_optional(&p.owned_account)
                 .map_err(Self::Error::OwnedAccount)?,
+            signed_ip_address: try_from_optional(&p.signed_ip_addr)
+                .map_err(Self::Error::SignedIpAddress)?,
         })
     }
 }

chain/network/src/network_protocol/proto_conv/net.rs

@@ -3,12 +3,76 @@ use super::*;
 
 use crate::network_protocol::proto;
 use crate::network_protocol::PeerAddr;
-use crate::network_protocol::{Edge, PartialEdgeInfo, PeerInfo};
+use crate::network_protocol::{Edge, PartialEdgeInfo, PeerInfo, SignedIpAddress};
 use borsh::{BorshDeserialize as _, BorshSerialize as _};
 use near_primitives::network::AnnounceAccount;
 use protobuf::MessageField as MF;
 use std::net::{IpAddr, SocketAddr};
 
+////////////////////////////////////////
+// Parse std::net::IpAddr to Protocol Buffer and back
+#[derive(thiserror::Error, Debug)]
+pub enum ParseIpAddrError {
+    #[error("invalid IP")]
+    InvalidIP,
+}
+
+impl From<&std::net::IpAddr> for proto::IpAddr {
+    fn from(x: &std::net::IpAddr) -> Self {
+        Self {
+            ip: match x {
+                std::net::IpAddr::V4(ip) => ip.octets().to_vec(),
+                std::net::IpAddr::V6(ip) => ip.octets().to_vec(),
+            },
+            ..Self::default()
+        }
+    }
+}
+
+impl TryFrom<&proto::IpAddr> for std::net::IpAddr {
+    type Error = ParseIpAddrError;
+    fn try_from(x: &proto::IpAddr) -> Result<Self, Self::Error> {
+        let ip = match x.ip.len() {
+            4 => IpAddr::from(<[u8; 4]>::try_from(&x.ip[..]).unwrap()),
+            16 => IpAddr::from(<[u8; 16]>::try_from(&x.ip[..]).unwrap()),
+            _ => return Err(Self::Error::InvalidIP),
+        };
+        Ok(ip)
+    }
+}
+
+////////////////////////////////////////
+// Parse SignedIpAddr to Protocol Buffer and back
+#[derive(thiserror::Error, Debug)]
+pub enum ParseSignedIpAddrError {
+    #[error("ip_addr: {0}")]
+    IpAddr(ParseRequiredError<ParseIpAddrError>),
+    #[error("signed_owned_ip_address: {0}")]
+    Signature(ParseRequiredError<ParseSignatureError>),
+}
+
+impl From<&SignedIpAddress> for proto::SignedIpAddr {
+    fn from(x: &SignedIpAddress) -> Self {
+        Self {
+            ip_addr: MF::some((&x.ip_address).into()),
+            signature: MF::some((&x.signature).into()),
+            ..Self::default()
+        }
+    }
+}
+
+impl TryFrom<&proto::SignedIpAddr> for SignedIpAddress {
+    type Error = ParseSignedIpAddrError;
+    fn try_from(p: &proto::SignedIpAddr) -> Result<Self, Self::Error> {
+        Ok(Self {
+            ip_address: try_from_required(&p.ip_addr).map_err(Self::Error::IpAddr)?,
+            signature: try_from_required(&p.signature).map_err(Self::Error::Signature)?,
+        })
+    }
+}
+
+////////////////////////////////////////
+
 #[derive(thiserror::Error, Debug)]
 pub enum ParseSocketAddrError {
     #[error("invalid IP")]

chain/network/src/network_protocol/testonly.rs

@@ -332,23 +332,44 @@ impl Chain {
     }
 }
 
-pub fn make_handshake<R: Rng>(rng: &mut R, chain: &Chain) -> Handshake {
-    let a = make_signer(rng);
-    let b = make_signer(rng);
-    let a_id = PeerId::new(a.public_key);
-    let b_id = PeerId::new(b.public_key);
+pub fn make_signed_ip_addr(
+    ip_addr: &std::net::IpAddr,
+    secret_key: &near_crypto::SecretKey,
+) -> SignedIpAddress {
+    let signed_ip_address: SignedIpAddress = SignedIpAddress::new(*ip_addr, secret_key);
+    return signed_ip_address;
+}
+
+pub fn make_handshake_with_ip<R: Rng>(
+    rng: &mut R,
+    chain: &Chain,
+    ip_addr: Option<std::net::IpAddr>,
+) -> Handshake {
+    let sender = make_signer(rng);
+    let target = make_signer(rng);
+    let sender_id = PeerId::new(sender.public_key);
+    let target_id = PeerId::new(target.public_key);
+    let signed_ip_address = match ip_addr {
+        Some(ip) => Some(make_signed_ip_addr(&ip, &sender.secret_key)),
+        _ => None,
+    };
     Handshake {
         protocol_version: version::PROTOCOL_VERSION,
         oldest_supported_version: version::PEER_MIN_ALLOWED_PROTOCOL_VERSION,
-        sender_peer_id: a_id,
-        target_peer_id: b_id,
+        sender_peer_id: sender_id,
+        target_peer_id: target_id,
         sender_listen_port: Some(rng.gen()),
         sender_chain_info: chain.get_peer_chain_info(),
         partial_edge_info: make_partial_edge(rng),
         owned_account: None,
+        signed_ip_address: signed_ip_address,
     }
 }
 
+pub fn make_handshake<R: Rng>(rng: &mut R, chain: &Chain) -> Handshake {
+    make_handshake_with_ip(rng, chain, None)
+}
+
 pub fn make_routed_message<R: Rng>(rng: &mut R, body: RoutedMessageBody) -> RoutedMessageV2 {
     let signer = make_signer(rng);
     let peer_id = PeerId::new(signer.public_key);

chain/network/src/network_protocol/tests.rs

@@ -7,7 +7,10 @@ use crate::types::{PartialEncodedChunkRequestMsg, PartialEncodedChunkResponseMsg
 use anyhow::{bail, Context as _};
 use itertools::Itertools as _;
 use near_async::time;
+use near_crypto::{KeyType, SecretKey};
+use near_primitives::network::PeerId;
 use rand::Rng as _;
+use std::net;
 
 #[test]
 fn deduplicate_edges() {
@@ -61,8 +64,11 @@ fn serialize_deserialize_protobuf_only() {
     let mut rng = make_rng(39521947542);
     let mut clock = time::FakeClock::default();
     let chain = data::Chain::make(&mut clock, &mut rng, 12);
+    let ip_addr: net::IpAddr = data::make_ipv4(&mut rng);
     let msgs = [
-        PeerMessage::Tier1Handshake(data::make_handshake(&mut rng, &chain)),
+        PeerMessage::Tier1Handshake(data::make_handshake_with_ip(&mut rng, &chain, Some(ip_addr))), // with ip address
+        PeerMessage::Tier1Handshake(data::make_handshake(&mut rng, &chain)), // without ip address, temporarily supported for backwards compatibility migration
+        PeerMessage::Tier2Handshake(data::make_handshake_with_ip(&mut rng, &chain, Some(ip_addr))), // Tier2 Handshake does not maintain ip address with borsh serialization
         PeerMessage::SyncAccountsData(SyncAccountsData {
             accounts_data: (0..4)
                 .map(|_| Arc::new(data::make_signed_account_data(&mut rng, &clock.clock())))
@@ -107,7 +113,7 @@ fn serialize_deserialize() -> anyhow::Result<()> {
         }),
     ));
     let msgs = [
-        PeerMessage::Tier2Handshake(data::make_handshake(&mut rng, &chain)),
+        PeerMessage::Tier2Handshake(data::make_handshake(&mut rng, &chain)), // without ip address, temporarily supported for backwards compatibility migration
         PeerMessage::HandshakeFailure(
             data::make_peer_info(&mut rng),
             HandshakeFailureReason::InvalidTarget,
@@ -170,3 +176,15 @@ fn serialize_deserialize() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn sign_and_verify_with_signed_ip_address_interface() {
+    let node_key = SecretKey::from_seed(KeyType::ED25519, "123");
+    let peer_id = PeerId::new(node_key.public_key());
+    let mut rng = make_rng(89028037453);
+    let ip_addr: net::IpAddr = data::make_ipv4(&mut rng);
+    // Wrap ip address sign and verify algorithm with interfaces SignedIpAddress
+    let signed_ip_address: SignedIpAddress = SignedIpAddress::new(ip_addr, &node_key);
+    assert!(signed_ip_address.verify(&node_key.public_key()));
+    assert!(signed_ip_address.verify(peer_id.public_key()));
+}

chain/network/src/peer/peer_actor.rs

@@ -2,6 +2,7 @@ use crate::accounts_data;
 use crate::concurrency::atomic_cell::AtomicCell;
 use crate::concurrency::demux;
 use crate::config::PEERS_RESPONSE_MAX_PEERS;
+use crate::network_protocol::SignedIpAddress;
 use crate::network_protocol::{
     Edge, EdgeState, Encoding, OwnedAccount, ParsePeerMessageError, PartialEdgeInfo,
     PeerChainInfoV2, PeerIdOrHash, PeerInfo, PeersRequest, PeersResponse, RawRoutedMessage,
@@ -110,6 +111,8 @@ pub(crate) enum ClosingReason {
     TooLargeClockSkew,
     #[error("owned_account.peer_id doesn't match handshake.sender_peer_id")]
     OwnedAccountMismatch,
+    #[error("signed_ip_address's peer address doesn't match tcp stream's peer_addr")]
+    IpAddressMismatch,
     #[error("PeerActor stopped NOT via PeerActor::stop()")]
     Unknown,
 }
@@ -132,6 +135,7 @@ impl ClosingReason {
             ClosingReason::TooLargeClockSkew => true, // reconnect will fail for the same reason
             ClosingReason::OwnedAccountMismatch => true, // misbehaving peer
             ClosingReason::Unknown => false,        // only happens in tests
+            ClosingReason::IpAddressMismatch => true, // invalid ip address or signature must be banned
         }
     }
 }
@@ -286,7 +290,7 @@ impl PeerActor {
         };
         let my_node_info = PeerInfo {
             id: network_state.config.node_id(),
-            addr: network_state.config.node_addr.as_ref().map(|a| **a),
+            addr: Some(stream.local_addr),
             account_id: network_state.config.validator.as_ref().map(|v| v.account_id()),
         };
         // recv is the HandshakeSignal returned by this spawn_inner() call.
@@ -420,6 +424,10 @@ impl PeerActor {
             } else {
                 (0, vec![])
             };
+        let my_signed_ip_address = SignedIpAddress::new(
+            self.my_node_info.addr.unwrap().ip(),
+            &self.network_state.config.node_key,
+        );
         let handshake = Handshake {
             protocol_version: spec.protocol_version,
             oldest_supported_version: PEER_MIN_ALLOWED_PROTOCOL_VERSION,
@@ -442,6 +450,7 @@ impl PeerActor {
                 }
                 .sign(vc.signer.as_ref())
             }),
+            signed_ip_address: Some(my_signed_ip_address),
         };
         let msg = match spec.tier {
             tcp::Tier::T1 => PeerMessage::Tier1Handshake(handshake),
@@ -565,6 +574,20 @@ impl PeerActor {
             }
         }
 
+        // Verify the signed IP address is valid.
+        if let Some(signed_ip_address) = &handshake.signed_ip_address {
+            if self.peer_addr.ip() != signed_ip_address.ip_address {
+                self.stop(ctx, ClosingReason::IpAddressMismatch);
+                return;
+            }
+            // Verify signature of the sender on its ip address
+            if !(signed_ip_address.verify(handshake.sender_peer_id.public_key())) {
+                self.stop(ctx, ClosingReason::Ban(ReasonForBan::InvalidSignature));
+                return;
+            }
+        } // else do nothing as temporary leniency for backward compatibility purposes
+          // TODO(soon): Fail the handshake if its doesn't include the required signed_ip_address after all production nodes have upgraded to latest handshake protocol
+
         // Verify that handshake.owned_account is valid.
         if let Some(owned_account) = &handshake.owned_account {
             if let Err(_) = owned_account.payload().verify(&owned_account.account_key) {