Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate litep2p into Polkadot SDK #2944

Merged
merged 59 commits into from
Apr 8, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
59 commits
Select commit Hold shift + click to select a range
70670ae
Introduce `sc-network-types` and `PeerId` for the crate
altonen Sep 25, 2023
265a674
Make Polkadot SDK generic over the networking implementation
altonen Sep 25, 2023
39a6428
Implement `litep2p` network backend for Polkadot SDK
altonen Sep 25, 2023
7472a60
Fix CI complaints
altonen Jan 18, 2024
38f0003
Apply review comments
altonen Jan 18, 2024
1ad2bdc
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Jan 22, 2024
e6fc19e
Fix clippy
altonen Jan 22, 2024
62aedb3
more clippy fixes
altonen Jan 22, 2024
39dc59c
bind to both ipv4 and ipv6
altonen Jan 22, 2024
1319e66
Apply suggestions from code review
altonen Jan 26, 2024
7d0aaee
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Jan 26, 2024
99a6705
Apply review comments
altonen Jan 26, 2024
b4cab2b
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Jan 29, 2024
0396c06
Add support for fallback request-responses
altonen Jan 29, 2024
f60faad
Apply review comments
altonen Feb 2, 2024
5f7c2a8
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Feb 2, 2024
e5bb679
fix
altonen Feb 5, 2024
2fec872
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Feb 5, 2024
b08c3d3
prdoc + workspace fix
altonen Feb 5, 2024
98379bd
fixes
altonen Feb 5, 2024
439e65e
Reimplement fallback request support
altonen Feb 8, 2024
cf2b6d5
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Feb 12, 2024
9ca02df
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Feb 14, 2024
669c428
updates
altonen Feb 27, 2024
9a68856
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Feb 27, 2024
b0e9567
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Mar 6, 2024
a0542ac
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Mar 7, 2024
7c3df5b
fixes
altonen Mar 9, 2024
83988d4
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Mar 9, 2024
cef84e6
fix metrics
altonen Mar 10, 2024
246ba63
fix prdoc
altonen Mar 10, 2024
bfbfaa0
fix url
altonen Mar 10, 2024
beae17e
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
altonen Mar 11, 2024
58722ba
fix Cargo.lock
altonen Mar 11, 2024
f212d54
Apply suggestions from code review
altonen Mar 12, 2024
33e030e
Apply suggestions from code review
dmitry-markin Mar 18, 2024
1812dc8
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
dmitry-markin Mar 18, 2024
4561871
rustfmt
dmitry-markin Mar 18, 2024
43cb163
Apply suggestion: `reserved_peer` -> `is_reserved_peer`
dmitry-markin Mar 21, 2024
d4d89c0
monor: rustfmt
dmitry-markin Mar 21, 2024
e4470e1
Upgrade litep2p identify implementation (#3785)
dmitry-markin Mar 22, 2024
62cc8b1
Merge branch 'master' into altonen-integrate-litep2p
dmitry-markin Mar 22, 2024
d88ba6d
Update `Cargo.lock`
dmitry-markin Mar 22, 2024
ddc98a2
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
lexnv Mar 27, 2024
188cf3f
node: Fix clippy
lexnv Mar 27, 2024
3496f29
bench: Remove unused imports
lexnv Mar 27, 2024
d475762
bench: Remove PeerId to fix clippy
lexnv Mar 27, 2024
73d5770
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
lexnv Apr 8, 2024
df191f4
cumulus: Merge conflict with network generic type
lexnv Apr 8, 2024
08b038c
Apply cargo fmt
lexnv Apr 8, 2024
0690577
Update prdoc/pr_2944.prdoc
lexnv Apr 8, 2024
0735488
beefy/tests: Impl missing method
lexnv Apr 8, 2024
a00b965
beefy/tests: Add #[async_trait::async_trait]
lexnv Apr 8, 2024
612cd7b
Fix clippy
lexnv Apr 8, 2024
25be974
tests: Apply rustfmt
lexnv Apr 8, 2024
02e072a
Remove unused import
lexnv Apr 8, 2024
5b9a70f
prdoc: Fix PR doc
lexnv Apr 8, 2024
f5d028a
Merge remote-tracking branch 'origin/master' into altonen-integrate-l…
lexnv Apr 8, 2024
656ac74
Remove unused bitswap (merge conflict)
lexnv Apr 8, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 13 additions & 9 deletions cumulus/client/relay-chain-minimal-node/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ async fn new_minimal_relay_chain<Block: BlockT, Network: NetworkBackend<RelayBlo
let metrics = Network::register_notification_metrics(
config.prometheus_config.as_ref().map(|cfg| &cfg.registry),
);
let peer_store_handle = net_config.peer_store_handle();

let prometheus_registry = config.prometheus_registry();
let task_manager = TaskManager::new(config.tokio_handle.clone(), prometheus_registry)?;
Expand All @@ -194,15 +195,18 @@ async fn new_minimal_relay_chain<Block: BlockT, Network: NetworkBackend<RelayBlo
let peer_set_protocol_names =
PeerSetProtocolNames::new(genesis_hash, config.chain_spec.fork_id());
let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No };
let notification_services =
peer_sets_info::<_, Network>(is_authority, &peer_set_protocol_names, metrics.clone())
.into_iter()
.map(|(config, (peerset, service))| {
net_config.add_notification_protocol(config);
(peerset, service)
})
.collect::<std::collections::HashMap<PeerSet, Box<dyn sc_network::NotificationService>>>(
);
let notification_services = peer_sets_info::<_, Network>(
is_authority,
&peer_set_protocol_names,
metrics.clone(),
Arc::clone(&peer_store_handle),
)
.into_iter()
.map(|(config, (peerset, service))| {
net_config.add_notification_protocol(config);
(peerset, service)
})
.collect::<std::collections::HashMap<PeerSet, Box<dyn sc_network::NotificationService>>>();

let request_protocol_names = ReqProtocolNames::new(genesis_hash, config.chain_spec.fork_id());
let (collation_req_receiver_v1, collation_req_receiver_v2, available_data_req_receiver) =
Expand Down
19 changes: 7 additions & 12 deletions cumulus/client/relay-chain-minimal-node/src/network.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ use sp_runtime::traits::NumberFor;

use sc_network::{
config::{
NetworkConfiguration, NonReservedPeerMode, NotificationHandshake, ProtocolId, SetConfig,
NetworkConfiguration, NonReservedPeerMode, NotificationHandshake, PeerStore, ProtocolId,
SetConfig,
},
peer_store::PeerStore,
peer_store::PeerStoreProvider,
service::traits::NetworkService,
NotificationMetrics,
};
Expand Down Expand Up @@ -53,21 +54,14 @@ pub(crate) fn build_collator_network<Network: NetworkBackend<Block, Hash>>(
best_header.hash(),
genesis_hash,
metrics,
network_config.peer_store_handle(),
);

// Since this node has no syncing, we do not want light-clients to connect to it.
// Here we set any potential light-client slots to 0.
adjust_network_config_light_in_peers(&mut network_config.network_config);

let peer_store = PeerStore::new(
network_config
.network_config
.boot_nodes
.iter()
.map(|bootnode| bootnode.peer_id.into())
.collect(),
);
let peer_store_handle = peer_store.handle();
let peer_store = network_config.take_peer_store();
spawn_handle.spawn("peer-store", Some("networking"), peer_store.run());

let network_params = sc_network::config::Params::<Block, Hash, Network> {
Expand All @@ -80,7 +74,6 @@ pub(crate) fn build_collator_network<Network: NetworkBackend<Block, Hash>>(
},
fork_id: None,
network_config,
peer_store: Arc::new(peer_store_handle),
genesis_hash,
protocol_id,
metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()),
Expand Down Expand Up @@ -150,6 +143,7 @@ fn get_block_announce_proto_config<Network: NetworkBackend<Block, Hash>>(
best_hash: Hash,
genesis_hash: Hash,
metrics: NotificationMetrics,
peer_store_handle: Arc<dyn PeerStoreProvider>,
) -> (Network::NotificationProtocolConfig, Box<dyn NotificationService>) {
let block_announces_protocol = {
let genesis_hash = genesis_hash.as_ref();
Expand Down Expand Up @@ -179,5 +173,6 @@ fn get_block_announce_proto_config<Network: NetworkBackend<Block, Hash>>(
non_reserved_mode: NonReservedPeerMode::Deny,
},
metrics,
peer_store_handle,
)
}
18 changes: 15 additions & 3 deletions polkadot/node/network/protocol/src/peer_set.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@
use derive_more::Display;
use polkadot_primitives::Hash;
use sc_network::{
config::SetConfig, service::NotificationMetrics, types::ProtocolName, NetworkBackend,
NotificationService,
config::SetConfig, peer_store::PeerStoreProvider, service::NotificationMetrics,
types::ProtocolName, NetworkBackend, NotificationService,
};
use sp_runtime::traits::Block;
use std::{
collections::{hash_map::Entry, HashMap},
ops::{Index, IndexMut},
sync::Arc,
};
use strum::{EnumIter, IntoEnumIterator};

Expand Down Expand Up @@ -70,6 +71,7 @@ impl PeerSet {
is_authority: IsAuthority,
peerset_protocol_names: &PeerSetProtocolNames,
metrics: NotificationMetrics,
peer_store_handle: Arc<dyn PeerStoreProvider>,
) -> (N::NotificationProtocolConfig, (PeerSet, Box<dyn NotificationService>)) {
// Networking layer relies on `get_main_name()` being the main name of the protocol
// for peersets and connection management.
Expand Down Expand Up @@ -99,6 +101,7 @@ impl PeerSet {
non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept,
},
metrics,
peer_store_handle,
);

(config, (PeerSet::Validation, notification_service))
Expand All @@ -122,6 +125,7 @@ impl PeerSet {
},
},
metrics,
peer_store_handle,
);

(config, (PeerSet::Collation, notification_service))
Expand Down Expand Up @@ -214,9 +218,17 @@ pub fn peer_sets_info<B: Block, N: NetworkBackend<B, <B as Block>::Hash>>(
is_authority: IsAuthority,
peerset_protocol_names: &PeerSetProtocolNames,
metrics: NotificationMetrics,
peer_store_handle: Arc<dyn PeerStoreProvider>,
) -> Vec<(N::NotificationProtocolConfig, (PeerSet, Box<dyn NotificationService>))> {
PeerSet::iter()
.map(|s| s.get_info::<B, N>(is_authority, &peerset_protocol_names, metrics.clone()))
.map(|s| {
s.get_info::<B, N>(
is_authority,
&peerset_protocol_names,
metrics.clone(),
Arc::clone(&peer_store_handle),
)
})
.collect()
}

Expand Down
23 changes: 15 additions & 8 deletions polkadot/node/service/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -813,8 +813,8 @@ pub fn new_full<
let auth_disc_publish_non_global_ips = config.network.allow_non_globals_in_dht;
let mut net_config =
sc_network::config::FullNetworkConfiguration::<_, _, Network>::new(&config.network);

let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed");
let peer_store_handle = net_config.peer_store_handle();

// Note: GrandPa is pushed before the Polkadot-specific protocols. This doesn't change
// anything in terms of behaviour, but makes the logs more consistent with the other
Expand All @@ -824,6 +824,7 @@ pub fn new_full<
grandpa::grandpa_peers_set_config::<_, Network>(
grandpa_protocol_name.clone(),
metrics.clone(),
Arc::clone(&peer_store_handle),
);
net_config.add_notification_protocol(grandpa_protocol_config);

Expand All @@ -845,6 +846,7 @@ pub fn new_full<
beefy::communication::beefy_peers_set_config::<_, Network>(
beefy_gossip_proto_name.clone(),
metrics.clone(),
Arc::clone(&peer_store_handle),
);

net_config.add_notification_protocol(beefy_notification_config);
Expand All @@ -867,13 +869,18 @@ pub fn new_full<
use polkadot_network_bridge::{peer_sets_info, IsAuthority};
let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No };

peer_sets_info::<_, Network>(is_authority, &peerset_protocol_names, metrics.clone())
.into_iter()
.map(|(config, (peerset, service))| {
net_config.add_notification_protocol(config);
(peerset, service)
})
.collect::<HashMap<PeerSet, Box<dyn sc_network::NotificationService>>>()
peer_sets_info::<_, Network>(
is_authority,
&peerset_protocol_names,
metrics.clone(),
Arc::clone(&peer_store_handle),
)
.into_iter()
.map(|(config, (peerset, service))| {
net_config.add_notification_protocol(config);
(peerset, service)
})
.collect::<HashMap<PeerSet, Box<dyn sc_network::NotificationService>>>()
} else {
std::collections::HashMap::new()
};
Expand Down
4 changes: 3 additions & 1 deletion substrate/bin/node-template/node/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ pub fn new_full<
>::new(&config.network);
let metrics = N::register_notification_metrics(config.prometheus_registry());

let peer_store_handle = net_config.peer_store_handle();
let grandpa_protocol_name = sc_consensus_grandpa::protocol_standard_name(
&client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"),
&config.chain_spec,
Expand All @@ -149,6 +150,7 @@ pub fn new_full<
sc_consensus_grandpa::grandpa_peers_set_config::<_, N>(
grandpa_protocol_name.clone(),
metrics.clone(),
peer_store_handle,
);
net_config.add_notification_protocol(grandpa_protocol_config);

Expand Down Expand Up @@ -212,7 +214,7 @@ pub fn new_full<
};

let _rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams {
network: Arc::new(network.clone()), // TODO: remove if possible
network: Arc::new(network.clone()),
client: client.clone(),
keystore: keystore_container.keystore(),
task_manager: &mut task_manager,
Expand Down
9 changes: 8 additions & 1 deletion substrate/bin/node/cli/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -430,10 +430,15 @@ pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
let mut net_config =
sc_network::config::FullNetworkConfiguration::<_, _, N>::new(&config.network);
let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed");
let peer_store_handle = net_config.peer_store_handle();

let grandpa_protocol_name = grandpa::protocol_standard_name(&genesis_hash, &config.chain_spec);
let (grandpa_protocol_config, grandpa_notification_service) =
grandpa::grandpa_peers_set_config::<_, N>(grandpa_protocol_name.clone(), metrics.clone());
grandpa::grandpa_peers_set_config::<_, N>(
grandpa_protocol_name.clone(),
metrics.clone(),
Arc::clone(&peer_store_handle),
);
net_config.add_notification_protocol(grandpa_protocol_config);

let beefy_gossip_proto_name =
Expand Down Expand Up @@ -463,6 +468,7 @@ pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
genesis_hash,
config.chain_spec.fork_id(),
metrics.clone(),
Arc::clone(&peer_store_handle),
);
net_config.add_notification_protocol(statement_config);

Expand All @@ -473,6 +479,7 @@ pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
mixnet_protocol_name.clone(),
mixnet_config,
metrics.clone(),
Arc::clone(&peer_store_handle),
);
net_config.add_notification_protocol(config);
notification_service
Expand Down
15 changes: 10 additions & 5 deletions substrate/client/authority-discovery/src/worker/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,12 +165,17 @@ impl NetworkSigner for TestNetwork {

fn verify(
&self,
_: sc_network_types::PeerId,
_: &Vec<u8>,
_: &Vec<u8>,
_: &Vec<u8>,
peer_id: sc_network_types::PeerId,
public_key: &Vec<u8>,
signature: &Vec<u8>,
message: &Vec<u8>,
) -> std::result::Result<bool, String> {
unimplemented!();
let public_key = libp2p::identity::PublicKey::try_decode_protobuf(&public_key)
.map_err(|error| error.to_string())?;
let peer_id: PeerId = peer_id.into();
let remote: libp2p::PeerId = public_key.to_peer_id();

Ok(peer_id == remote && public_key.verify(message, signature))
}
}

Expand Down
2 changes: 2 additions & 0 deletions substrate/client/consensus/beefy/src/communication/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ pub fn beefy_peers_set_config<
>(
gossip_protocol_name: sc_network::ProtocolName,
metrics: sc_network::service::NotificationMetrics,
peer_store_handle: std::sync::Arc<dyn sc_network::peer_store::PeerStoreProvider>,
) -> (N::NotificationProtocolConfig, Box<dyn sc_network::NotificationService>) {
let (cfg, notification_service) = N::notification_config(
gossip_protocol_name,
Expand All @@ -84,6 +85,7 @@ pub fn beefy_peers_set_config<
non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept,
},
metrics,
peer_store_handle,
);
(cfg, notification_service)
}
Expand Down
2 changes: 2 additions & 0 deletions substrate/client/consensus/grandpa/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,7 @@ pub struct GrandpaParams<Block: BlockT, C, N, S, SC, VR> {
pub fn grandpa_peers_set_config<B: BlockT, N: NetworkBackend<B, <B as BlockT>::Hash>>(
protocol_name: ProtocolName,
metrics: sc_network::service::NotificationMetrics,
peer_store_handle: Arc<dyn sc_network::peer_store::PeerStoreProvider>,
) -> (N::NotificationProtocolConfig, Box<dyn NotificationService>) {
use communication::grandpa_protocol_name;
N::notification_config(
Expand All @@ -725,6 +726,7 @@ pub fn grandpa_peers_set_config<B: BlockT, N: NetworkBackend<B, <B as BlockT>::H
non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny,
},
metrics,
peer_store_handle,
)
}

Expand Down
12 changes: 11 additions & 1 deletion substrate/client/mixnet/src/protocol.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use super::config::Config;
use mixnet::core::PACKET_SIZE;
use sc_network::{
config::{NonReservedPeerMode, SetConfig},
peer_store::PeerStoreProvider,
service::NotificationMetrics,
NetworkBackend, NotificationService, ProtocolName,
};
Expand All @@ -40,6 +41,7 @@ pub fn peers_set_config<Block: BlockT, Network: NetworkBackend<Block, <Block as
name: ProtocolName,
config: &Config,
metrics: NotificationMetrics,
peerstore_handle: std::sync::Arc<dyn PeerStoreProvider>,
) -> (Network::NotificationProtocolConfig, Box<dyn NotificationService>) {
let set_config = if config.substrate.num_gateway_slots != 0 {
// out_peers is always 0; we are only interested in connecting to mixnodes, which we do by
Expand All @@ -59,5 +61,13 @@ pub fn peers_set_config<Block: BlockT, Network: NetworkBackend<Block, <Block as
}
};

Network::notification_config(name, Vec::new(), PACKET_SIZE as u64, None, set_config, metrics)
Network::notification_config(
name,
Vec::new(),
PACKET_SIZE as u64,
None,
set_config,
metrics,
peerstore_handle,
)
}
Loading
Loading