From 994b9da1dda667c11b392868c0700a0b6d7f01a4 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 7 Jan 2025 16:47:37 +0800 Subject: [PATCH 1/4] feat(f3): F3 sidecar should work with Kademlia disabled --- f3-sidecar/p2p.go | 20 ++++++++++++++++---- scripts/tests/calibnet_no_discovery_check.sh | 5 +++++ scripts/tests/calibnet_other_check.sh | 5 +++++ src/libp2p/discovery.rs | 15 +++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/f3-sidecar/p2p.go b/f3-sidecar/p2p.go index 969861131a7b..7818e71d7e0f 100644 --- a/f3-sidecar/p2p.go +++ b/f3-sidecar/p2p.go @@ -14,9 +14,10 @@ import ( const ListenAddr = "/ip4/127.0.0.1/tcp/0" type P2PHost struct { - Host host.Host - DHT *dht.IpfsDHT - PubSub *pubsub.PubSub + Host host.Host + DHT *dht.IpfsDHT + BackupDHT *dht.IpfsDHT + PubSub *pubsub.PubSub } func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) { @@ -36,6 +37,17 @@ func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) { return nil, err } + backupDthOpts := []dht.Option{ + dht.Mode(dht.ModeAutoServer), + dht.ProtocolPrefix(protocol.ID(fmt.Sprintf("/fil/kad/f3-sidecar/%s", networkName))), + dht.DisableProviders(), + dht.DisableValues(), + } + backupHostDHT, err := dht.New(ctx, host, backupDthOpts...) + if err != nil { + return nil, err + } + ps, err := pubsub.NewGossipSub(ctx, host, pubsub.WithPeerExchange(true), pubsub.WithFloodPublish(true), @@ -44,5 +56,5 @@ func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) { return nil, err } - return &P2PHost{host, hostDHT, ps}, nil + return &P2PHost{host, hostDHT, backupHostDHT, ps}, nil } diff --git a/scripts/tests/calibnet_no_discovery_check.sh b/scripts/tests/calibnet_no_discovery_check.sh index 813f2c590ed3..199decc6b9d6 100755 --- a/scripts/tests/calibnet_no_discovery_check.sh +++ b/scripts/tests/calibnet_no_discovery_check.sh @@ -13,3 +13,8 @@ export FULLNODE_API_INFO until $FOREST_CLI_PATH net peers | grep "calib"; do sleep 1s; done + +# Verify F3 is getting certificates from the network +until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -gt 100 ]]; do + sleep 1s; +done diff --git a/scripts/tests/calibnet_other_check.sh b/scripts/tests/calibnet_other_check.sh index c692bf3f7f49..d7e3887d0a14 100755 --- a/scripts/tests/calibnet_other_check.sh +++ b/scripts/tests/calibnet_other_check.sh @@ -40,3 +40,8 @@ echo "Test subcommand: net info" $FOREST_CLI_PATH net info $FOREST_CLI_PATH sync wait # allow the node to re-sync + +# Verify F3 is getting certificates from the network +until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -gt 100 ]]; do + sleep 1s; +done diff --git a/src/libp2p/discovery.rs b/src/libp2p/discovery.rs index 1f21a9f7fe9a..02bc81a8312a 100644 --- a/src/libp2p/discovery.rs +++ b/src/libp2p/discovery.rs @@ -35,6 +35,8 @@ use crate::utils::version::FOREST_VERSION_STRING; pub struct DerivedDiscoveryBehaviour { /// Kademlia discovery. kademlia: Toggle>, + /// Kademlia discovery for bootstrapping F3 sidecar when the main Kademlia is disabled. + kademlia_f3_sidecar: kad::Behaviour, /// Discovers nodes on the local network. mdns: Toggle, /// [`identify::Behaviour`] needs to be manually hooked up with [`kad::Behaviour`] to make discovery work. See @@ -158,6 +160,12 @@ impl<'a> DiscoveryConfig<'a> { } else { None }; + let kademlia_f3_sidecar = new_kademlia( + local_peer_id, + StreamProtocol::try_from_owned(format!( + "/fil/kad/f3-sidecar/{network_name}/kad/1.0.0" + ))?, + ); let mdns_opt = if enable_mdns { Some(Mdns::new(Default::default(), local_peer_id).expect("Could not start mDNS")) @@ -168,6 +176,7 @@ impl<'a> DiscoveryConfig<'a> { Ok(DiscoveryBehaviour { discovery: DerivedDiscoveryBehaviour { kademlia: kademlia_opt.into(), + kademlia_f3_sidecar, mdns: mdns_opt.into(), identify: identify::Behaviour::new( identify::Config::new("ipfs/0.1.0".into(), local_public_key) @@ -441,6 +450,11 @@ impl NetworkBehaviour for DiscoveryBehaviour { kademlia.add_address(peer_id, address.clone()); } } + for address in &info.listen_addrs { + self.discovery + .kademlia_f3_sidecar + .add_address(peer_id, address.clone()); + } } } DerivedDiscoveryBehaviourEvent::Autonat(_) => {} @@ -470,6 +484,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { trace!("Libp2p => Unhandled Kademlia event: {:?}", other) } }, + DerivedDiscoveryBehaviourEvent::KademliaF3Sidecar(_) => {} DerivedDiscoveryBehaviourEvent::Mdns(ev) => match ev { MdnsEvent::Discovered(list) => { if self.n_node_connected >= self.target_peer_count { From cdd00faa73934417586cd6407006e1ab08c6dd2b Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 7 Jan 2025 17:21:33 +0800 Subject: [PATCH 2/4] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 091cbf439928..db0e56032249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ ### Fixed +- [#5111](https://github.com/ChainSafe/forest/issues/5111) Make F3 work when the node Kademlia is disabled. + ## Forest v.0.23.3 "Plumber" Mandatory release for calibnet node operators. It fixes a sync error at epoch 2281645. From 2004a19d0d5a11b6ab2a8627414dc4a7a49c87d8 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 7 Jan 2025 22:59:18 +0800 Subject: [PATCH 3/4] fix test scripts --- scripts/tests/calibnet_no_discovery_check.sh | 4 +++- scripts/tests/calibnet_other_check.sh | 5 ----- src/f3/mod.rs | 1 + 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/tests/calibnet_no_discovery_check.sh b/scripts/tests/calibnet_no_discovery_check.sh index 199decc6b9d6..e14dab392cf8 100755 --- a/scripts/tests/calibnet_no_discovery_check.sh +++ b/scripts/tests/calibnet_no_discovery_check.sh @@ -5,7 +5,8 @@ set -euxo pipefail source "$(dirname "$0")/harness.sh" -$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" --detach --save-token ./admin_token +$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" --save-token ./admin_token & +FOREST_NODE_PID=$! FULLNODE_API_INFO="$(cat admin_token):/ip4/127.0.0.1/tcp/2345/http" export FULLNODE_API_INFO @@ -18,3 +19,4 @@ done until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -gt 100 ]]; do sleep 1s; done +kill -KILL $FOREST_NODE_PID diff --git a/scripts/tests/calibnet_other_check.sh b/scripts/tests/calibnet_other_check.sh index d7e3887d0a14..c692bf3f7f49 100755 --- a/scripts/tests/calibnet_other_check.sh +++ b/scripts/tests/calibnet_other_check.sh @@ -40,8 +40,3 @@ echo "Test subcommand: net info" $FOREST_CLI_PATH net info $FOREST_CLI_PATH sync wait # allow the node to re-sync - -# Verify F3 is getting certificates from the network -until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -gt 100 ]]; do - sleep 1s; -done diff --git a/src/f3/mod.rs b/src/f3/mod.rs index 3e4186e7dfff..d23fa786ffd3 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -100,6 +100,7 @@ pub fn run_f3_sidecar_if_enabled( if is_sidecar_ffi_enabled(chain_config) { #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { + tracing::info!("Starting F3 sidecar service ..."); GoF3NodeImpl::run( _rpc_endpoint, _jwt, From 9b75b0ef268a8f2058d887f083be5487fedff65d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 8 Jan 2025 00:33:30 +0800 Subject: [PATCH 4/4] no sleep --- scripts/tests/calibnet_no_discovery_check.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/tests/calibnet_no_discovery_check.sh b/scripts/tests/calibnet_no_discovery_check.sh index 516cf4949330..879fec318ab1 100755 --- a/scripts/tests/calibnet_no_discovery_check.sh +++ b/scripts/tests/calibnet_no_discovery_check.sh @@ -5,9 +5,10 @@ set -euxo pipefail source "$(dirname "$0")/harness.sh" -$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" --save-token ./admin_token & +$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --save-token ./admin_token --exit-after-init +$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" & FOREST_NODE_PID=$! -sleep 10s # to allow admin_token being saved + FULLNODE_API_INFO="$(cat admin_token):/ip4/127.0.0.1/tcp/2345/http" export FULLNODE_API_INFO @@ -21,4 +22,4 @@ until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -g sleep 1s; done kill -KILL $FOREST_NODE_PID -sleep 5s +wait $FOREST_NODE_PID || true