fix(f3): F3 sidecar should work with Kademlia disabled

CHANGELOG.md

@@ -35,6 +35,8 @@
 
 ### Fixed
 
+- [#5111](https://github.com/ChainSafe/forest/issues/5111) Make F3 work when the node Kademlia is disabled.
+
 ## Forest v.0.23.3 "Plumber"
 
 Mandatory release for calibnet node operators. It fixes a sync error at epoch 2281645.

f3-sidecar/p2p.go

@@ -14,9 +14,10 @@ import (
 const ListenAddr = "/ip4/127.0.0.1/tcp/0"
 
 type P2PHost struct {
-	Host   host.Host
-	DHT    *dht.IpfsDHT
-	PubSub *pubsub.PubSub
+	Host      host.Host
+	DHT       *dht.IpfsDHT
+	BackupDHT *dht.IpfsDHT
+	PubSub    *pubsub.PubSub
 }
 
 func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) {
@@ -36,6 +37,17 @@ func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) {
 		return nil, err
 	}
 
+	backupDthOpts := []dht.Option{
+		dht.Mode(dht.ModeAutoServer),
+		dht.ProtocolPrefix(protocol.ID(fmt.Sprintf("/fil/kad/f3-sidecar/%s", networkName))),
+		dht.DisableProviders(),
+		dht.DisableValues(),
+	}
+	backupHostDHT, err := dht.New(ctx, host, backupDthOpts...)
+	if err != nil {
+		return nil, err
+	}
+
 	ps, err := pubsub.NewGossipSub(ctx, host,
 		pubsub.WithPeerExchange(true),
 		pubsub.WithFloodPublish(true),
@@ -44,5 +56,5 @@ func createP2PHost(ctx context.Context, networkName string) (*P2PHost, error) {
 		return nil, err
 	}
 
-	return &P2PHost{host, hostDHT, ps}, nil
+	return &P2PHost{host, hostDHT, backupHostDHT, ps}, nil
 }

scripts/tests/calibnet_no_discovery_check.sh

@@ -5,11 +5,20 @@ set -euxo pipefail
 
 source "$(dirname "$0")/harness.sh"
 
-$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" --detach --save-token ./admin_token
+$FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" --save-token ./admin_token &
+FOREST_NODE_PID=$!
+sleep 10s # to allow admin_token being saved
 FULLNODE_API_INFO="$(cat admin_token):/ip4/127.0.0.1/tcp/2345/http"
 export FULLNODE_API_INFO
 
 # Verify that one of the seed nodes has been connected to
 until $FOREST_CLI_PATH net peers | grep "calib"; do
     sleep 1s;
 done
+
+# Verify F3 is getting certificates from the network
+until [[ $($FOREST_CLI_PATH f3 certs get --output json | jq '.GPBFTInstance') -gt 100 ]]; do
+    sleep 1s;
+done
+kill -KILL $FOREST_NODE_PID
+sleep 5s

src/f3/mod.rs

@@ -100,6 +100,7 @@ pub fn run_f3_sidecar_if_enabled(
     if is_sidecar_ffi_enabled(chain_config) {
         #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))]
         {
+            tracing::info!("Starting F3 sidecar service ...");
             GoF3NodeImpl::run(
                 _rpc_endpoint,
                 _jwt,

src/libp2p/discovery.rs

@@ -35,6 +35,8 @@ use crate::utils::version::FOREST_VERSION_STRING;
 pub struct DerivedDiscoveryBehaviour {
     /// Kademlia discovery.
     kademlia: Toggle<kad::Behaviour<kad::store::MemoryStore>>,
+    /// Kademlia discovery for bootstrapping F3 sidecar when the main Kademlia is disabled.
+    kademlia_f3_sidecar: kad::Behaviour<kad::store::MemoryStore>,
     /// Discovers nodes on the local network.
     mdns: Toggle<Mdns>,
     /// [`identify::Behaviour`] needs to be manually hooked up with [`kad::Behaviour`] to make discovery work. See <https://docs.rs/libp2p/latest/libp2p/kad/index.html#important-discrepancies>
@@ -158,6 +160,12 @@ impl<'a> DiscoveryConfig<'a> {
         } else {
             None
         };
+        let kademlia_f3_sidecar = new_kademlia(
+            local_peer_id,
+            StreamProtocol::try_from_owned(format!(
+                "/fil/kad/f3-sidecar/{network_name}/kad/1.0.0"
+            ))?,
+        );
 
         let mdns_opt = if enable_mdns {
             Some(Mdns::new(Default::default(), local_peer_id).expect("Could not start mDNS"))
@@ -168,6 +176,7 @@ impl<'a> DiscoveryConfig<'a> {
         Ok(DiscoveryBehaviour {
             discovery: DerivedDiscoveryBehaviour {
                 kademlia: kademlia_opt.into(),
+                kademlia_f3_sidecar,
                 mdns: mdns_opt.into(),
                 identify: identify::Behaviour::new(
                     identify::Config::new("ipfs/0.1.0".into(), local_public_key)
@@ -441,6 +450,11 @@ impl NetworkBehaviour for DiscoveryBehaviour {
                                         kademlia.add_address(peer_id, address.clone());
                                     }
                                 }
+                                for address in &info.listen_addrs {
+                                    self.discovery
+                                        .kademlia_f3_sidecar
+                                        .add_address(peer_id, address.clone());
+                                }
                             }
                         }
                         DerivedDiscoveryBehaviourEvent::Autonat(_) => {}
@@ -470,6 +484,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
                                 trace!("Libp2p => Unhandled Kademlia event: {:?}", other)
                             }
                         },
+                        DerivedDiscoveryBehaviourEvent::KademliaF3Sidecar(_) => {}
                         DerivedDiscoveryBehaviourEvent::Mdns(ev) => match ev {
                             MdnsEvent::Discovered(list) => {
                                 if self.n_node_connected >= self.target_peer_count {