Integration tests for chunk distribution network

chain/client/src/test_utils/setup.rs

@@ -21,7 +21,9 @@ use near_chain::state_snapshot_actor::SnapshotCallbacks;
 use near_chain::test_utils::{KeyValueRuntime, MockEpochManager, ValidatorSchedule};
 use near_chain::types::{ChainConfig, RuntimeAdapter};
 use near_chain::{Chain, ChainGenesis, DoomslugThresholdMode};
-use near_chain_configs::{ClientConfig, MutableConfigValue, ReshardingConfig};
+use near_chain_configs::{
+    ChunkDistributionNetworkConfig, ClientConfig, MutableConfigValue, ReshardingConfig,
+};
 use near_chunks::adapter::ShardsManagerRequestFromClient;
 use near_chunks::client::ShardsManagerResponse;
 use near_chunks::shards_manager_actor::start_shards_manager;
@@ -82,6 +84,7 @@ pub fn setup(
     transaction_validity_period: NumBlocks,
     genesis_time: DateTime<Utc>,
     ctx: &Context<ClientActor>,
+    chunk_distribution_config: Option<ChunkDistributionNetworkConfig>,
 ) -> (Block, ClientActor, Addr<ViewClientActor>, ShardsManagerAdapterForTest) {
     let store = create_test_store();
     let num_validator_seats = vs.all_block_producers().count() as NumSeats;
@@ -126,16 +129,20 @@ pub fn setup(
 
     let signer = Arc::new(create_test_signer(account_id.as_str()));
     let telemetry = TelemetryActor::default().start();
-    let config = ClientConfig::test(
-        skip_sync_wait,
-        min_block_prod_time,
-        max_block_prod_time,
-        num_validator_seats,
-        archive,
-        true,
-        epoch_sync_enabled,
-        state_sync_enabled,
-    );
+    let config = {
+        let mut base = ClientConfig::test(
+            skip_sync_wait,
+            min_block_prod_time,
+            max_block_prod_time,
+            num_validator_seats,
+            archive,
+            true,
+            epoch_sync_enabled,
+            state_sync_enabled,
+        );
+        base.chunk_distribution_network = chunk_distribution_config;
+        base
+    };
 
     let adv = crate::adversarial::Controls::default();
 
@@ -336,6 +343,7 @@ pub fn setup_mock_with_validity_period_and_no_epoch_sync(
             transaction_validity_period,
             StaticClock::utc(),
             ctx,
+            None,
         );
         vca = Some(view_client_addr);
         sma = Some(shards_manager_adapter);
@@ -435,6 +443,7 @@ pub fn setup_mock_all_validators(
     archive: Vec<bool>,
     epoch_sync_enabled: Vec<bool>,
     check_block_stats: bool,
+    chunk_distribution_config: Option<ChunkDistributionNetworkConfig>,
     peer_manager_mock: Box<
         dyn FnMut(
             // Peer validators
@@ -485,6 +494,7 @@ pub fn setup_mock_all_validators(
         let hash_to_height1 = hash_to_height.clone();
         let archive1 = archive.clone();
         let epoch_sync_enabled1 = epoch_sync_enabled.clone();
+        let chunk_distribution_config1 = chunk_distribution_config.clone();
         let client_addr = ClientActor::create(|ctx| {
             let client_addr = ctx.address();
             let _account_id = account_id.clone();
@@ -860,6 +870,7 @@ pub fn setup_mock_all_validators(
                 10000,
                 genesis_time,
                 ctx,
+                chunk_distribution_config1,
             );
             view_client_addr_slot = Some(view_client_addr);
             shards_manager_adapter_slot = Some(shards_manager_adapter);

chain/client/src/tests/bug_repros.rs

@@ -65,6 +65,7 @@ fn repro_1183() {
             vec![false; validators.len()],
             vec![true; validators.len()],
             false,
+            None,
             Box::new(move |_, _account_id: _, msg: &PeerManagerMessageRequest| {
                 if let NetworkRequests::Block { block } = msg.as_network_requests_ref() {
                     let mut last_block = last_block.write().unwrap();
@@ -191,6 +192,7 @@ fn test_sync_from_archival_node() {
             vec![true, false, false, false],
             vec![false, true, true, true],
             false,
+            None,
             Box::new(
                 move |conns,
                       _,
@@ -294,6 +296,7 @@ fn test_long_gap_between_blocks() {
             vec![false, false],
             vec![true, true],
             false,
+            None,
             Box::new(
                 move |conns,
                       _,

chain/client/src/tests/catching_up.rs

@@ -161,6 +161,7 @@ fn test_catchup_receipts_sync_common(wait_till: u64, send: u64, sync_hold: bool)
             archive,
             epoch_sync_enabled,
             false,
+            None,
             Box::new(move |_, _account_id: _, msg: &PeerManagerMessageRequest| {
                 let msg = msg.as_network_requests_ref();
                 let account_from = "test3.3".parse().unwrap();
@@ -454,6 +455,7 @@ fn test_catchup_random_single_part_sync_common(skip_15: bool, non_zero: bool, he
             vec![false; validators.len()],
             vec![true; validators.len()],
             false,
+            None,
             Box::new(move |_, _account_id: _, msg: &PeerManagerMessageRequest| {
                 let msg = msg.as_network_requests_ref();
                 let mut seen_heights_same_block = seen_heights_same_block.write().unwrap();
@@ -642,6 +644,7 @@ fn test_catchup_sanity_blocks_produced() {
             archive,
             epoch_sync_enabled,
             false,
+            None,
             Box::new(move |_, _account_id: _, msg: &PeerManagerMessageRequest| {
                 let msg = msg.as_network_requests_ref();
                 let propagate = if let NetworkRequests::Block { block } = msg {
@@ -721,6 +724,7 @@ fn test_all_chunks_accepted_common(
             archive,
             epoch_sync_enabled,
             false,
+            None,
             Box::new(move |_, sender_account_id: AccountId, msg: &PeerManagerMessageRequest| {
                 let msg = msg.as_network_requests_ref();
                 let mut seen_chunk_same_sender = seen_chunk_same_sender.write().unwrap();

chain/client/src/tests/consensus.rs

@@ -79,6 +79,7 @@ fn test_consensus_with_epoch_switches() {
             archive,
             epoch_sync_enabled,
             false,
+            None,
             Box::new(move |_, from_whom: AccountId, msg: &PeerManagerMessageRequest| {
                 let mut all_blocks: RwLockWriteGuard<BTreeMap<BlockHeight, Block>> =
                     all_blocks.write().unwrap();

chain/client/src/tests/cross_shard_tx.rs

@@ -58,6 +58,7 @@ fn test_keyvalue_runtime_balances() {
             vec![false; validators.len()],
             vec![true; validators.len()],
             false,
+            None,
             Box::new(move |_, _account_id: _, _msg: &PeerManagerMessageRequest| {
                 (NetworkResponses::NoResponse.into(), true)
             }),
@@ -460,6 +461,7 @@ fn test_cross_shard_tx_common(
             vec![true; validators.len()],
             vec![false; validators.len()],
             true,
+            None,
             Box::new(move |_, _account_id: _, _msg: &PeerManagerMessageRequest| {
                 (PeerManagerMessageResponse::NetworkResponses(NetworkResponses::NoResponse), true)
             }),

chain/client/src/tests/query_client.rs

@@ -291,6 +291,7 @@ fn test_garbage_collection() {
             vec![false, true], // first validator non-archival, second archival
             vec![true, true],
             true,
+            None,
             Box::new(
                 move |conns,
                       _,

integration-tests/src/tests/client/chunks_management.rs

@@ -4,6 +4,7 @@ use futures::{future, FutureExt};
 use near_actix_test_utils::run_actix;
 use near_async::time;
 use near_chain::test_utils::ValidatorSchedule;
+use near_chain_configs::{ChunkDistributionNetworkConfig, ChunkDistributionUris};
 use near_chunks::{
     CHUNK_REQUEST_RETRY, CHUNK_REQUEST_SWITCH_TO_FULL_FETCH, CHUNK_REQUEST_SWITCH_TO_OTHERS,
 };
@@ -32,13 +33,17 @@ struct Test {
 
 impl Test {
     fn run(self) {
-        heavy_test(move || run_actix(async move { self.run_impl() }))
+        heavy_test(move || run_actix(async move { self.run_impl(None) }))
+    }
+
+    fn run_with_chunk_distribution_network(self, config: ChunkDistributionNetworkConfig) {
+        heavy_test(move || run_actix(async move { self.run_impl(Some(config)) }))
     }
 
     /// Runs block producing client and stops after network mock received seven blocks
     /// Confirms that the blocks form a chain (which implies the chunks are distributed).
     /// Confirms that the number of messages transmitting the chunks matches the expected number.
-    fn run_impl(self) {
+    fn run_impl(self, chunk_distribution_config: Option<ChunkDistributionNetworkConfig>) {
         init_test_logger();
 
         let connectors: Arc<RwLock<Vec<ActorHandlesForTesting>>> = Arc::new(RwLock::new(vec![]));
@@ -111,6 +116,7 @@ impl Test {
             archive,
             epoch_sync_enabled,
             false,
+            chunk_distribution_config,
             Box::new(move |_, from_whom: AccountId, msg: &PeerManagerMessageRequest| {
                 let msg = msg.as_network_requests_ref();
                 match msg {
@@ -311,6 +317,67 @@ fn chunks_produced_and_distributed_one_val_per_shard() {
     .run()
 }
 
+// Nothing the chunk distribution config can do should break chunk distribution
+// because we always fallback on the p2p mechanism. This test runs with a config
+// where `enabled: false`.
+#[test]
+fn chunks_produced_and_distributed_chunk_distribution_network_disabled() {
+    let config = ChunkDistributionNetworkConfig {
+        enabled: false,
+        uris: ChunkDistributionUris { set: String::new(), get: String::new() },
+    };
+    Test {
+        validator_groups: 4,
+        chunk_only_producers: false,
+        drop_to_4_from: &[],
+        drop_all_chunk_forward_msgs: false,
+        block_timeout: CHUNK_REQUEST_RETRY * 15,
+    }
+    .run_with_chunk_distribution_network(config)
+}
+
+// Nothing the chunk distribution config can do should break chunk distribution
+// because we always fallback on the p2p mechanism. This test runs with a config
+// where the URIs are not real endpoints.
+#[test]
+fn chunks_produced_and_distributed_chunk_distribution_network_wrong_urls() {
+    let config = ChunkDistributionNetworkConfig {
+        enabled: false,
+        uris: ChunkDistributionUris {
+            set: "http://www.fake-set-url.com".into(),
+            get: "http://www.fake-get-url.com".into(),
+        },
+    };
+    Test {
+        validator_groups: 4,
+        chunk_only_producers: false,
+        drop_to_4_from: &[],
+        drop_all_chunk_forward_msgs: false,
+        block_timeout: CHUNK_REQUEST_RETRY * 15,
+    }
+    .run_with_chunk_distribution_network(config)
+}
+
+// Nothing the chunk distribution config can do should break chunk distribution
+// because we always fallback on the p2p mechanism. This test runs with a config
+// where the `get` URI points at a random http server (therefore it does not
+// return valid chunks).
+#[test]
+fn chunks_produced_and_distributed_chunk_distribution_network_incorrect_get_return() {
+    let config = ChunkDistributionNetworkConfig {
+        enabled: false,
+        uris: ChunkDistributionUris { set: String::new(), get: "https://www.google.com".into() },
+    };
+    Test {
+        validator_groups: 4,
+        chunk_only_producers: false,
+        drop_to_4_from: &[],
+        drop_all_chunk_forward_msgs: false,
+        block_timeout: CHUNK_REQUEST_RETRY * 15,
+    }
+    .run_with_chunk_distribution_network(config)
+}
+
 #[test]
 #[ignore] // TODO: #8853
 fn chunks_produced_and_distributed_all_in_all_shards_should_succeed_even_without_forwarding() {

integration-tests/src/tests/client/process_blocks.rs

@@ -502,6 +502,7 @@ fn produce_block_with_approvals_arrived_early() {
             archive,
             epoch_sync_enabled,
             false,
+            None,
             Box::new(
                 move |conns,
                       _,
@@ -783,6 +784,7 @@ fn ban_peer_for_invalid_block_common(mode: InvalidBlockMode) {
             vec![false; validators.len()],
             vec![true; validators.len()],
             false,
+            None,
             Box::new(
                 move |conns,
                       _,