From c2ea15a4ef32fd226e4e3c36d3675e8b96b2d684 Mon Sep 17 00:00:00 2001 From: Stefan Neamtu Date: Wed, 29 Jan 2025 15:05:22 +0100 Subject: [PATCH 1/6] . --- chain/chain/Cargo.toml | 78 +++++++++---------- chain/chain/src/chain.rs | 5 +- chain/chain/src/sharding.rs | 26 +++++++ chain/chain/src/test_utils/kv_runtime.rs | 17 +--- chain/chain/src/validate.rs | 4 +- chain/chunks/src/logic.rs | 4 +- chain/chunks/src/shards_manager_actor.rs | 46 +++++------ chain/chunks/src/test_utils.rs | 5 +- chain/client/src/client.rs | 5 +- chain/client/src/test_utils/client.rs | 5 +- chain/epoch-manager/src/adapter.rs | 48 +++++------- .../src/tests/client/challenges.rs | 10 ++- tools/mock-node/src/lib.rs | 3 +- 13 files changed, 140 insertions(+), 116 deletions(-) diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index 1b520c29cef..61d7c3f85fe 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -37,8 +37,8 @@ tracing.workspace = true near-async.workspace = true near-cache.workspace = true near-chain-configs = { workspace = true, features = [ - "test_genesis", - "test_utils", + "test_genesis", + "test_utils", ] } near-chain-primitives.workspace = true near-client-primitives.workspace = true @@ -70,50 +70,50 @@ assert_matches.workspace = true byzantine_asserts = [] expensive_tests = [] test_features = [ - "near-vm-runner/test_features", - "near-primitives/test_features", - "near-store/test_features", - "node-runtime/test_features", + "near-vm-runner/test_features", + "near-primitives/test_features", + "near-store/test_features", + "node-runtime/test_features", ] shadow_chunk_validation = [] nightly = [ - "near-async/nightly", - "near-chain-configs/nightly", - "near-client-primitives/nightly", - "near-epoch-manager/nightly", - "near-mainnet-res/nightly", - "near-network/nightly", - "near-o11y/nightly", - "near-parameters/nightly", - "near-pool/nightly", - "near-primitives/nightly", - "near-store/nightly", - "near-vm-runner/nightly", - "nightly_protocol", - "node-runtime/nightly", + "near-async/nightly", + "near-chain-configs/nightly", + "near-client-primitives/nightly", + "near-epoch-manager/nightly", + "near-mainnet-res/nightly", + "near-network/nightly", + "near-o11y/nightly", + "near-parameters/nightly", + "near-pool/nightly", + "near-primitives/nightly", + "near-store/nightly", + "near-vm-runner/nightly", + "nightly_protocol", + "node-runtime/nightly", ] nightly_protocol = [ - "near-async/nightly_protocol", - "near-chain-configs/nightly_protocol", - "near-client-primitives/nightly_protocol", - "near-epoch-manager/nightly_protocol", - "near-mainnet-res/nightly_protocol", - "near-network/nightly_protocol", - "near-o11y/nightly_protocol", - "near-parameters/nightly_protocol", - "near-pool/nightly_protocol", - "near-primitives/nightly_protocol", - "near-store/nightly_protocol", - "near-vm-runner/nightly_protocol", - "node-runtime/nightly_protocol", + "near-async/nightly_protocol", + "near-chain-configs/nightly_protocol", + "near-client-primitives/nightly_protocol", + "near-epoch-manager/nightly_protocol", + "near-mainnet-res/nightly_protocol", + "near-network/nightly_protocol", + "near-o11y/nightly_protocol", + "near-parameters/nightly_protocol", + "near-pool/nightly_protocol", + "near-primitives/nightly_protocol", + "near-store/nightly_protocol", + "near-vm-runner/nightly_protocol", + "node-runtime/nightly_protocol", ] sandbox = ["near-o11y/sandbox", "near-primitives/sandbox"] protocol_schema = [ - "near-schema-checker-lib/protocol_schema", - "near-crypto/protocol_schema", - "near-parameters/protocol_schema", - "near-primitives/protocol_schema", - "near-store/protocol_schema", - "near-epoch-manager/protocol_schema", + "near-schema-checker-lib/protocol_schema", + "near-crypto/protocol_schema", + "near-parameters/protocol_schema", + "near-primitives/protocol_schema", + "near-store/protocol_schema", + "near-epoch-manager/protocol_schema", ] diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index c8a5154e783..dea52e1b967 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -13,7 +13,7 @@ use crate::orphan::{Orphan, OrphanBlockPool}; use crate::rayon_spawner::RayonAsyncComputationSpawner; use crate::resharding::manager::ReshardingManager; use crate::resharding::types::ReshardingSender; -use crate::sharding::{get_receipts_shuffle_salt, shuffle_receipt_proofs}; +use crate::sharding::{get_receipts_shuffle_salt, num_total_parts, shuffle_receipt_proofs}; use crate::signature_verification::{ verify_block_header_signature_with_epoch_manager, verify_block_vrf, verify_chunk_header_signature_with_epoch_manager, @@ -1408,7 +1408,8 @@ impl Chain { return Ok(true); } } - for part_id in 0..self.epoch_manager.num_total_parts() { + let total_parts = num_total_parts(self.epoch_manager.as_ref()); + for part_id in 0..total_parts { if &Some(self.epoch_manager.get_part_owner(&epoch_id, part_id as u64)?) == me { return Ok(true); } diff --git a/chain/chain/src/sharding.rs b/chain/chain/src/sharding.rs index 447b6cbbd20..e909461ba0b 100644 --- a/chain/chain/src/sharding.rs +++ b/chain/chain/src/sharding.rs @@ -7,6 +7,32 @@ use rand::seq::SliceRandom; use rand::SeedableRng; use rand_chacha::ChaCha20Rng; +/// Number of Reed-Solomon parts we split each chunk into. +/// +/// Note: this shouldn't be too large, our Reed-Solomon supports at most 256 +/// parts. +pub fn num_total_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { + let seats = epoch_manager.get_genesis_num_block_producer_seats(); + if seats > 1 { + seats as usize + } else { + 2 + } +} + +/// How many Reed-Solomon parts are data parts. +/// +/// That is, fetching this many parts should be enough to reconstruct a +/// chunk, if there are no errors. +pub fn num_data_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { + let total_parts = num_total_parts(epoch_manager); + if total_parts <= 3 { + 1 + } else { + (total_parts - 1) / 3 + } +} + /// Gets salt for shuffling receipts grouped by **source shards** before /// processing them in the target shard. pub fn get_receipts_shuffle_salt<'a>( diff --git a/chain/chain/src/test_utils/kv_runtime.rs b/chain/chain/src/test_utils/kv_runtime.rs index 47e10c8f9c9..59afcd35f68 100644 --- a/chain/chain/src/test_utils/kv_runtime.rs +++ b/chain/chain/src/test_utils/kv_runtime.rs @@ -1,4 +1,5 @@ use super::ValidatorSchedule; +use crate::sharding::{num_data_parts, num_total_parts}; use crate::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, PrepareTransactionsBlockContext, PrepareTransactionsChunkContext, PreparedTransactions, @@ -431,18 +432,8 @@ impl EpochManagerAdapter for MockEpochManager { Ok(self.get_shard_layout(epoch_id)?.shard_ids().collect()) } - fn num_total_parts(&self) -> usize { - 12 + (self.num_shards as usize + 1) % 50 - } - - fn num_data_parts(&self) -> usize { - // Same as in Nightshade Runtime - let total_parts = self.num_total_parts(); - if total_parts <= 3 { - 1 - } else { - (total_parts - 1) / 3 - } + fn get_genesis_num_block_producer_seats(&self) -> u64 { + 12 + (self.num_shards + 1) % 50 } fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { @@ -450,7 +441,7 @@ impl EpochManagerAdapter for MockEpochManager { &self.get_epoch_block_producers_ordered(epoch_id, &CryptoHash::default())?; // if we don't use data_parts and total_parts as part of the formula here, the part owner // would not depend on height, and tests wouldn't catch passing wrong height here - let idx = part_id as usize + self.num_data_parts() + self.num_total_parts(); + let idx = part_id as usize + num_data_parts(self) + num_total_parts(self); Ok(validators[idx as usize % validators.len()].0.account_id().clone()) } diff --git a/chain/chain/src/validate.rs b/chain/chain/src/validate.rs index a499c690f6b..7d81c1b0270 100644 --- a/chain/chain/src/validate.rs +++ b/chain/chain/src/validate.rs @@ -18,6 +18,7 @@ use near_primitives::transaction::SignedTransaction; use near_primitives::types::chunk_extra::ChunkExtra; use near_primitives::types::{AccountId, BlockHeight, EpochId, Nonce}; +use crate::sharding::num_data_parts; use crate::signature_verification::{ verify_block_header_signature_with_epoch_manager, verify_chunk_header_signature_with_epoch_manager, @@ -343,7 +344,8 @@ fn validate_chunk_proofs_challenge( let tmp_chunk; let chunk_ref = match &*chunk_proofs.chunk { MaybeEncodedShardChunk::Encoded(encoded_chunk) => { - match encoded_chunk.decode_chunk(epoch_manager.num_data_parts()) { + let data_parts = num_data_parts(epoch_manager); + match encoded_chunk.decode_chunk(data_parts) { Ok(chunk) => { tmp_chunk = Some(chunk); tmp_chunk.as_ref().unwrap() diff --git a/chain/chunks/src/logic.rs b/chain/chunks/src/logic.rs index 2687e15fe89..acefaa0c88f 100644 --- a/chain/chunks/src/logic.rs +++ b/chain/chunks/src/logic.rs @@ -1,3 +1,4 @@ +use near_chain::sharding::num_data_parts; use near_chain::ChainStoreAccess; use near_chain::{ types::EpochManagerAdapter, validate::validate_chunk_proofs, BlockHeader, Chain, ChainStore, @@ -166,8 +167,9 @@ pub fn decode_encoded_chunk( ?chunk_hash) .entered(); + let data_parts = num_data_parts(epoch_manager); if let Ok(shard_chunk) = encoded_chunk - .decode_chunk(epoch_manager.num_data_parts()) + .decode_chunk(data_parts) .map_err(|err| Error::from(err)) .and_then(|shard_chunk| { if !validate_chunk_proofs(&shard_chunk, epoch_manager)? { diff --git a/chain/chunks/src/shards_manager_actor.rs b/chain/chunks/src/shards_manager_actor.rs index 8356f8a83d1..fa908286824 100644 --- a/chain/chunks/src/shards_manager_actor.rs +++ b/chain/chunks/src/shards_manager_actor.rs @@ -95,6 +95,7 @@ use near_async::time::Duration; use near_async::time::{self, Clock}; use near_chain::byzantine_assert; use near_chain::near_chain_primitives::error::Error::DBNotFoundErr; +use near_chain::sharding::{num_data_parts, num_total_parts}; use near_chain::signature_verification::{ verify_chunk_header_signature_with_epoch_manager, verify_chunk_header_signature_with_epoch_manager_and_parts, @@ -359,6 +360,7 @@ impl ShardsManagerActor { initial_chain_header_head: Tip, chunk_request_retry_period: Duration, ) -> Self { + let total_parts = num_total_parts(epoch_manager.as_ref()); Self { clock, validator_signer, @@ -368,11 +370,8 @@ impl ShardsManagerActor { shard_tracker, peer_manager_adapter: network_adapter, client_adapter, - rs: ReedSolomon::new( - epoch_manager.num_data_parts(), - epoch_manager.num_total_parts() - epoch_manager.num_data_parts(), - ) - .unwrap(), + rs: ReedSolomon::new(total_parts, total_parts - num_data_parts(epoch_manager.as_ref())) + .unwrap(), encoded_chunks: EncodedChunksCache::new(), requested_partial_encoded_chunks: RequestPool::new( CHUNK_REQUEST_RETRY, @@ -480,7 +479,8 @@ impl ShardsManagerActor { let epoch_id = self.epoch_manager.get_epoch_id_from_prev_block(ancestor_hash)?; - for part_ord in 0..self.epoch_manager.num_total_parts() { + let total_parts = num_total_parts(self.epoch_manager.as_ref()); + for part_ord in 0..total_parts { let part_ord = part_ord as u64; if cache_entry.is_some_and(|cache_entry| cache_entry.parts.contains_key(&part_ord)) { continue; @@ -1126,7 +1126,7 @@ impl ShardsManagerActor { chunk_hash = ?chunk.chunk_hash()) .entered(); - let data_parts = self.epoch_manager.num_data_parts(); + let data_parts = num_data_parts(self.epoch_manager.as_ref()); if chunk.content().num_fetched_parts() < data_parts { debug!(target: "chunks", num_fetched_parts = chunk.content().num_fetched_parts(), data_parts, "Incomplete"); return ChunkStatus::Incomplete; @@ -1212,7 +1212,7 @@ impl ShardsManagerActor { } // check part merkle proofs - let num_total_parts = self.epoch_manager.num_total_parts(); + let num_total_parts = num_total_parts(self.epoch_manager.as_ref()); for part_info in forward.parts.iter() { self.validate_part(forward.merkle_root, part_info, num_total_parts)?; } @@ -1260,7 +1260,7 @@ impl ShardsManagerActor { fn insert_forwarded_chunk(&mut self, forward: PartialEncodedChunkForwardMsg) { let chunk_hash = forward.chunk_hash.clone(); - let num_total_parts = self.epoch_manager.num_total_parts() as u64; + let num_total_parts = num_total_parts(self.epoch_manager.as_ref()) as u64; match self.chunk_forwards_cache.get_mut(&chunk_hash) { None => { // Never seen this chunk hash before, collect the parts and cache them @@ -1505,9 +1505,9 @@ impl ShardsManagerActor { if entry.complete { return Ok(ProcessPartialEncodedChunkResult::Known); } - debug!(target: "chunks", num_parts_in_cache = entry.parts.len(), total_needed = self.epoch_manager.num_data_parts()); + debug!(target: "chunks", num_parts_in_cache = entry.parts.len(), total_needed = num_data_parts(self.epoch_manager.as_ref())); } else { - debug!(target: "chunks", num_parts_in_cache = 0, total_needed = self.epoch_manager.num_data_parts()); + debug!(target: "chunks", num_parts_in_cache = 0, total_needed = num_data_parts(self.epoch_manager.as_ref())); } // 1.b Checking chunk height @@ -1548,7 +1548,7 @@ impl ShardsManagerActor { let partial_encoded_chunk = partial_encoded_chunk.as_ref().into_inner(); // 1.d Checking part_ords' validity - let num_total_parts = self.epoch_manager.num_total_parts(); + let num_total_parts = num_total_parts(self.epoch_manager.as_ref()); for part_info in partial_encoded_chunk.parts.iter() { // TODO: only validate parts we care about // https://github.com/near/nearcore/issues/5885 @@ -1714,7 +1714,7 @@ impl ShardsManagerActor { let have_all_parts = self.has_all_parts(&prev_block_hash, entry, me)?; let have_all_receipts = self.has_all_receipts(&prev_block_hash, entry, me)?; - let can_reconstruct = entry.parts.len() >= self.epoch_manager.num_data_parts(); + let can_reconstruct = entry.parts.len() >= num_data_parts(self.epoch_manager.as_ref()); let chunk_producer = self .epoch_manager .get_chunk_producer_info(&ChunkProductionKey { @@ -1765,7 +1765,7 @@ impl ShardsManagerActor { let protocol_version = self.epoch_manager.get_epoch_protocol_version(&epoch_id)?; let mut encoded_chunk = EncodedShardChunk::from_header( header.clone(), - self.epoch_manager.num_total_parts(), + num_total_parts(self.epoch_manager.as_ref()), protocol_version, ); @@ -1994,7 +1994,8 @@ impl ShardsManagerActor { chunk_entry: &EncodedChunksCacheEntry, me: Option<&AccountId>, ) -> Result { - for part_ord in 0..self.epoch_manager.num_total_parts() { + let total_parts = num_total_parts(self.epoch_manager.as_ref()); + for part_ord in 0..total_parts { let part_ord = part_ord as u64; if !chunk_entry.parts.contains_key(&part_ord) { if need_part(prev_block_hash, part_ord, me, self.epoch_manager.as_ref())? { @@ -2072,7 +2073,8 @@ impl ShardsManagerActor { let mut block_producer_mapping = HashMap::new(); let epoch_id = self.epoch_manager.get_epoch_id_from_prev_block(&prev_block_hash)?; - for part_ord in 0..self.epoch_manager.num_total_parts() { + let total_parts = num_total_parts(self.epoch_manager.as_ref()); + for part_ord in 0..total_parts { let part_ord = part_ord as u64; let to_whom = self.epoch_manager.get_part_owner(&epoch_id, part_ord).unwrap(); @@ -2518,7 +2520,7 @@ mod test { }) .count() }; - let non_owned_part_ords: Vec = (0..(fixture.epoch_manager.num_total_parts() as u64)) + let non_owned_part_ords: Vec = (0..(num_total_parts(&fixture.epoch_manager) as u64)) .filter(|ord| !fixture.mock_part_ords.contains(ord)) .collect(); // Received 3 partial encoded chunks; the owned part is received 3 times, but should @@ -2934,7 +2936,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(fixture.epoch_manager.num_data_parts()) + .decode_chunk(num_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); @@ -3026,7 +3028,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(fixture.epoch_manager.num_data_parts()) + .decode_chunk(num_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); @@ -3158,7 +3160,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(fixture.epoch_manager.num_data_parts()) + .decode_chunk(num_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); @@ -3166,7 +3168,7 @@ mod test { let (source, response) = shards_manager.prepare_partial_encoded_chunk_response(PartialEncodedChunkRequestMsg { chunk_hash: fixture.mock_chunk_header.chunk_hash(), - part_ords: vec![0, fixture.epoch_manager.num_total_parts() as u64], + part_ords: vec![0, num_total_parts(&fixture.epoch_manager) as u64], tracking_shards: HashSet::new(), }); assert_eq!(source, PartialEncodedChunkResponseSource::ShardChunkOnDisk); @@ -3194,7 +3196,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(fixture.epoch_manager.num_data_parts()) + .decode_chunk(num_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); diff --git a/chain/chunks/src/test_utils.rs b/chain/chunks/src/test_utils.rs index db8d77e143f..73477e2699c 100644 --- a/chain/chunks/src/test_utils.rs +++ b/chain/chunks/src/test_utils.rs @@ -1,4 +1,5 @@ use near_async::messaging::CanSend; +use near_chain::sharding::{num_data_parts, num_total_parts}; use near_chain::types::{EpochManagerAdapter, Tip}; use near_chain::{Chain, ChainStore}; use near_epoch_manager::shard_tracker::{ShardTracker, TrackedConfig}; @@ -87,8 +88,8 @@ impl ChunkTestFixture { let mock_network = Arc::new(MockPeerManagerAdapter::default()); let mock_client_adapter = Arc::new(MockClientAdapterForShardsManager::default()); - let data_parts = epoch_manager.num_data_parts(); - let parity_parts = epoch_manager.num_total_parts() - data_parts; + let data_parts = num_data_parts(&epoch_manager); + let parity_parts = num_total_parts(&epoch_manager) - data_parts; let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let mock_ancestor_hash = CryptoHash::default(); // generate a random block hash for the block at height 1 diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index 2f00e485f80..fd2bfb3b2b9 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -22,6 +22,7 @@ use near_chain::chain::{ VerifyBlockHashAndSignatureResult, }; use near_chain::orphan::OrphanMissingChunks; +use near_chain::sharding::{num_data_parts, num_total_parts}; use near_chain::state_snapshot_actor::SnapshotCallbacks; use near_chain::test_utils::format_hash; use near_chain::types::PrepareTransactionsChunkContext; @@ -325,8 +326,8 @@ impl Client { false, ); let num_block_producer_seats = config.num_block_producer_seats as usize; - let data_parts = epoch_manager.num_data_parts(); - let parity_parts = epoch_manager.num_total_parts() - data_parts; + let data_parts = num_data_parts(epoch_manager.as_ref()); + let parity_parts = num_total_parts(epoch_manager.as_ref()) - data_parts; let doomslug = Doomslug::new( clock.clone(), diff --git a/chain/client/src/test_utils/client.rs b/chain/client/src/test_utils/client.rs index 337eb59b434..b96889f75a8 100644 --- a/chain/client/src/test_utils/client.rs +++ b/chain/client/src/test_utils/client.rs @@ -11,6 +11,7 @@ use actix_rt::System; use itertools::Itertools; use near_async::messaging::Sender; use near_chain::chain::{do_apply_chunks, BlockCatchUpRequest}; +use near_chain::sharding::{num_data_parts, num_total_parts}; use near_chain::test_utils::{wait_for_all_blocks_in_processing, wait_for_block_in_processing}; use near_chain::{Chain, ChainStoreAccess, Provenance}; use near_client_primitives::types::Error; @@ -204,8 +205,8 @@ pub fn create_chunk( // reconstruct the chunk with changes (if any) if should_replace { // The best way it to decode chunk, replace transactions and then recreate encoded chunk. - let total_parts = client.chain.epoch_manager.num_total_parts(); - let data_parts = client.chain.epoch_manager.num_data_parts(); + let total_parts = num_total_parts(client.chain.epoch_manager.as_ref()); + let data_parts = num_data_parts(client.chain.epoch_manager.as_ref()); let decoded_chunk = chunk.decode_chunk(data_parts).unwrap(); let parity_parts = total_parts - data_parts; let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); diff --git a/chain/epoch-manager/src/adapter.rs b/chain/epoch-manager/src/adapter.rs index 408d261ecd3..8a2c6e709df 100644 --- a/chain/epoch-manager/src/adapter.rs +++ b/chain/epoch-manager/src/adapter.rs @@ -42,21 +42,11 @@ pub trait EpochManagerAdapter: Send + Sync { /// Get the list of shard ids fn shard_ids(&self, epoch_id: &EpochId) -> Result, EpochError>; - /// Number of Reed-Solomon parts we split each chunk into. - /// - /// Note: this shouldn't be too large, our Reed-Solomon supports at most 256 - /// parts. - fn num_total_parts(&self) -> usize; - - /// How many Reed-Solomon parts are data parts. - /// - /// That is, fetching this many parts should be enough to reconstruct a - /// chunk, if there are no errors. - fn num_data_parts(&self) -> usize; - /// Returns `account_id` that is supposed to have the `part_id`. fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result; + fn get_genesis_num_block_producer_seats(&self) -> u64; + /// Which shard the account belongs to in the given epoch. fn account_id_to_shard_id( &self, @@ -481,23 +471,27 @@ impl EpochManagerAdapter for EpochManagerHandle { Ok(epoch_manager.get_shard_layout(epoch_id)?.shard_ids().collect()) } - fn num_total_parts(&self) -> usize { - let seats = self.read().genesis_num_block_producer_seats; - if seats > 1 { - seats as usize - } else { - 2 - } + fn get_genesis_num_block_producer_seats(&self) -> u64 { + self.read().genesis_num_block_producer_seats } - fn num_data_parts(&self) -> usize { - let total_parts = self.num_total_parts(); - if total_parts <= 3 { - 1 - } else { - (total_parts - 1) / 3 - } - } + // fn num_total_parts(&self) -> usize { + // let seats = self.read().genesis_num_block_producer_seats; + // if seats > 1 { + // seats as usize + // } else { + // 2 + // } + // } + + // fn num_data_parts(&self) -> usize { + // let total_parts = self.num_total_parts(); + // if total_parts <= 3 { + // 1 + // } else { + // (total_parts - 1) / 3 + // } + // } fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { let epoch_manager = self.read(); diff --git a/integration-tests/src/tests/client/challenges.rs b/integration-tests/src/tests/client/challenges.rs index 5402b49db61..e9079b7c04f 100644 --- a/integration-tests/src/tests/client/challenges.rs +++ b/integration-tests/src/tests/client/challenges.rs @@ -1,5 +1,6 @@ use assert_matches::assert_matches; use near_async::time::Clock; +use near_chain::sharding::{num_data_parts, num_total_parts}; use near_chain::validate::validate_challenge; use near_chain::{Block, ChainStoreAccess, Error, Provenance}; use near_chain_configs::Genesis; @@ -213,8 +214,9 @@ fn test_verify_chunk_invalid_proofs_challenge_decoded_chunk() { env.produce_block(0, 1); let (ProduceChunkResult { chunk: encoded_chunk, .. }, block) = create_invalid_proofs_chunk(&mut env.clients[0]); - let chunk = - encoded_chunk.decode_chunk(env.clients[0].chain.epoch_manager.num_data_parts()).unwrap(); + let chunk = encoded_chunk + .decode_chunk(num_data_parts(env.clients[0].chain.epoch_manager.as_ref())) + .unwrap(); let shard_id = chunk.shard_id(); let challenge_result = @@ -364,8 +366,8 @@ fn test_verify_chunk_invalid_state_challenge() { // Invalid chunk & block. let last_block_hash = env.clients[0].chain.head().unwrap().last_block_hash; let last_block = env.clients[0].chain.get_block(&last_block_hash).unwrap(); - let total_parts = env.clients[0].epoch_manager.num_total_parts(); - let data_parts = env.clients[0].epoch_manager.num_data_parts(); + let total_parts = num_total_parts(env.clients[0].epoch_manager.as_ref()); + let data_parts = num_data_parts(env.clients[0].epoch_manager.as_ref()); let parity_parts = total_parts - data_parts; let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let congestion_info = ProtocolFeature::CongestionControl diff --git a/tools/mock-node/src/lib.rs b/tools/mock-node/src/lib.rs index 7a705419245..5d3753c3d3a 100644 --- a/tools/mock-node/src/lib.rs +++ b/tools/mock-node/src/lib.rs @@ -2,6 +2,7 @@ //! components of the mock network. use anyhow::{anyhow, Context as AnyhowContext}; +use near_chain::sharding::num_total_parts; use near_chain::{Block, Chain, ChainStoreAccess, Error}; use near_client::sync::header::MAX_BLOCK_HEADERS; use near_crypto::SecretKey; @@ -463,7 +464,7 @@ fn retrieve_partial_encoded_chunk( chain: &Chain, request: &PartialEncodedChunkRequestMsg, ) -> Result { - let num_total_parts = chain.epoch_manager.num_total_parts(); + let num_total_parts = num_total_parts(chain.epoch_manager.as_ref()); let partial_chunk = chain.chain_store().get_partial_chunk(&request.chunk_hash)?; let present_parts: HashMap = partial_chunk.parts().iter().map(|part| (part.part_ord, part)).collect(); From 81a2c75d0be56cce10e76d39d56f87daefed98e8 Mon Sep 17 00:00:00 2001 From: Stefan Neamtu Date: Wed, 29 Jan 2025 15:08:34 +0100 Subject: [PATCH 2/6] . --- chain/chain/Cargo.toml | 78 +++++++++++++++--------------- chain/epoch-manager/src/adapter.rs | 18 ------- 2 files changed, 39 insertions(+), 57 deletions(-) diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index 61d7c3f85fe..1b520c29cef 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -37,8 +37,8 @@ tracing.workspace = true near-async.workspace = true near-cache.workspace = true near-chain-configs = { workspace = true, features = [ - "test_genesis", - "test_utils", + "test_genesis", + "test_utils", ] } near-chain-primitives.workspace = true near-client-primitives.workspace = true @@ -70,50 +70,50 @@ assert_matches.workspace = true byzantine_asserts = [] expensive_tests = [] test_features = [ - "near-vm-runner/test_features", - "near-primitives/test_features", - "near-store/test_features", - "node-runtime/test_features", + "near-vm-runner/test_features", + "near-primitives/test_features", + "near-store/test_features", + "node-runtime/test_features", ] shadow_chunk_validation = [] nightly = [ - "near-async/nightly", - "near-chain-configs/nightly", - "near-client-primitives/nightly", - "near-epoch-manager/nightly", - "near-mainnet-res/nightly", - "near-network/nightly", - "near-o11y/nightly", - "near-parameters/nightly", - "near-pool/nightly", - "near-primitives/nightly", - "near-store/nightly", - "near-vm-runner/nightly", - "nightly_protocol", - "node-runtime/nightly", + "near-async/nightly", + "near-chain-configs/nightly", + "near-client-primitives/nightly", + "near-epoch-manager/nightly", + "near-mainnet-res/nightly", + "near-network/nightly", + "near-o11y/nightly", + "near-parameters/nightly", + "near-pool/nightly", + "near-primitives/nightly", + "near-store/nightly", + "near-vm-runner/nightly", + "nightly_protocol", + "node-runtime/nightly", ] nightly_protocol = [ - "near-async/nightly_protocol", - "near-chain-configs/nightly_protocol", - "near-client-primitives/nightly_protocol", - "near-epoch-manager/nightly_protocol", - "near-mainnet-res/nightly_protocol", - "near-network/nightly_protocol", - "near-o11y/nightly_protocol", - "near-parameters/nightly_protocol", - "near-pool/nightly_protocol", - "near-primitives/nightly_protocol", - "near-store/nightly_protocol", - "near-vm-runner/nightly_protocol", - "node-runtime/nightly_protocol", + "near-async/nightly_protocol", + "near-chain-configs/nightly_protocol", + "near-client-primitives/nightly_protocol", + "near-epoch-manager/nightly_protocol", + "near-mainnet-res/nightly_protocol", + "near-network/nightly_protocol", + "near-o11y/nightly_protocol", + "near-parameters/nightly_protocol", + "near-pool/nightly_protocol", + "near-primitives/nightly_protocol", + "near-store/nightly_protocol", + "near-vm-runner/nightly_protocol", + "node-runtime/nightly_protocol", ] sandbox = ["near-o11y/sandbox", "near-primitives/sandbox"] protocol_schema = [ - "near-schema-checker-lib/protocol_schema", - "near-crypto/protocol_schema", - "near-parameters/protocol_schema", - "near-primitives/protocol_schema", - "near-store/protocol_schema", - "near-epoch-manager/protocol_schema", + "near-schema-checker-lib/protocol_schema", + "near-crypto/protocol_schema", + "near-parameters/protocol_schema", + "near-primitives/protocol_schema", + "near-store/protocol_schema", + "near-epoch-manager/protocol_schema", ] diff --git a/chain/epoch-manager/src/adapter.rs b/chain/epoch-manager/src/adapter.rs index 8a2c6e709df..ecb7fe45711 100644 --- a/chain/epoch-manager/src/adapter.rs +++ b/chain/epoch-manager/src/adapter.rs @@ -475,24 +475,6 @@ impl EpochManagerAdapter for EpochManagerHandle { self.read().genesis_num_block_producer_seats } - // fn num_total_parts(&self) -> usize { - // let seats = self.read().genesis_num_block_producer_seats; - // if seats > 1 { - // seats as usize - // } else { - // 2 - // } - // } - - // fn num_data_parts(&self) -> usize { - // let total_parts = self.num_total_parts(); - // if total_parts <= 3 { - // 1 - // } else { - // (total_parts - 1) / 3 - // } - // } - fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { let epoch_manager = self.read(); let epoch_info = epoch_manager.get_epoch_info(&epoch_id)?; From 58c5e3141ce392ef84bbcb83412c261f888560fd Mon Sep 17 00:00:00 2001 From: Stefan Neamtu Date: Wed, 29 Jan 2025 15:50:50 +0100 Subject: [PATCH 3/6] fix --- chain/chunks/src/shards_manager_actor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/chunks/src/shards_manager_actor.rs b/chain/chunks/src/shards_manager_actor.rs index fa908286824..31a0d291ea2 100644 --- a/chain/chunks/src/shards_manager_actor.rs +++ b/chain/chunks/src/shards_manager_actor.rs @@ -360,7 +360,7 @@ impl ShardsManagerActor { initial_chain_header_head: Tip, chunk_request_retry_period: Duration, ) -> Self { - let total_parts = num_total_parts(epoch_manager.as_ref()); + let data_parts = num_data_parts(epoch_manager.as_ref()); Self { clock, validator_signer, @@ -370,7 +370,7 @@ impl ShardsManagerActor { shard_tracker, peer_manager_adapter: network_adapter, client_adapter, - rs: ReedSolomon::new(total_parts, total_parts - num_data_parts(epoch_manager.as_ref())) + rs: ReedSolomon::new(data_parts, num_total_parts(epoch_manager.as_ref()) - data_parts) .unwrap(), encoded_chunks: EncodedChunksCache::new(), requested_partial_encoded_chunks: RequestPool::new( From 4bae1233daf9c7218742135ee0fe73efdc520219 Mon Sep 17 00:00:00 2001 From: Stefan Neamtu Date: Wed, 29 Jan 2025 16:07:26 +0100 Subject: [PATCH 4/6] remove part owner fn --- chain/chain/src/chain.rs | 7 +++++-- chain/chain/src/sharding.rs | 13 +++++++++++++ chain/chain/src/test_utils/kv_runtime.rs | 17 ++++++++--------- chain/chunks/src/logic.rs | 4 ++-- chain/chunks/src/shards_manager_actor.rs | 10 ++++------ chain/chunks/src/test_utils.rs | 4 ++-- chain/epoch-manager/src/adapter.rs | 11 ----------- 7 files changed, 34 insertions(+), 32 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index dea52e1b967..0b9c0ebc6c9 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -13,7 +13,9 @@ use crate::orphan::{Orphan, OrphanBlockPool}; use crate::rayon_spawner::RayonAsyncComputationSpawner; use crate::resharding::manager::ReshardingManager; use crate::resharding::types::ReshardingSender; -use crate::sharding::{get_receipts_shuffle_salt, num_total_parts, shuffle_receipt_proofs}; +use crate::sharding::{ + get_part_owner, get_receipts_shuffle_salt, num_total_parts, shuffle_receipt_proofs, +}; use crate::signature_verification::{ verify_block_header_signature_with_epoch_manager, verify_block_vrf, verify_chunk_header_signature_with_epoch_manager, @@ -1410,7 +1412,8 @@ impl Chain { } let total_parts = num_total_parts(self.epoch_manager.as_ref()); for part_id in 0..total_parts { - if &Some(self.epoch_manager.get_part_owner(&epoch_id, part_id as u64)?) == me { + if &Some(get_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_id as u64)?) == me + { return Ok(true); } } diff --git a/chain/chain/src/sharding.rs b/chain/chain/src/sharding.rs index e909461ba0b..6a07d243700 100644 --- a/chain/chain/src/sharding.rs +++ b/chain/chain/src/sharding.rs @@ -2,6 +2,7 @@ use near_epoch_manager::EpochManagerAdapter; use near_primitives::block::Block; use near_primitives::errors::EpochError; use near_primitives::hash::CryptoHash; +use near_primitives::types::{AccountId, EpochId}; use near_primitives::version::ProtocolFeature; use rand::seq::SliceRandom; use rand::SeedableRng; @@ -33,6 +34,18 @@ pub fn num_data_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { } } +/// Returns `account_id` that is supposed to have the `part_id`. +pub fn get_part_owner( + epoch_manager: &dyn EpochManagerAdapter, + epoch_id: &EpochId, + part_id: u64, +) -> Result { + let epoch_info = epoch_manager.get_epoch_info(&epoch_id)?; + let settlement = epoch_info.block_producers_settlement(); + let validator_id = settlement[part_id as usize % settlement.len()]; + Ok(epoch_info.get_validator(validator_id).account_id().clone()) +} + /// Gets salt for shuffling receipts grouped by **source shards** before /// processing them in the target shard. pub fn get_receipts_shuffle_salt<'a>( diff --git a/chain/chain/src/test_utils/kv_runtime.rs b/chain/chain/src/test_utils/kv_runtime.rs index 59afcd35f68..42c161023b0 100644 --- a/chain/chain/src/test_utils/kv_runtime.rs +++ b/chain/chain/src/test_utils/kv_runtime.rs @@ -1,5 +1,4 @@ use super::ValidatorSchedule; -use crate::sharding::{num_data_parts, num_total_parts}; use crate::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, PrepareTransactionsBlockContext, PrepareTransactionsChunkContext, PreparedTransactions, @@ -436,14 +435,14 @@ impl EpochManagerAdapter for MockEpochManager { 12 + (self.num_shards + 1) % 50 } - fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { - let validators = - &self.get_epoch_block_producers_ordered(epoch_id, &CryptoHash::default())?; - // if we don't use data_parts and total_parts as part of the formula here, the part owner - // would not depend on height, and tests wouldn't catch passing wrong height here - let idx = part_id as usize + num_data_parts(self) + num_total_parts(self); - Ok(validators[idx as usize % validators.len()].0.account_id().clone()) - } + // fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { + // let validators = + // &self.get_epoch_block_producers_ordered(epoch_id, &CryptoHash::default())?; + // // if we don't use data_parts and total_parts as part of the formula here, the part owner + // // would not depend on height, and tests wouldn't catch passing wrong height here + // let idx = part_id as usize + num_data_parts(self) + num_total_parts(self); + // Ok(validators[idx as usize % validators.len()].0.account_id().clone()) + // } fn account_id_to_shard_id( &self, diff --git a/chain/chunks/src/logic.rs b/chain/chunks/src/logic.rs index acefaa0c88f..ecaf88517ab 100644 --- a/chain/chunks/src/logic.rs +++ b/chain/chunks/src/logic.rs @@ -1,4 +1,4 @@ -use near_chain::sharding::num_data_parts; +use near_chain::sharding::{get_part_owner, num_data_parts}; use near_chain::ChainStoreAccess; use near_chain::{ types::EpochManagerAdapter, validate::validate_chunk_proofs, BlockHeader, Chain, ChainStore, @@ -35,7 +35,7 @@ pub fn need_part( epoch_manager: &dyn EpochManagerAdapter, ) -> Result { let epoch_id = epoch_manager.get_epoch_id_from_prev_block(prev_block_hash)?; - Ok(Some(&epoch_manager.get_part_owner(&epoch_id, part_ord)?) == me) + Ok(Some(&get_part_owner(epoch_manager, &epoch_id, part_ord)?) == me) } pub fn get_shards_cares_about_this_or_next_epoch( diff --git a/chain/chunks/src/shards_manager_actor.rs b/chain/chunks/src/shards_manager_actor.rs index 31a0d291ea2..2bfb5eb3630 100644 --- a/chain/chunks/src/shards_manager_actor.rs +++ b/chain/chunks/src/shards_manager_actor.rs @@ -95,7 +95,7 @@ use near_async::time::Duration; use near_async::time::{self, Clock}; use near_chain::byzantine_assert; use near_chain::near_chain_primitives::error::Error::DBNotFoundErr; -use near_chain::sharding::{num_data_parts, num_total_parts}; +use near_chain::sharding::{get_part_owner, num_data_parts, num_total_parts}; use near_chain::signature_verification::{ verify_chunk_header_signature_with_epoch_manager, verify_chunk_header_signature_with_epoch_manager_and_parts, @@ -489,7 +489,7 @@ impl ShardsManagerActor { // Note: If request_from_archival is true, we potentially call // get_part_owner unnecessarily. It’s probably not worth optimizing // though unless you can think of a concise way to do it. - let part_owner = self.epoch_manager.get_part_owner(&epoch_id, part_ord)?; + let part_owner = get_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_ord)?; let we_own_part = Some(&part_owner) == me; if !request_full && !we_own_part { continue; @@ -1854,9 +1854,7 @@ impl ShardsManagerActor { .iter() .filter(|part| { part_ords.contains(&part.part_ord) - && self - .epoch_manager - .get_part_owner(epoch_id, part.part_ord) + && get_part_owner(self.epoch_manager.as_ref(), epoch_id, part.part_ord) .is_ok_and(|owner| &owner == me) }) .cloned() @@ -2076,7 +2074,7 @@ impl ShardsManagerActor { let total_parts = num_total_parts(self.epoch_manager.as_ref()); for part_ord in 0..total_parts { let part_ord = part_ord as u64; - let to_whom = self.epoch_manager.get_part_owner(&epoch_id, part_ord).unwrap(); + let to_whom = get_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_ord).unwrap(); let entry = block_producer_mapping.entry(to_whom).or_insert_with(Vec::new); entry.push(part_ord); diff --git a/chain/chunks/src/test_utils.rs b/chain/chunks/src/test_utils.rs index 73477e2699c..844572b1d2b 100644 --- a/chain/chunks/src/test_utils.rs +++ b/chain/chunks/src/test_utils.rs @@ -1,5 +1,5 @@ use near_async::messaging::CanSend; -use near_chain::sharding::{num_data_parts, num_total_parts}; +use near_chain::sharding::{get_part_owner, num_data_parts, num_total_parts}; use near_chain::types::{EpochManagerAdapter, Tip}; use near_chain::{Chain, ChainStore}; use near_epoch_manager::shard_tracker::{ShardTracker, TrackedConfig}; @@ -176,7 +176,7 @@ impl ChunkTestFixture { .iter() .copied() .filter(|p| { - epoch_manager.get_part_owner(&mock_epoch_id, *p).unwrap() == mock_chunk_part_owner + get_part_owner(&epoch_manager, &mock_epoch_id, *p).unwrap() == mock_chunk_part_owner }) .collect(); let encoded_chunk = mock_chunk.create_partial_encoded_chunk( diff --git a/chain/epoch-manager/src/adapter.rs b/chain/epoch-manager/src/adapter.rs index ecb7fe45711..e767eff380e 100644 --- a/chain/epoch-manager/src/adapter.rs +++ b/chain/epoch-manager/src/adapter.rs @@ -42,9 +42,6 @@ pub trait EpochManagerAdapter: Send + Sync { /// Get the list of shard ids fn shard_ids(&self, epoch_id: &EpochId) -> Result, EpochError>; - /// Returns `account_id` that is supposed to have the `part_id`. - fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result; - fn get_genesis_num_block_producer_seats(&self) -> u64; /// Which shard the account belongs to in the given epoch. @@ -475,14 +472,6 @@ impl EpochManagerAdapter for EpochManagerHandle { self.read().genesis_num_block_producer_seats } - fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { - let epoch_manager = self.read(); - let epoch_info = epoch_manager.get_epoch_info(&epoch_id)?; - let settlement = epoch_info.block_producers_settlement(); - let validator_id = settlement[part_id as usize % settlement.len()]; - Ok(epoch_info.get_validator(validator_id).account_id().clone()) - } - fn account_id_to_shard_id( &self, account_id: &AccountId, From bdd4eaaa8d7926433e3382ca82e3a77f68e9d4be Mon Sep 17 00:00:00 2001 From: Stefan Neamtu Date: Wed, 29 Jan 2025 17:17:21 +0100 Subject: [PATCH 5/6] comments --- chain/chain/src/chain.rs | 7 +-- chain/chain/src/sharding.rs | 10 ++-- chain/chain/src/test_utils/kv_runtime.rs | 11 +--- chain/chain/src/validate.rs | 4 +- chain/chunks/src/logic.rs | 6 +-- chain/chunks/src/shards_manager_actor.rs | 53 ++++++++++--------- chain/chunks/src/test_utils.rs | 9 ++-- chain/client/src/client.rs | 6 +-- chain/client/src/test_utils/client.rs | 6 +-- chain/epoch-manager/src/adapter.rs | 4 +- .../src/tests/client/challenges.rs | 8 +-- tools/mock-node/src/lib.rs | 4 +- 12 files changed, 63 insertions(+), 65 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index 0b9c0ebc6c9..ea128a6ec63 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -14,7 +14,7 @@ use crate::rayon_spawner::RayonAsyncComputationSpawner; use crate::resharding::manager::ReshardingManager; use crate::resharding::types::ReshardingSender; use crate::sharding::{ - get_part_owner, get_receipts_shuffle_salt, num_total_parts, shuffle_receipt_proofs, + chunk_part_owner, get_receipts_shuffle_salt, num_total_chunk_parts, shuffle_receipt_proofs, }; use crate::signature_verification::{ verify_block_header_signature_with_epoch_manager, verify_block_vrf, @@ -1410,9 +1410,10 @@ impl Chain { return Ok(true); } } - let total_parts = num_total_parts(self.epoch_manager.as_ref()); + let total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()); for part_id in 0..total_parts { - if &Some(get_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_id as u64)?) == me + if &Some(chunk_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_id as u64)?) + == me { return Ok(true); } diff --git a/chain/chain/src/sharding.rs b/chain/chain/src/sharding.rs index 6a07d243700..d19db39694f 100644 --- a/chain/chain/src/sharding.rs +++ b/chain/chain/src/sharding.rs @@ -12,8 +12,8 @@ use rand_chacha::ChaCha20Rng; /// /// Note: this shouldn't be too large, our Reed-Solomon supports at most 256 /// parts. -pub fn num_total_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { - let seats = epoch_manager.get_genesis_num_block_producer_seats(); +pub fn num_total_chunk_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { + let seats = epoch_manager.genesis_num_block_producer_seats(); if seats > 1 { seats as usize } else { @@ -25,8 +25,8 @@ pub fn num_total_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { /// /// That is, fetching this many parts should be enough to reconstruct a /// chunk, if there are no errors. -pub fn num_data_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { - let total_parts = num_total_parts(epoch_manager); +pub fn num_chunk_data_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { + let total_parts = num_total_chunk_parts(epoch_manager); if total_parts <= 3 { 1 } else { @@ -35,7 +35,7 @@ pub fn num_data_parts(epoch_manager: &dyn EpochManagerAdapter) -> usize { } /// Returns `account_id` that is supposed to have the `part_id`. -pub fn get_part_owner( +pub fn chunk_part_owner( epoch_manager: &dyn EpochManagerAdapter, epoch_id: &EpochId, part_id: u64, diff --git a/chain/chain/src/test_utils/kv_runtime.rs b/chain/chain/src/test_utils/kv_runtime.rs index 42c161023b0..9271ee704f5 100644 --- a/chain/chain/src/test_utils/kv_runtime.rs +++ b/chain/chain/src/test_utils/kv_runtime.rs @@ -431,19 +431,10 @@ impl EpochManagerAdapter for MockEpochManager { Ok(self.get_shard_layout(epoch_id)?.shard_ids().collect()) } - fn get_genesis_num_block_producer_seats(&self) -> u64 { + fn genesis_num_block_producer_seats(&self) -> u64 { 12 + (self.num_shards + 1) % 50 } - // fn get_part_owner(&self, epoch_id: &EpochId, part_id: u64) -> Result { - // let validators = - // &self.get_epoch_block_producers_ordered(epoch_id, &CryptoHash::default())?; - // // if we don't use data_parts and total_parts as part of the formula here, the part owner - // // would not depend on height, and tests wouldn't catch passing wrong height here - // let idx = part_id as usize + num_data_parts(self) + num_total_parts(self); - // Ok(validators[idx as usize % validators.len()].0.account_id().clone()) - // } - fn account_id_to_shard_id( &self, account_id: &AccountId, diff --git a/chain/chain/src/validate.rs b/chain/chain/src/validate.rs index 7d81c1b0270..5fd37c277c1 100644 --- a/chain/chain/src/validate.rs +++ b/chain/chain/src/validate.rs @@ -18,7 +18,7 @@ use near_primitives::transaction::SignedTransaction; use near_primitives::types::chunk_extra::ChunkExtra; use near_primitives::types::{AccountId, BlockHeight, EpochId, Nonce}; -use crate::sharding::num_data_parts; +use crate::sharding::num_chunk_data_parts; use crate::signature_verification::{ verify_block_header_signature_with_epoch_manager, verify_chunk_header_signature_with_epoch_manager, @@ -344,7 +344,7 @@ fn validate_chunk_proofs_challenge( let tmp_chunk; let chunk_ref = match &*chunk_proofs.chunk { MaybeEncodedShardChunk::Encoded(encoded_chunk) => { - let data_parts = num_data_parts(epoch_manager); + let data_parts = num_chunk_data_parts(epoch_manager); match encoded_chunk.decode_chunk(data_parts) { Ok(chunk) => { tmp_chunk = Some(chunk); diff --git a/chain/chunks/src/logic.rs b/chain/chunks/src/logic.rs index ecaf88517ab..6082fcb24cb 100644 --- a/chain/chunks/src/logic.rs +++ b/chain/chunks/src/logic.rs @@ -1,4 +1,4 @@ -use near_chain::sharding::{get_part_owner, num_data_parts}; +use near_chain::sharding::{chunk_part_owner, num_chunk_data_parts}; use near_chain::ChainStoreAccess; use near_chain::{ types::EpochManagerAdapter, validate::validate_chunk_proofs, BlockHeader, Chain, ChainStore, @@ -35,7 +35,7 @@ pub fn need_part( epoch_manager: &dyn EpochManagerAdapter, ) -> Result { let epoch_id = epoch_manager.get_epoch_id_from_prev_block(prev_block_hash)?; - Ok(Some(&get_part_owner(epoch_manager, &epoch_id, part_ord)?) == me) + Ok(Some(&chunk_part_owner(epoch_manager, &epoch_id, part_ord)?) == me) } pub fn get_shards_cares_about_this_or_next_epoch( @@ -167,7 +167,7 @@ pub fn decode_encoded_chunk( ?chunk_hash) .entered(); - let data_parts = num_data_parts(epoch_manager); + let data_parts = num_chunk_data_parts(epoch_manager); if let Ok(shard_chunk) = encoded_chunk .decode_chunk(data_parts) .map_err(|err| Error::from(err)) diff --git a/chain/chunks/src/shards_manager_actor.rs b/chain/chunks/src/shards_manager_actor.rs index 2bfb5eb3630..0862a7ed7d1 100644 --- a/chain/chunks/src/shards_manager_actor.rs +++ b/chain/chunks/src/shards_manager_actor.rs @@ -95,7 +95,7 @@ use near_async::time::Duration; use near_async::time::{self, Clock}; use near_chain::byzantine_assert; use near_chain::near_chain_primitives::error::Error::DBNotFoundErr; -use near_chain::sharding::{get_part_owner, num_data_parts, num_total_parts}; +use near_chain::sharding::{chunk_part_owner, num_chunk_data_parts, num_total_chunk_parts}; use near_chain::signature_verification::{ verify_chunk_header_signature_with_epoch_manager, verify_chunk_header_signature_with_epoch_manager_and_parts, @@ -360,7 +360,7 @@ impl ShardsManagerActor { initial_chain_header_head: Tip, chunk_request_retry_period: Duration, ) -> Self { - let data_parts = num_data_parts(epoch_manager.as_ref()); + let data_parts = num_chunk_data_parts(epoch_manager.as_ref()); Self { clock, validator_signer, @@ -370,8 +370,11 @@ impl ShardsManagerActor { shard_tracker, peer_manager_adapter: network_adapter, client_adapter, - rs: ReedSolomon::new(data_parts, num_total_parts(epoch_manager.as_ref()) - data_parts) - .unwrap(), + rs: ReedSolomon::new( + data_parts, + num_total_chunk_parts(epoch_manager.as_ref()) - data_parts, + ) + .unwrap(), encoded_chunks: EncodedChunksCache::new(), requested_partial_encoded_chunks: RequestPool::new( CHUNK_REQUEST_RETRY, @@ -479,7 +482,7 @@ impl ShardsManagerActor { let epoch_id = self.epoch_manager.get_epoch_id_from_prev_block(ancestor_hash)?; - let total_parts = num_total_parts(self.epoch_manager.as_ref()); + let total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()); for part_ord in 0..total_parts { let part_ord = part_ord as u64; if cache_entry.is_some_and(|cache_entry| cache_entry.parts.contains_key(&part_ord)) { @@ -489,7 +492,7 @@ impl ShardsManagerActor { // Note: If request_from_archival is true, we potentially call // get_part_owner unnecessarily. It’s probably not worth optimizing // though unless you can think of a concise way to do it. - let part_owner = get_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_ord)?; + let part_owner = chunk_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_ord)?; let we_own_part = Some(&part_owner) == me; if !request_full && !we_own_part { continue; @@ -1126,7 +1129,7 @@ impl ShardsManagerActor { chunk_hash = ?chunk.chunk_hash()) .entered(); - let data_parts = num_data_parts(self.epoch_manager.as_ref()); + let data_parts = num_chunk_data_parts(self.epoch_manager.as_ref()); if chunk.content().num_fetched_parts() < data_parts { debug!(target: "chunks", num_fetched_parts = chunk.content().num_fetched_parts(), data_parts, "Incomplete"); return ChunkStatus::Incomplete; @@ -1212,7 +1215,7 @@ impl ShardsManagerActor { } // check part merkle proofs - let num_total_parts = num_total_parts(self.epoch_manager.as_ref()); + let num_total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()); for part_info in forward.parts.iter() { self.validate_part(forward.merkle_root, part_info, num_total_parts)?; } @@ -1260,7 +1263,7 @@ impl ShardsManagerActor { fn insert_forwarded_chunk(&mut self, forward: PartialEncodedChunkForwardMsg) { let chunk_hash = forward.chunk_hash.clone(); - let num_total_parts = num_total_parts(self.epoch_manager.as_ref()) as u64; + let num_total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()) as u64; match self.chunk_forwards_cache.get_mut(&chunk_hash) { None => { // Never seen this chunk hash before, collect the parts and cache them @@ -1505,9 +1508,9 @@ impl ShardsManagerActor { if entry.complete { return Ok(ProcessPartialEncodedChunkResult::Known); } - debug!(target: "chunks", num_parts_in_cache = entry.parts.len(), total_needed = num_data_parts(self.epoch_manager.as_ref())); + debug!(target: "chunks", num_parts_in_cache = entry.parts.len(), total_needed = num_chunk_data_parts(self.epoch_manager.as_ref())); } else { - debug!(target: "chunks", num_parts_in_cache = 0, total_needed = num_data_parts(self.epoch_manager.as_ref())); + debug!(target: "chunks", num_parts_in_cache = 0, total_needed = num_chunk_data_parts(self.epoch_manager.as_ref())); } // 1.b Checking chunk height @@ -1548,7 +1551,7 @@ impl ShardsManagerActor { let partial_encoded_chunk = partial_encoded_chunk.as_ref().into_inner(); // 1.d Checking part_ords' validity - let num_total_parts = num_total_parts(self.epoch_manager.as_ref()); + let num_total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()); for part_info in partial_encoded_chunk.parts.iter() { // TODO: only validate parts we care about // https://github.com/near/nearcore/issues/5885 @@ -1714,7 +1717,7 @@ impl ShardsManagerActor { let have_all_parts = self.has_all_parts(&prev_block_hash, entry, me)?; let have_all_receipts = self.has_all_receipts(&prev_block_hash, entry, me)?; - let can_reconstruct = entry.parts.len() >= num_data_parts(self.epoch_manager.as_ref()); + let can_reconstruct = entry.parts.len() >= num_chunk_data_parts(self.epoch_manager.as_ref()); let chunk_producer = self .epoch_manager .get_chunk_producer_info(&ChunkProductionKey { @@ -1765,7 +1768,7 @@ impl ShardsManagerActor { let protocol_version = self.epoch_manager.get_epoch_protocol_version(&epoch_id)?; let mut encoded_chunk = EncodedShardChunk::from_header( header.clone(), - num_total_parts(self.epoch_manager.as_ref()), + num_total_chunk_parts(self.epoch_manager.as_ref()), protocol_version, ); @@ -1854,7 +1857,7 @@ impl ShardsManagerActor { .iter() .filter(|part| { part_ords.contains(&part.part_ord) - && get_part_owner(self.epoch_manager.as_ref(), epoch_id, part.part_ord) + && chunk_part_owner(self.epoch_manager.as_ref(), epoch_id, part.part_ord) .is_ok_and(|owner| &owner == me) }) .cloned() @@ -1992,7 +1995,7 @@ impl ShardsManagerActor { chunk_entry: &EncodedChunksCacheEntry, me: Option<&AccountId>, ) -> Result { - let total_parts = num_total_parts(self.epoch_manager.as_ref()); + let total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()); for part_ord in 0..total_parts { let part_ord = part_ord as u64; if !chunk_entry.parts.contains_key(&part_ord) { @@ -2071,10 +2074,11 @@ impl ShardsManagerActor { let mut block_producer_mapping = HashMap::new(); let epoch_id = self.epoch_manager.get_epoch_id_from_prev_block(&prev_block_hash)?; - let total_parts = num_total_parts(self.epoch_manager.as_ref()); + let total_parts = num_total_chunk_parts(self.epoch_manager.as_ref()); for part_ord in 0..total_parts { let part_ord = part_ord as u64; - let to_whom = get_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_ord).unwrap(); + let to_whom = + chunk_part_owner(self.epoch_manager.as_ref(), &epoch_id, part_ord).unwrap(); let entry = block_producer_mapping.entry(to_whom).or_insert_with(Vec::new); entry.push(part_ord); @@ -2518,7 +2522,8 @@ mod test { }) .count() }; - let non_owned_part_ords: Vec = (0..(num_total_parts(&fixture.epoch_manager) as u64)) + let non_owned_part_ords: Vec = (0..(num_total_chunk_parts(&fixture.epoch_manager) + as u64)) .filter(|ord| !fixture.mock_part_ords.contains(ord)) .collect(); // Received 3 partial encoded chunks; the owned part is received 3 times, but should @@ -2934,7 +2939,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(num_data_parts(&fixture.epoch_manager)) + .decode_chunk(num_chunk_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); @@ -3026,7 +3031,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(num_data_parts(&fixture.epoch_manager)) + .decode_chunk(num_chunk_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); @@ -3158,7 +3163,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(num_data_parts(&fixture.epoch_manager)) + .decode_chunk(num_chunk_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); @@ -3166,7 +3171,7 @@ mod test { let (source, response) = shards_manager.prepare_partial_encoded_chunk_response(PartialEncodedChunkRequestMsg { chunk_hash: fixture.mock_chunk_header.chunk_hash(), - part_ords: vec![0, num_total_parts(&fixture.epoch_manager) as u64], + part_ords: vec![0, num_total_chunk_parts(&fixture.epoch_manager) as u64], tracking_shards: HashSet::new(), }); assert_eq!(source, PartialEncodedChunkResponseSource::ShardChunkOnDisk); @@ -3194,7 +3199,7 @@ mod test { let mut update = fixture.chain_store.store_update(); let shard_chunk = fixture .mock_encoded_chunk - .decode_chunk(num_data_parts(&fixture.epoch_manager)) + .decode_chunk(num_chunk_data_parts(&fixture.epoch_manager)) .unwrap(); update.save_chunk(shard_chunk); update.commit().unwrap(); diff --git a/chain/chunks/src/test_utils.rs b/chain/chunks/src/test_utils.rs index 844572b1d2b..f48be6eb096 100644 --- a/chain/chunks/src/test_utils.rs +++ b/chain/chunks/src/test_utils.rs @@ -1,5 +1,5 @@ use near_async::messaging::CanSend; -use near_chain::sharding::{get_part_owner, num_data_parts, num_total_parts}; +use near_chain::sharding::{chunk_part_owner, num_chunk_data_parts, num_total_chunk_parts}; use near_chain::types::{EpochManagerAdapter, Tip}; use near_chain::{Chain, ChainStore}; use near_epoch_manager::shard_tracker::{ShardTracker, TrackedConfig}; @@ -88,8 +88,8 @@ impl ChunkTestFixture { let mock_network = Arc::new(MockPeerManagerAdapter::default()); let mock_client_adapter = Arc::new(MockClientAdapterForShardsManager::default()); - let data_parts = num_data_parts(&epoch_manager); - let parity_parts = num_total_parts(&epoch_manager) - data_parts; + let data_parts = num_chunk_data_parts(&epoch_manager); + let parity_parts = num_total_chunk_parts(&epoch_manager) - data_parts; let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let mock_ancestor_hash = CryptoHash::default(); // generate a random block hash for the block at height 1 @@ -176,7 +176,8 @@ impl ChunkTestFixture { .iter() .copied() .filter(|p| { - get_part_owner(&epoch_manager, &mock_epoch_id, *p).unwrap() == mock_chunk_part_owner + chunk_part_owner(&epoch_manager, &mock_epoch_id, *p).unwrap() + == mock_chunk_part_owner }) .collect(); let encoded_chunk = mock_chunk.create_partial_encoded_chunk( diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index fd2bfb3b2b9..e4536336791 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -22,7 +22,7 @@ use near_chain::chain::{ VerifyBlockHashAndSignatureResult, }; use near_chain::orphan::OrphanMissingChunks; -use near_chain::sharding::{num_data_parts, num_total_parts}; +use near_chain::sharding::{num_chunk_data_parts, num_total_chunk_parts}; use near_chain::state_snapshot_actor::SnapshotCallbacks; use near_chain::test_utils::format_hash; use near_chain::types::PrepareTransactionsChunkContext; @@ -326,8 +326,8 @@ impl Client { false, ); let num_block_producer_seats = config.num_block_producer_seats as usize; - let data_parts = num_data_parts(epoch_manager.as_ref()); - let parity_parts = num_total_parts(epoch_manager.as_ref()) - data_parts; + let data_parts = num_chunk_data_parts(epoch_manager.as_ref()); + let parity_parts = num_total_chunk_parts(epoch_manager.as_ref()) - data_parts; let doomslug = Doomslug::new( clock.clone(), diff --git a/chain/client/src/test_utils/client.rs b/chain/client/src/test_utils/client.rs index b96889f75a8..70673041ee0 100644 --- a/chain/client/src/test_utils/client.rs +++ b/chain/client/src/test_utils/client.rs @@ -11,7 +11,7 @@ use actix_rt::System; use itertools::Itertools; use near_async::messaging::Sender; use near_chain::chain::{do_apply_chunks, BlockCatchUpRequest}; -use near_chain::sharding::{num_data_parts, num_total_parts}; +use near_chain::sharding::{num_chunk_data_parts, num_total_chunk_parts}; use near_chain::test_utils::{wait_for_all_blocks_in_processing, wait_for_block_in_processing}; use near_chain::{Chain, ChainStoreAccess, Provenance}; use near_client_primitives::types::Error; @@ -205,8 +205,8 @@ pub fn create_chunk( // reconstruct the chunk with changes (if any) if should_replace { // The best way it to decode chunk, replace transactions and then recreate encoded chunk. - let total_parts = num_total_parts(client.chain.epoch_manager.as_ref()); - let data_parts = num_data_parts(client.chain.epoch_manager.as_ref()); + let total_parts = num_total_chunk_parts(client.chain.epoch_manager.as_ref()); + let data_parts = num_chunk_data_parts(client.chain.epoch_manager.as_ref()); let decoded_chunk = chunk.decode_chunk(data_parts).unwrap(); let parity_parts = total_parts - data_parts; let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); diff --git a/chain/epoch-manager/src/adapter.rs b/chain/epoch-manager/src/adapter.rs index e767eff380e..396c215bad3 100644 --- a/chain/epoch-manager/src/adapter.rs +++ b/chain/epoch-manager/src/adapter.rs @@ -42,7 +42,7 @@ pub trait EpochManagerAdapter: Send + Sync { /// Get the list of shard ids fn shard_ids(&self, epoch_id: &EpochId) -> Result, EpochError>; - fn get_genesis_num_block_producer_seats(&self) -> u64; + fn genesis_num_block_producer_seats(&self) -> u64; /// Which shard the account belongs to in the given epoch. fn account_id_to_shard_id( @@ -468,7 +468,7 @@ impl EpochManagerAdapter for EpochManagerHandle { Ok(epoch_manager.get_shard_layout(epoch_id)?.shard_ids().collect()) } - fn get_genesis_num_block_producer_seats(&self) -> u64 { + fn genesis_num_block_producer_seats(&self) -> u64 { self.read().genesis_num_block_producer_seats } diff --git a/integration-tests/src/tests/client/challenges.rs b/integration-tests/src/tests/client/challenges.rs index e9079b7c04f..15a3ac48b95 100644 --- a/integration-tests/src/tests/client/challenges.rs +++ b/integration-tests/src/tests/client/challenges.rs @@ -1,6 +1,6 @@ use assert_matches::assert_matches; use near_async::time::Clock; -use near_chain::sharding::{num_data_parts, num_total_parts}; +use near_chain::sharding::{num_chunk_data_parts, num_total_chunk_parts}; use near_chain::validate::validate_challenge; use near_chain::{Block, ChainStoreAccess, Error, Provenance}; use near_chain_configs::Genesis; @@ -215,7 +215,7 @@ fn test_verify_chunk_invalid_proofs_challenge_decoded_chunk() { let (ProduceChunkResult { chunk: encoded_chunk, .. }, block) = create_invalid_proofs_chunk(&mut env.clients[0]); let chunk = encoded_chunk - .decode_chunk(num_data_parts(env.clients[0].chain.epoch_manager.as_ref())) + .decode_chunk(num_chunk_data_parts(env.clients[0].chain.epoch_manager.as_ref())) .unwrap(); let shard_id = chunk.shard_id(); @@ -366,8 +366,8 @@ fn test_verify_chunk_invalid_state_challenge() { // Invalid chunk & block. let last_block_hash = env.clients[0].chain.head().unwrap().last_block_hash; let last_block = env.clients[0].chain.get_block(&last_block_hash).unwrap(); - let total_parts = num_total_parts(env.clients[0].epoch_manager.as_ref()); - let data_parts = num_data_parts(env.clients[0].epoch_manager.as_ref()); + let total_parts = num_total_chunk_parts(env.clients[0].epoch_manager.as_ref()); + let data_parts = num_chunk_data_parts(env.clients[0].epoch_manager.as_ref()); let parity_parts = total_parts - data_parts; let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let congestion_info = ProtocolFeature::CongestionControl diff --git a/tools/mock-node/src/lib.rs b/tools/mock-node/src/lib.rs index 5d3753c3d3a..8effb90e57f 100644 --- a/tools/mock-node/src/lib.rs +++ b/tools/mock-node/src/lib.rs @@ -2,7 +2,7 @@ //! components of the mock network. use anyhow::{anyhow, Context as AnyhowContext}; -use near_chain::sharding::num_total_parts; +use near_chain::sharding::num_total_chunk_parts; use near_chain::{Block, Chain, ChainStoreAccess, Error}; use near_client::sync::header::MAX_BLOCK_HEADERS; use near_crypto::SecretKey; @@ -464,7 +464,7 @@ fn retrieve_partial_encoded_chunk( chain: &Chain, request: &PartialEncodedChunkRequestMsg, ) -> Result { - let num_total_parts = num_total_parts(chain.epoch_manager.as_ref()); + let num_total_parts = num_total_chunk_parts(chain.epoch_manager.as_ref()); let partial_chunk = chain.chain_store().get_partial_chunk(&request.chunk_hash)?; let present_parts: HashMap = partial_chunk.parts().iter().map(|part| (part.part_ord, part)).collect(); From 35f274a82063048b18eddb6d68c6581f794382dc Mon Sep 17 00:00:00 2001 From: Stefan Neamtu Date: Thu, 30 Jan 2025 11:19:07 +0100 Subject: [PATCH 6/6] fmt --- chain/chunks/src/shards_manager_actor.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chain/chunks/src/shards_manager_actor.rs b/chain/chunks/src/shards_manager_actor.rs index 0862a7ed7d1..dfc29618699 100644 --- a/chain/chunks/src/shards_manager_actor.rs +++ b/chain/chunks/src/shards_manager_actor.rs @@ -1717,7 +1717,8 @@ impl ShardsManagerActor { let have_all_parts = self.has_all_parts(&prev_block_hash, entry, me)?; let have_all_receipts = self.has_all_receipts(&prev_block_hash, entry, me)?; - let can_reconstruct = entry.parts.len() >= num_chunk_data_parts(self.epoch_manager.as_ref()); + let can_reconstruct = + entry.parts.len() >= num_chunk_data_parts(self.epoch_manager.as_ref()); let chunk_producer = self .epoch_manager .get_chunk_producer_info(&ChunkProductionKey {