Merge branch 'master' into resharding-test-comparison-fix

chain/chain/src/chain.rs

@@ -14,7 +14,9 @@ use crate::rayon_spawner::RayonAsyncComputationSpawner;
 use crate::resharding::manager::ReshardingManager;
 use crate::resharding::types::ReshardingSender;
 use crate::sharding::shuffle_receipt_proofs;
-use crate::signature_verification::verify_chunk_header_signature_with_epoch_manager;
+use crate::signature_verification::{
+    verify_block_vrf, verify_chunk_header_signature_with_epoch_manager,
+};
 use crate::state_request_tracker::StateRequestTracker;
 use crate::state_snapshot_actor::SnapshotCallbacks;
 use crate::stateless_validation::chunk_endorsement::{
@@ -2284,13 +2286,9 @@ impl Chain {
         // Check the header is valid before we proceed with the full block.
         self.validate_header(header, provenance, challenges)?;
 
-        self.epoch_manager.verify_block_vrf(
-            header.epoch_id(),
-            header.height(),
-            &prev_random_value,
-            block.vrf_value(),
-            block.vrf_proof(),
-        )?;
+        let validator =
+            self.epoch_manager.get_block_producer_info(header.epoch_id(), header.height())?;
+        verify_block_vrf(validator, &prev_random_value, block.vrf_value(), block.vrf_proof())?;
 
         if header.random_value() != &hash(block.vrf_value().0.as_ref()) {
             return Err(Error::InvalidRandomnessBeaconOutput);

chain/chain/src/runtime/tests.rs

@@ -691,7 +691,7 @@ fn test_verify_validator_signature() {
         (0..2).map(|i| AccountId::try_from(format!("test{}", i + 1)).unwrap()).collect::<Vec<_>>();
     let env = TestEnv::new(vec![validators.clone()], 2, true);
     let data = [0; 32];
-    let signer = InMemorySigner::test(&validators[0]);
+    let signer = InMemorySigner::test_signer(&validators[0]);
     let signature = signer.sign(&data);
     assert!(env
         .epoch_manager
@@ -1371,17 +1371,17 @@ fn test_delete_account_after_unstake() {
     let mut env = TestEnv::new(vec![validators.clone()], 4, false);
     let block_producers: Vec<_> =
         validators.iter().map(|id| create_test_signer(id.as_str())).collect();
-    let signers: Vec<_> = validators.iter().map(|id| InMemorySigner::test(&id)).collect();
+    let signers: Vec<_> = validators.iter().map(|id| InMemorySigner::test_signer(&id)).collect();
 
-    let staking_transaction1 = stake(1, &signers[1].clone().into(), &block_producers[1], 0);
+    let staking_transaction1 = stake(1, &signers[1], &block_producers[1], 0);
     env.step_default(vec![staking_transaction1]);
     let account = env.view_account(block_producers[1].validator_id());
     assert_eq!(account.amount, TESTING_INIT_BALANCE - TESTING_INIT_STAKE);
     assert_eq!(account.locked, TESTING_INIT_STAKE);
     for _ in 2..=5 {
         env.step_default(vec![]);
     }
-    let staking_transaction2 = stake(2, &signers[1].clone().into(), &block_producers[1], 1);
+    let staking_transaction2 = stake(2, &signers[1], &block_producers[1], 1);
     env.step_default(vec![staking_transaction2]);
     for _ in 7..=13 {
         env.step_default(vec![]);
@@ -1391,11 +1391,11 @@ fn test_delete_account_after_unstake() {
 
     let delete_account_transaction = SignedTransaction::from_actions(
         4,
-        signers[1].account_id.clone(),
-        signers[1].account_id.clone(),
-        &signers[1].clone().into(),
+        signers[1].get_account_id(),
+        signers[1].get_account_id(),
+        &signers[1],
         vec![Action::DeleteAccount(DeleteAccountAction {
-            beneficiary_id: signers[0].account_id.clone(),
+            beneficiary_id: signers[0].get_account_id(),
         })],
         // runtime does not validate block history
         CryptoHash::default(),
@@ -1471,13 +1471,13 @@ fn test_trie_and_flat_state_equality() {
         .map(|i| AccountId::try_from(format!("test{}", i + 1)).unwrap())
         .collect::<Vec<_>>();
     let mut env = TestEnv::new(vec![validators.clone()], 4, false);
-    let signers: Vec<_> = validators.iter().map(|id| InMemorySigner::test(&id)).collect();
+    let signers: Vec<_> = validators.iter().map(|id| InMemorySigner::test_signer(&id)).collect();
 
     let transfer_tx = SignedTransaction::from_actions(
         4,
-        signers[0].account_id.clone(),
+        signers[0].get_account_id(),
         validators[1].clone(),
-        &signers[0].clone().into(),
+        &signers[0],
         vec![Action::Transfer(TransferAction { deposit: 10 })],
         // runtime does not validate block history
         CryptoHash::default(),
@@ -1559,10 +1559,7 @@ fn test_genesis_hash() {
 /// Creates a signed transaction between each pair of `signers`,
 /// where transactions outcoming from a single signer differ by nonce.
 /// The transactions are then shuffled and used to fill a transaction pool.
-fn generate_transaction_pool(
-    signers: &Vec<InMemorySigner>,
-    block_hash: CryptoHash,
-) -> TransactionPool {
+fn generate_transaction_pool(signers: &Vec<Signer>, block_hash: CryptoHash) -> TransactionPool {
     const TEST_SEED: RngSeed = [3; 32];
     let mut rng = StdRng::from_seed(TEST_SEED);
     let signer_count = signers.len();
@@ -1572,9 +1569,9 @@ fn generate_transaction_pool(
         for i in 0..signer_count {
             let transaction = SignedTransaction::send_money(
                 round.try_into().unwrap(),
-                signers[i].account_id.clone(),
-                signers[(i + round) % signer_count].account_id.clone(),
-                &signers[i].clone().into(),
+                signers[i].get_account_id(),
+                signers[(i + round) % signer_count].get_account_id(),
+                &signers[i],
                 round.try_into().unwrap(),
                 block_hash,
             );
@@ -1627,7 +1624,7 @@ fn get_test_env_with_chain_and_pool() -> (TestEnv, Chain, TransactionPool) {
     // Produce a single block, so that `prev_block_hash` is valid.
     env.step_default(vec![]);
 
-    let signers: Vec<_> = validators.iter().map(|id| InMemorySigner::test(&id)).collect();
+    let signers: Vec<_> = validators.iter().map(|id| InMemorySigner::test_signer(&id)).collect();
 
     let transaction_pool = generate_transaction_pool(&signers, env.head.prev_block_hash);
     (env, chain, transaction_pool)
@@ -1876,7 +1873,7 @@ fn stake(
         nonce,
         sender.validator_id().clone(),
         sender.validator_id().clone(),
-        &*signer,
+        &signer,
         vec![Action::Stake(Box::new(StakeAction { stake, public_key: sender.public_key() }))],
         // runtime does not validate block history
         CryptoHash::default(),

chain/chain/src/signature_verification.rs

@@ -5,9 +5,25 @@ use near_primitives::{
     hash::CryptoHash,
     sharding::{ChunkHash, ShardChunkHeader},
     stateless_validation::ChunkProductionKey,
-    types::{BlockHeight, EpochId, ShardId},
+    types::{validator_stake::ValidatorStake, BlockHeight, EpochId, ShardId},
 };
 
+pub fn verify_block_vrf(
+    validator: ValidatorStake,
+    prev_random_value: &CryptoHash,
+    vrf_value: &near_crypto::vrf::Value,
+    vrf_proof: &near_crypto::vrf::Proof,
+) -> Result<(), Error> {
+    let public_key =
+        near_crypto::key_conversion::convert_public_key(validator.public_key().unwrap_as_ed25519())
+            .unwrap();
+
+    if !public_key.is_vrf_valid(&prev_random_value.as_ref(), vrf_value, vrf_proof) {
+        return Err(Error::InvalidRandomnessBeaconOutput);
+    }
+    Ok(())
+}
+
 /// Verify chunk header signature.
 /// return false if the header signature does not match the key for the assigned chunk producer
 /// for this chunk, or if the chunk producer has been slashed

chain/chain/src/test_utils/kv_runtime.rs

@@ -777,8 +777,16 @@ impl EpochManagerAdapter for MockEpochManager {
         epoch_id: &EpochId,
         height: BlockHeight,
     ) -> Result<AccountId, EpochError> {
+        self.get_block_producer_info(epoch_id, height).map(|validator| validator.take_account_id())
+    }
+
+    fn get_block_producer_info(
+        &self,
+        epoch_id: &EpochId,
+        height: BlockHeight,
+    ) -> Result<ValidatorStake, EpochError> {
         let validators = self.get_block_producers(self.get_valset_for_epoch(epoch_id)?);
-        Ok(validators[(height as usize) % validators.len()].account_id().clone())
+        Ok(validators[(height as usize) % validators.len()].clone())
     }
 
     fn get_chunk_producer_info(
@@ -888,17 +896,6 @@ impl EpochManagerAdapter for MockEpochManager {
         false
     }
 
-    fn verify_block_vrf(
-        &self,
-        _epoch_id: &EpochId,
-        _block_height: BlockHeight,
-        _prev_random_value: &CryptoHash,
-        _vrf_value: &near_crypto::vrf::Value,
-        _vrf_proof: &near_crypto::vrf::Proof,
-    ) -> Result<(), Error> {
-        Ok(())
-    }
-
     fn verify_validator_signature(
         &self,
         _epoch_id: &EpochId,

chain/chain/src/validate.rs

@@ -491,7 +491,7 @@ mod tests {
             nonce,
             account_id,
             "bob".parse().unwrap(),
-            &signer.into(),
+            &signer,
             10,
             CryptoHash::default(),
         )

chain/chunks/src/client.rs

@@ -232,7 +232,7 @@ mod tests {
                 nonce,
                 signer_id.clone(),
                 receiver_id.clone(),
-                &signer.into(),
+                &signer,
                 deposit,
                 CryptoHash::default(),
             );

chain/client/src/test_utils/test_env.rs

@@ -12,7 +12,7 @@ use near_chain_configs::GenesisConfig;
 use near_chain_primitives::error::QueryError;
 use near_chunks::client::ShardsManagerResponse;
 use near_chunks::test_utils::{MockClientAdapterForShardsManager, SynchronousShardsManagerAdapter};
-use near_crypto::InMemorySigner;
+use near_crypto::{InMemorySigner, Signer};
 use near_network::client::ProcessTxResponse;
 use near_network::shards_manager::ShardsManagerRequestFromNetwork;
 use near_network::test_utils::MockPeerManagerAdapter;
@@ -737,15 +737,15 @@ impl TestEnv {
     pub fn tx_from_actions(
         &mut self,
         actions: Vec<Action>,
-        signer: &InMemorySigner,
+        signer: &Signer,
         receiver: AccountId,
     ) -> SignedTransaction {
         let tip = self.clients[0].chain.head().unwrap();
         SignedTransaction::from_actions(
             tip.height + 1,
-            signer.account_id.clone(),
+            signer.get_account_id(),
             receiver,
-            &signer.clone().into(),
+            &signer,
             actions,
             tip.last_block_hash,
             0,
@@ -760,13 +760,13 @@ impl TestEnv {
         relayer: AccountId,
         receiver_id: AccountId,
     ) -> SignedTransaction {
-        let inner_signer = InMemorySigner::test(&sender);
+        let inner_signer = InMemorySigner::test_signer(&sender);
         let relayer_signer = InMemorySigner::test_signer(&relayer);
         let tip = self.clients[0].chain.head().unwrap();
         let user_nonce = tip.height + 1;
         let relayer_nonce = tip.height + 1;
         let delegate_action = DelegateAction {
-            sender_id: inner_signer.account_id.clone(),
+            sender_id: inner_signer.get_account_id(),
             receiver_id,
             actions: actions
                 .into_iter()
@@ -821,14 +821,14 @@ impl TestEnv {
     /// `InMemorySigner::from_seed` produces a valid signer that has it's key
     /// deployed already.
     pub fn call_main(&mut self, account: &AccountId) -> FinalExecutionOutcomeView {
-        let signer = InMemorySigner::test(&account.clone());
+        let signer = InMemorySigner::test_signer(&account);
         let actions = vec![Action::FunctionCall(Box::new(FunctionCallAction {
             method_name: "main".to_string(),
             args: vec![],
             gas: 3 * 10u64.pow(14),
             deposit: 0,
         }))];
-        let tx = self.tx_from_actions(actions, &signer, signer.account_id.clone());
+        let tx = self.tx_from_actions(actions, &signer, signer.get_account_id());
         self.execute_tx(tx).unwrap()
     }
 

chain/epoch-manager/src/adapter.rs

@@ -233,12 +233,20 @@ pub trait EpochManagerAdapter: Send + Sync {
     ) -> Result<Vec<ValidatorStake>, EpochError>;
 
     /// Block producers for given height for the main block. Return EpochError if outside of known boundaries.
+    /// TODO: Deprecate in favour of get_block_producer_info
     fn get_block_producer(
         &self,
         epoch_id: &EpochId,
         height: BlockHeight,
     ) -> Result<AccountId, EpochError>;
 
+    /// Block producers and stake for given height for the main block. Return EpochError if outside of known boundaries.
+    fn get_block_producer_info(
+        &self,
+        epoch_id: &EpochId,
+        height: BlockHeight,
+    ) -> Result<ValidatorStake, EpochError>;
+
     /// Chunk producer info for given height for given shard. Return EpochError if outside of known boundaries.
     fn get_chunk_producer_info(
         &self,
@@ -331,15 +339,6 @@ pub trait EpochManagerAdapter: Send + Sync {
         true
     }
 
-    fn verify_block_vrf(
-        &self,
-        epoch_id: &EpochId,
-        block_height: BlockHeight,
-        prev_random_value: &CryptoHash,
-        vrf_value: &near_crypto::vrf::Value,
-        vrf_proof: &near_crypto::vrf::Proof,
-    ) -> Result<(), Error>;
-
     /// Verify validator signature for the given epoch.
     /// Note: doesn't account for slashed accounts within given epoch. USE WITH CAUTION.
     fn verify_validator_signature(
@@ -760,8 +759,16 @@ impl EpochManagerAdapter for EpochManagerHandle {
         epoch_id: &EpochId,
         height: BlockHeight,
     ) -> Result<AccountId, EpochError> {
+        self.get_block_producer_info(epoch_id, height).map(|validator| validator.take_account_id())
+    }
+
+    fn get_block_producer_info(
+        &self,
+        epoch_id: &EpochId,
+        height: BlockHeight,
+    ) -> Result<ValidatorStake, EpochError> {
         let epoch_manager = self.read();
-        Ok(epoch_manager.get_block_producer_info(epoch_id, height)?.take_account_id())
+        Ok(epoch_manager.get_block_producer_info(epoch_id, height)?)
     }
 
     fn get_chunk_producer_info(
@@ -860,27 +867,6 @@ impl EpochManagerAdapter for EpochManagerHandle {
         )
     }
 
-    fn verify_block_vrf(
-        &self,
-        epoch_id: &EpochId,
-        block_height: BlockHeight,
-        prev_random_value: &CryptoHash,
-        vrf_value: &near_crypto::vrf::Value,
-        vrf_proof: &near_crypto::vrf::Proof,
-    ) -> Result<(), Error> {
-        let epoch_manager = self.read();
-        let validator = epoch_manager.get_block_producer_info(epoch_id, block_height)?;
-        let public_key = near_crypto::key_conversion::convert_public_key(
-            validator.public_key().unwrap_as_ed25519(),
-        )
-        .unwrap();
-
-        if !public_key.is_vrf_valid(&prev_random_value.as_ref(), vrf_value, vrf_proof) {
-            return Err(Error::InvalidRandomnessBeaconOutput);
-        }
-        Ok(())
-    }
-
     fn verify_validator_signature(
         &self,
         epoch_id: &EpochId,

chain/indexer/src/streamer/mod.rs

@@ -100,7 +100,7 @@ pub async fn build_streamer_message(
         })
         .collect::<Vec<_>>();
 
-    for (shard_index, chunk) in chunks.into_iter().enumerate() {
+    for chunk in chunks.into_iter() {
         let views::ChunkView { transactions, author, header, receipts: chunk_non_local_receipts } =
             chunk;
 
@@ -232,7 +232,12 @@ pub async fn build_streamer_message(
         }
 
         chunk_receipts.extend(chunk_non_local_receipts);
-
+        // Find the shard index for the chunk by shard_id
+        let shard_index = protocol_config_view
+            .shard_layout
+            .get_shard_index(header.shard_id)
+            .map_err(|e| FailedToFetchData::String(e.to_string()))?;
+        // Add receipt_execution_outcomes into corresponding indexer shard
         indexer_shards[shard_index].receipt_execution_outcomes = receipt_execution_outcomes;
         // Put the chunk into corresponding indexer shard
         indexer_shards[shard_index].chunk = Some(IndexerChunkView {
@@ -250,7 +255,7 @@ pub async fn build_streamer_message(
         let shard_index = protocol_config_view
             .shard_layout
             .get_shard_index(shard_id)
-            .expect("Failed to get shard index");
+            .map_err(|e| FailedToFetchData::String(e.to_string()))?;
         indexer_shards[shard_index].receipt_execution_outcomes.extend(outcomes.into_iter().map(
             |outcome| IndexerExecutionOutcomeWithReceipt {
                 execution_outcome: outcome.execution_outcome,