From bf3400fa72db56178a34458f56543ffb2ffc272b Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Fri, 17 Jan 2025 09:33:59 +0000 Subject: [PATCH 1/7] feat(optimistic_block): Introduce the optimistic block struct This PR introduces the shape of the optimistic block described in #10584. Along with it I have added the functions to create and sign it. In the next PR, I will link this in the path of the block production. --- core/primitives/src/lib.rs | 1 + core/primitives/src/optimistic_block.rs | 76 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 core/primitives/src/optimistic_block.rs diff --git a/core/primitives/src/lib.rs b/core/primitives/src/lib.rs index dd0126e7ae4..2109759a947 100644 --- a/core/primitives/src/lib.rs +++ b/core/primitives/src/lib.rs @@ -22,6 +22,7 @@ pub mod epoch_sync; pub mod errors; pub mod merkle; pub mod network; +pub mod optimistic_block; pub mod profile_data_v2; pub mod profile_data_v3; pub mod rand; diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs new file mode 100644 index 00000000000..6102fb05020 --- /dev/null +++ b/core/primitives/src/optimistic_block.rs @@ -0,0 +1,76 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use near_crypto::Signature; + +use crate::block::BlockHeader; +use crate::hash::{hash, CryptoHash}; +use crate::types::BlockHeight; + +#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +pub struct OptimisticBlock { + pub block_height: BlockHeight, + pub prev_block_hash: CryptoHash, + pub block_timestamp: u64, + + // Data to confirm the correctness of randomness beacon output + pub random_value: CryptoHash, + pub vrf_value: near_crypto::vrf::Value, + pub vrf_proof: near_crypto::vrf::Proof, + + /// Signature of the block producer. + pub signature: Signature, + pub hash: CryptoHash, +} + +impl OptimisticBlock { + #[cfg(feature = "clock")] + pub fn new( + prev: &BlockHeader, + height: BlockHeight, + signer: &crate::validator_signer::ValidatorSigner, + clock: near_time::Clock, + sandbox_delta_time: Option, + ) -> Self { + let prev_block_hash = *prev.hash(); + let (vrf_value, vrf_proof) = signer.compute_vrf_with_proof(prev.random_value().as_ref()); + let random_value = hash(vrf_value.0.as_ref()); + + let now = clock.now_utc().unix_timestamp_nanos() as u64; + #[cfg(feature = "sandbox")] + let now = now + sandbox_delta_time.unwrap().whole_nanoseconds() as u64; + #[cfg(not(feature = "sandbox"))] + debug_assert!(sandbox_delta_time.is_none()); + let time = if now <= prev.raw_timestamp() { prev.raw_timestamp() + 1 } else { now }; + + Self { + block_height: height, + prev_block_hash, + block_timestamp: time, + random_value, + vrf_value, + vrf_proof, + signature: Default::default(), + hash: Default::default(), + } + } + + pub fn produce_optimistic( + prev: &BlockHeader, + height: BlockHeight, + signer: &crate::validator_signer::ValidatorSigner, + clock: near_time::Clock, + sandbox_delta_time: Option, + ) -> Self { + let mut optimistic_block = Self::new(prev, height, signer, clock, sandbox_delta_time); + optimistic_block.hash_and_sign(signer); + optimistic_block + } + + pub fn hash_and_sign(&mut self, signer: &crate::validator_signer::ValidatorSigner) { + self.signature = Default::default(); + self.hash = Default::default(); + let hash = hash(&borsh::to_vec(self).expect("Failed to serialize")); + let signature = signer.sign_bytes(hash.as_ref()); + self.signature = signature; + self.hash = hash; + } +} From 4a65ac439ac716b73095505a2c91e9419fa8ec6f Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Fri, 17 Jan 2025 14:10:18 +0000 Subject: [PATCH 2/7] Introduce Inner field, change hashing logic --- core/primitives/src/optimistic_block.rs | 63 +++++++++++++------------ 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs index 6102fb05020..0bda814e699 100644 --- a/core/primitives/src/optimistic_block.rs +++ b/core/primitives/src/optimistic_block.rs @@ -3,35 +3,44 @@ use near_crypto::Signature; use crate::block::BlockHeader; use crate::hash::{hash, CryptoHash}; +use crate::merkle::combine_hash; use crate::types::BlockHeight; #[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] -pub struct OptimisticBlock { +pub struct OptimisticBlockInner { pub block_height: BlockHeight, - pub prev_block_hash: CryptoHash, pub block_timestamp: u64, - // Data to confirm the correctness of randomness beacon output pub random_value: CryptoHash, pub vrf_value: near_crypto::vrf::Value, pub vrf_proof: near_crypto::vrf::Proof, +} +/// An optimistic block is independent of specific chunks and can be generated +/// and distributed immediately after the previous block is processed. +#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +#[borsh(init=init)] +pub struct OptimisticBlock { + pub prev_block_hash: CryptoHash, + pub inner: OptimisticBlockInner, /// Signature of the block producer. pub signature: Signature, + #[borsh(skip)] pub hash: CryptoHash, } impl OptimisticBlock { #[cfg(feature = "clock")] - pub fn new( - prev: &BlockHeader, + pub fn produce( + prev_block_header: &BlockHeader, height: BlockHeight, signer: &crate::validator_signer::ValidatorSigner, clock: near_time::Clock, sandbox_delta_time: Option, ) -> Self { - let prev_block_hash = *prev.hash(); - let (vrf_value, vrf_proof) = signer.compute_vrf_with_proof(prev.random_value().as_ref()); + let prev_block_hash = *prev_block_header.hash(); + let (vrf_value, vrf_proof) = + signer.compute_vrf_with_proof(prev_block_header.random_value().as_ref()); let random_value = hash(vrf_value.0.as_ref()); let now = clock.now_utc().unix_timestamp_nanos() as u64; @@ -39,38 +48,30 @@ impl OptimisticBlock { let now = now + sandbox_delta_time.unwrap().whole_nanoseconds() as u64; #[cfg(not(feature = "sandbox"))] debug_assert!(sandbox_delta_time.is_none()); - let time = if now <= prev.raw_timestamp() { prev.raw_timestamp() + 1 } else { now }; + let time = if now <= prev_block_header.raw_timestamp() { + prev_block_header.raw_timestamp() + 1 + } else { + now + }; - Self { + let inner = OptimisticBlockInner { block_height: height, - prev_block_hash, block_timestamp: time, random_value, vrf_value, vrf_proof, - signature: Default::default(), - hash: Default::default(), - } - } + }; - pub fn produce_optimistic( - prev: &BlockHeader, - height: BlockHeight, - signer: &crate::validator_signer::ValidatorSigner, - clock: near_time::Clock, - sandbox_delta_time: Option, - ) -> Self { - let mut optimistic_block = Self::new(prev, height, signer, clock, sandbox_delta_time); - optimistic_block.hash_and_sign(signer); - optimistic_block + let inner_hash = hash(&borsh::to_vec(&inner).expect("Failed to serialize")); + let hash = combine_hash(&inner_hash, &prev_block_hash); + let signature = signer.sign_bytes(hash.as_ref()); + + Self { prev_block_hash, inner, signature, hash } } - pub fn hash_and_sign(&mut self, signer: &crate::validator_signer::ValidatorSigner) { - self.signature = Default::default(); - self.hash = Default::default(); - let hash = hash(&borsh::to_vec(self).expect("Failed to serialize")); - let signature = signer.sign_bytes(hash.as_ref()); - self.signature = signature; - self.hash = hash; + /// Recompute the hash after deserialization. + pub fn init(&mut self) { + let inner_hash = hash(&borsh::to_vec(&self.inner).expect("Failed to serialize")); + self.hash = combine_hash(&inner_hash, &self.prev_block_hash); } } From 53fff6aaafd11c2300581e0168fabbc59c917298 Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Fri, 17 Jan 2025 16:59:39 +0000 Subject: [PATCH 3/7] Add signature differentiator --- core/primitives/src/optimistic_block.rs | 3 +++ core/primitives/src/stateless_validation/mod.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs index 0bda814e699..54dfb471aa7 100644 --- a/core/primitives/src/optimistic_block.rs +++ b/core/primitives/src/optimistic_block.rs @@ -4,10 +4,12 @@ use near_crypto::Signature; use crate::block::BlockHeader; use crate::hash::{hash, CryptoHash}; use crate::merkle::combine_hash; +use crate::stateless_validation::SignatureDifferentiator; use crate::types::BlockHeight; #[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] pub struct OptimisticBlockInner { + signature_differentiator: SignatureDifferentiator, pub block_height: BlockHeight, pub block_timestamp: u64, // Data to confirm the correctness of randomness beacon output @@ -55,6 +57,7 @@ impl OptimisticBlock { }; let inner = OptimisticBlockInner { + signature_differentiator: "OptimisticBlock".to_owned(), block_height: height, block_timestamp: time, random_value, diff --git a/core/primitives/src/stateless_validation/mod.rs b/core/primitives/src/stateless_validation/mod.rs index 72cbdaeb912..d2871a39a67 100644 --- a/core/primitives/src/stateless_validation/mod.rs +++ b/core/primitives/src/stateless_validation/mod.rs @@ -18,7 +18,7 @@ pub mod validator_assignment; /// this signature means something different. /// /// This is a messy workaround until we know what to do with NEP 483. -type SignatureDifferentiator = String; +pub type SignatureDifferentiator = String; /// This struct contains combination of fields that uniquely identify chunk production. /// It means that for a given instance only one chunk could be produced. From 8fcade96dadc32827e6fe8ff97a4388f5e5b14dc Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Fri, 17 Jan 2025 17:02:39 +0000 Subject: [PATCH 4/7] Comment --- core/primitives/src/optimistic_block.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs index 54dfb471aa7..e372e55408e 100644 --- a/core/primitives/src/optimistic_block.rs +++ b/core/primitives/src/optimistic_block.rs @@ -20,6 +20,8 @@ pub struct OptimisticBlockInner { /// An optimistic block is independent of specific chunks and can be generated /// and distributed immediately after the previous block is processed. +/// This block is shared with the validators and used to optimistically process +/// chunks before they get included in the block. #[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] #[borsh(init=init)] pub struct OptimisticBlock { From 6e429717e82e4bd489098f7509e2e0c81e04620d Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Mon, 20 Jan 2025 09:51:45 +0000 Subject: [PATCH 5/7] move previous block hash to inner --- core/primitives/src/optimistic_block.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs index e372e55408e..9806d54f062 100644 --- a/core/primitives/src/optimistic_block.rs +++ b/core/primitives/src/optimistic_block.rs @@ -3,19 +3,19 @@ use near_crypto::Signature; use crate::block::BlockHeader; use crate::hash::{hash, CryptoHash}; -use crate::merkle::combine_hash; use crate::stateless_validation::SignatureDifferentiator; use crate::types::BlockHeight; #[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] pub struct OptimisticBlockInner { - signature_differentiator: SignatureDifferentiator, + pub prev_block_hash: CryptoHash, pub block_height: BlockHeight, pub block_timestamp: u64, // Data to confirm the correctness of randomness beacon output pub random_value: CryptoHash, pub vrf_value: near_crypto::vrf::Value, pub vrf_proof: near_crypto::vrf::Proof, + signature_differentiator: SignatureDifferentiator, } /// An optimistic block is independent of specific chunks and can be generated @@ -25,7 +25,6 @@ pub struct OptimisticBlockInner { #[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] #[borsh(init=init)] pub struct OptimisticBlock { - pub prev_block_hash: CryptoHash, pub inner: OptimisticBlockInner, /// Signature of the block producer. pub signature: Signature, @@ -59,24 +58,23 @@ impl OptimisticBlock { }; let inner = OptimisticBlockInner { - signature_differentiator: "OptimisticBlock".to_owned(), + prev_block_hash, block_height: height, block_timestamp: time, random_value, vrf_value, vrf_proof, + signature_differentiator: "OptimisticBlock".to_owned(), }; - let inner_hash = hash(&borsh::to_vec(&inner).expect("Failed to serialize")); - let hash = combine_hash(&inner_hash, &prev_block_hash); + let hash = hash(&borsh::to_vec(&inner).expect("Failed to serialize")); let signature = signer.sign_bytes(hash.as_ref()); - Self { prev_block_hash, inner, signature, hash } + Self { inner, signature, hash } } /// Recompute the hash after deserialization. pub fn init(&mut self) { - let inner_hash = hash(&borsh::to_vec(&self.inner).expect("Failed to serialize")); - self.hash = combine_hash(&inner_hash, &self.prev_block_hash); + self.hash = hash(&borsh::to_vec(&self.inner).expect("Failed to serialize")); } } From c53e836984e14a3af8c9e8caae985e6c0f578ed5 Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Mon, 20 Jan 2025 10:05:22 +0000 Subject: [PATCH 6/7] Refactor SignatureDifferentiator --- core/primitives/src/optimistic_block.rs | 3 +-- .../src/stateless_validation/chunk_endorsement.rs | 4 ++-- .../src/stateless_validation/contract_distribution.rs | 4 ++-- core/primitives/src/stateless_validation/mod.rs | 8 -------- .../src/stateless_validation/partial_witness.rs | 4 ++-- core/primitives/src/stateless_validation/state_witness.rs | 4 ++-- core/primitives/src/types.rs | 8 ++++++++ 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs index 9806d54f062..40ea2172229 100644 --- a/core/primitives/src/optimistic_block.rs +++ b/core/primitives/src/optimistic_block.rs @@ -3,8 +3,7 @@ use near_crypto::Signature; use crate::block::BlockHeader; use crate::hash::{hash, CryptoHash}; -use crate::stateless_validation::SignatureDifferentiator; -use crate::types::BlockHeight; +use crate::types::{BlockHeight, SignatureDifferentiator}; #[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] pub struct OptimisticBlockInner { diff --git a/core/primitives/src/stateless_validation/chunk_endorsement.rs b/core/primitives/src/stateless_validation/chunk_endorsement.rs index d04e17b01a6..b36f7a1f5bb 100644 --- a/core/primitives/src/stateless_validation/chunk_endorsement.rs +++ b/core/primitives/src/stateless_validation/chunk_endorsement.rs @@ -1,14 +1,14 @@ use std::fmt::Debug; use crate::sharding::{ChunkHash, ShardChunkHeader}; -use crate::types::EpochId; +use crate::types::{EpochId, SignatureDifferentiator}; use crate::validator_signer::ValidatorSigner; use borsh::{BorshDeserialize, BorshSerialize}; use near_crypto::{PublicKey, Signature}; use near_primitives_core::types::{AccountId, BlockHeight, ShardId}; use near_schema_checker_lib::ProtocolSchema; -use super::{ChunkProductionKey, SignatureDifferentiator}; +use super::ChunkProductionKey; /// The endorsement of a chunk by a chunk validator. By providing this, a /// chunk validator has verified that the chunk state witness is correct. diff --git a/core/primitives/src/stateless_validation/contract_distribution.rs b/core/primitives/src/stateless_validation/contract_distribution.rs index 214b55e84bf..faabe79e980 100644 --- a/core/primitives/src/stateless_validation/contract_distribution.rs +++ b/core/primitives/src/stateless_validation/contract_distribution.rs @@ -9,12 +9,12 @@ use near_primitives_core::hash::{hash, CryptoHash}; use near_primitives_core::types::{AccountId, ShardId}; use near_schema_checker_lib::ProtocolSchema; +use super::ChunkProductionKey; #[cfg(feature = "solomon")] use crate::reed_solomon::{ReedSolomonEncoderDeserialize, ReedSolomonEncoderSerialize}; +use crate::types::SignatureDifferentiator; use crate::{utils::compression::CompressedData, validator_signer::ValidatorSigner}; -use super::{ChunkProductionKey, SignatureDifferentiator}; - // Data structures for chunk producers to send accessesed contracts to chunk validators. /// Contains contracts (as code-hashes) accessed during the application of a chunk. diff --git a/core/primitives/src/stateless_validation/mod.rs b/core/primitives/src/stateless_validation/mod.rs index d2871a39a67..272bfeef23e 100644 --- a/core/primitives/src/stateless_validation/mod.rs +++ b/core/primitives/src/stateless_validation/mod.rs @@ -12,14 +12,6 @@ pub mod state_witness; pub mod stored_chunk_state_transition_data; pub mod validator_assignment; -/// An arbitrary static string to make sure that this struct cannot be -/// serialized to look identical to another serialized struct. For chunk -/// production we are signing a chunk hash, so we need to make sure that -/// this signature means something different. -/// -/// This is a messy workaround until we know what to do with NEP 483. -pub type SignatureDifferentiator = String; - /// This struct contains combination of fields that uniquely identify chunk production. /// It means that for a given instance only one chunk could be produced. #[derive(Debug, Hash, PartialEq, Eq, Clone, BorshSerialize, BorshDeserialize, ProtocolSchema)] diff --git a/core/primitives/src/stateless_validation/partial_witness.rs b/core/primitives/src/stateless_validation/partial_witness.rs index 44d235b1657..cc944e36eea 100644 --- a/core/primitives/src/stateless_validation/partial_witness.rs +++ b/core/primitives/src/stateless_validation/partial_witness.rs @@ -1,8 +1,8 @@ use std::fmt::{Debug, Formatter}; -use super::{ChunkProductionKey, SignatureDifferentiator}; +use super::ChunkProductionKey; use crate::sharding::ShardChunkHeader; -use crate::types::EpochId; +use crate::types::{EpochId, SignatureDifferentiator}; use crate::validator_signer::ValidatorSigner; use borsh::{BorshDeserialize, BorshSerialize}; use bytesize::ByteSize; diff --git a/core/primitives/src/stateless_validation/state_witness.rs b/core/primitives/src/stateless_validation/state_witness.rs index f6c1aff510b..d5ed9d6d249 100644 --- a/core/primitives/src/stateless_validation/state_witness.rs +++ b/core/primitives/src/stateless_validation/state_witness.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::fmt::Debug; -use super::{ChunkProductionKey, SignatureDifferentiator}; +use super::ChunkProductionKey; use crate::bandwidth_scheduler::BandwidthRequests; use crate::challenge::PartialState; use crate::congestion_info::CongestionInfo; @@ -9,7 +9,7 @@ use crate::congestion_info::CongestionInfo; use crate::reed_solomon::{ReedSolomonEncoderDeserialize, ReedSolomonEncoderSerialize}; use crate::sharding::{ChunkHash, ReceiptProof, ShardChunkHeader, ShardChunkHeaderV3}; use crate::transaction::SignedTransaction; -use crate::types::EpochId; +use crate::types::{EpochId, SignatureDifferentiator}; use crate::utils::compression::CompressedData; use crate::validator_signer::EmptyValidatorSigner; use borsh::{BorshDeserialize, BorshSerialize}; diff --git a/core/primitives/src/types.rs b/core/primitives/src/types.rs index bf39509f232..dfb9d61548d 100644 --- a/core/primitives/src/types.rs +++ b/core/primitives/src/types.rs @@ -22,6 +22,14 @@ pub use chunk_validator_stats::ChunkStats; /// Hash used by to store state root. pub type StateRoot = CryptoHash; +/// An arbitrary static string to make sure that this struct cannot be +/// serialized to look identical to another serialized struct. For chunk +/// production we are signing a chunk hash, so we need to make sure that +/// this signature means something different. +/// +/// This is a messy workaround until we know what to do with NEP 483. +pub(crate) type SignatureDifferentiator = String; + /// Different types of finality. #[derive( serde::Serialize, serde::Deserialize, Default, Clone, Debug, PartialEq, Eq, arbitrary::Arbitrary, From 702fae693d68c2785ad71e775dbd5921daecbe33 Mon Sep 17 00:00:00 2001 From: Razvan Barbascu Date: Mon, 20 Jan 2025 12:44:56 +0000 Subject: [PATCH 7/7] Protocol Schema --- core/primitives/src/optimistic_block.rs | 5 +++-- tools/protocol-schema-check/res/protocol_schema.toml | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/primitives/src/optimistic_block.rs b/core/primitives/src/optimistic_block.rs index 40ea2172229..3b90edde385 100644 --- a/core/primitives/src/optimistic_block.rs +++ b/core/primitives/src/optimistic_block.rs @@ -1,11 +1,12 @@ use borsh::{BorshDeserialize, BorshSerialize}; use near_crypto::Signature; +use near_schema_checker_lib::ProtocolSchema; use crate::block::BlockHeader; use crate::hash::{hash, CryptoHash}; use crate::types::{BlockHeight, SignatureDifferentiator}; -#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq, ProtocolSchema)] pub struct OptimisticBlockInner { pub prev_block_hash: CryptoHash, pub block_height: BlockHeight, @@ -21,7 +22,7 @@ pub struct OptimisticBlockInner { /// and distributed immediately after the previous block is processed. /// This block is shared with the validators and used to optimistically process /// chunks before they get included in the block. -#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq)] +#[derive(BorshSerialize, BorshDeserialize, Clone, Debug, Eq, PartialEq, ProtocolSchema)] #[borsh(init=init)] pub struct OptimisticBlock { pub inner: OptimisticBlockInner, diff --git a/tools/protocol-schema-check/res/protocol_schema.toml b/tools/protocol-schema-check/res/protocol_schema.toml index c98b8c73c0e..b8e6cef13fa 100644 --- a/tools/protocol-schema-check/res/protocol_schema.toml +++ b/tools/protocol-schema-check/res/protocol_schema.toml @@ -167,6 +167,8 @@ MethodResolveError = 1206790835 MissingTrieValueContext = 2666011379 NextEpochValidatorInfo = 3660299258 NonDelegateAction = 2970737551 +OptimisticBlock = 1384126355 +OptimisticBlockInner = 1534008891 ParentSplitParameters = 2945469052 PartialEdgeInfo = 1350359189 PartialEncodedChunk = 2321210648