feat: Added support for making EN a (non-leader) consensus validator …

…(BFT-426) (#1905)

With this change external nodes will be able to be consensus validators,
while the main node will still be the only leader.

TODO: figure out how to add this setup to the typescript tests.

Cargo.toml

@@ -133,6 +133,7 @@ rlp = "0.5"
 rocksdb = "0.21.0"
 rustc_version = "0.4.0"
 secp256k1 = { version = "0.27.0", features = ["recovery", "global-context"] }
+secrecy = "0.8.0"
 semver = "1"
 sentry = "0.31"
 serde = "1"

core/bin/external_node/src/main.rs

@@ -289,7 +289,7 @@ async fn run_core(
             // but we only need to wait for stop signal once, and it will be propagated to all child contexts.
             let ctx = ctx::root();
             scope::run!(&ctx, |ctx, s| async move {
-                s.spawn_bg(consensus::era::run_fetcher(
+                s.spawn_bg(consensus::era::run_en(
                     ctx,
                     cfg,
                     pool,

core/lib/config/Cargo.toml

@@ -16,5 +16,6 @@ zksync_consensus_utils.workspace = true
 
 anyhow.workspace = true
 rand.workspace = true
+secrecy.workspace = true
 serde = { workspace = true, features = ["derive"] }
 url.workspace = true

core/lib/config/src/configs/consensus.rs

@@ -1,40 +1,70 @@
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    fmt,
-};
+use std::collections::{BTreeMap, BTreeSet};
 
-/// Public key of the validator (consensus participant) of the form "validator:public:<signature scheme>:<hex encoded key material>"
+use secrecy::{ExposeSecret as _, Secret};
+use zksync_basic_types::L2ChainId;
+
+/// `zksync_consensus_crypto::TextFmt` representation of `zksync_consensus_roles::validator::PublicKey`.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct ValidatorPublicKey(pub String);
 
-// Secret key of the validator (consensus participant) of the form "validator:secret:<signature scheme>:<hex encoded key material>"
-#[derive(PartialEq)]
-pub struct ValidatorSecretKey(pub String);
+/// `zksync_consensus_crypto::TextFmt` representation of `zksync_consensus_roles::validator::SecretKey`.
+#[derive(Debug, Clone)]
+pub struct ValidatorSecretKey(pub Secret<String>);
 
-/// Public key of the node (gossip network participant) of the form "node:public:<signature scheme>:<hex encoded key material>"
+/// `zksync_consensus_crypto::TextFmt` representation of `zksync_consensus_roles::node::PublicKey`.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct NodePublicKey(pub String);
 
-// Secret key of the node (gossip network participant) of the form "node:secret:<signature scheme>:<hex encoded key material>"
-#[derive(PartialEq)]
-pub struct NodeSecretKey(pub String);
+/// `zksync_consensus_crypto::TextFmt` representation of `zksync_consensus_roles::node::SecretKey`.
+#[derive(Debug, Clone)]
+pub struct NodeSecretKey(pub Secret<String>);
 
-impl fmt::Debug for ValidatorSecretKey {
-    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        fmt.write_str("<redacted>")
+impl PartialEq for ValidatorSecretKey {
+    fn eq(&self, other: &Self) -> bool {
+        self.0.expose_secret().eq(other.0.expose_secret())
     }
 }
 
-impl fmt::Debug for NodeSecretKey {
-    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        fmt.write_str("<redacted>")
+impl PartialEq for NodeSecretKey {
+    fn eq(&self, other: &Self) -> bool {
+        self.0.expose_secret().eq(other.0.expose_secret())
     }
 }
 
-/// Network address in the `<domain/ip>:port` format.
+/// Copy-paste of `zksync_consensus_roles::validator::WeightedValidator`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WeightedValidator {
+    /// Validator key
+    pub key: ValidatorPublicKey,
+    /// Validator weight inside the Committee.
+    pub weight: u64,
+}
+
+/// Copy-paste of `zksync_concurrency::net::Host`.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Host(pub String);
 
+/// Copy-paste of `zksync_consensus_roles::validator::ProtocolVersion`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct ProtocolVersion(pub u32);
+
+/// Consensus genesis specification.
+/// It is a digest of the `validator::Genesis`,
+/// which allows to initialize genesis (if not present)
+/// decide whether a hard fork is necessary (if present).
+#[derive(Clone, Debug, PartialEq)]
+pub struct GenesisSpec {
+    /// Chain ID.
+    pub chain_id: L2ChainId,
+    /// Consensus protocol version.
+    pub protocol_version: ProtocolVersion,
+    /// The validator committee. Represents `zksync_consensus_roles::validator::Committee`.
+    pub validators: Vec<WeightedValidator>,
+    /// Leader of the committee. Represents
+    /// `zksync_consensus_roles::validator::LeaderSelectionMode::Sticky`.
+    pub leader: ValidatorPublicKey,
+}
+
 /// Config (shared between main node and external node).
 #[derive(Clone, Debug, PartialEq)]
 pub struct ConsensusConfig {
@@ -56,10 +86,15 @@ pub struct ConsensusConfig {
     /// Outbound gossip connections that the node should actively try to
     /// establish and maintain.
     pub gossip_static_outbound: BTreeMap<NodePublicKey, Host>,
+
+    /// MAIN NODE ONLY: consensus genesis specification.
+    /// Used to (re)initialize genesis if needed.
+    /// External nodes fetch the genesis from the main node.
+    pub genesis_spec: Option<GenesisSpec>,
 }
 
 /// Secrets need for consensus.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct ConsensusSecrets {
     pub validator_key: Option<ValidatorSecretKey>,
     pub node_key: Option<NodeSecretKey>,

core/lib/config/src/testonly.rs

@@ -705,6 +705,28 @@ impl Distribution<configs::EcosystemContracts> for EncodeDist {
     }
 }
 
+impl Distribution<configs::consensus::WeightedValidator> for EncodeDist {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::consensus::WeightedValidator {
+        use configs::consensus::{ValidatorPublicKey, WeightedValidator};
+        WeightedValidator {
+            key: ValidatorPublicKey(self.sample(rng)),
+            weight: self.sample(rng),
+        }
+    }
+}
+
+impl Distribution<configs::consensus::GenesisSpec> for EncodeDist {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::consensus::GenesisSpec {
+        use configs::consensus::{GenesisSpec, ProtocolVersion, ValidatorPublicKey};
+        GenesisSpec {
+            chain_id: L2ChainId::default(),
+            protocol_version: ProtocolVersion(self.sample(rng)),
+            validators: self.sample_collect(rng),
+            leader: ValidatorPublicKey(self.sample(rng)),
+        }
+    }
+}
+
 impl Distribution<configs::consensus::ConsensusConfig> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::consensus::ConsensusConfig {
         use configs::consensus::{ConsensusConfig, Host, NodePublicKey};
@@ -721,6 +743,7 @@ impl Distribution<configs::consensus::ConsensusConfig> for EncodeDist {
                 .sample_range(rng)
                 .map(|_| (NodePublicKey(self.sample(rng)), Host(self.sample(rng))))
                 .collect(),
+            genesis_spec: self.sample(rng),
         }
     }
 }
@@ -729,8 +752,8 @@ impl Distribution<configs::consensus::ConsensusSecrets> for EncodeDist {
     fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> configs::consensus::ConsensusSecrets {
         use configs::consensus::{ConsensusSecrets, NodeSecretKey, ValidatorSecretKey};
         ConsensusSecrets {
-            validator_key: self.sample_opt(|| ValidatorSecretKey(self.sample(rng))),
-            node_key: self.sample_opt(|| NodeSecretKey(self.sample(rng))),
+            validator_key: self.sample_opt(|| ValidatorSecretKey(String::into(self.sample(rng)))),
+            node_key: self.sample_opt(|| NodeSecretKey(String::into(self.sample(rng)))),
         }
     }
 }

core/lib/dal/src/consensus_dal.rs

@@ -47,6 +47,8 @@ impl ConsensusDal<'_, '_> {
     }
 
     /// Attempts to update the genesis.
+    /// Fails if the new genesis is invalid.
+    /// Fails if the new genesis has different `chain_id`.
     /// Fails if the storage contains a newer genesis (higher fork number).
     /// Noop if the new genesis is the same as the current one.
     /// Resets the stored consensus state otherwise and purges all certificates.
@@ -57,12 +59,19 @@ impl ConsensusDal<'_, '_> {
             if &got == genesis {
                 return Ok(());
             }
+            anyhow::ensure!(
+                got.chain_id == genesis.chain_id,
+                "changing chain_id is not allowed: old = {:?}, new = {:?}",
+                got.chain_id,
+                genesis.chain_id,
+            );
             anyhow::ensure!(
                 got.fork_number < genesis.fork_number,
                 "transition to a past fork is not allowed: old = {:?}, new = {:?}",
                 got.fork_number,
                 genesis.fork_number,
             );
+            genesis.verify().context("genesis.verify()")?;
         }
         let genesis =
             zksync_protobuf::serde::serialize(genesis, serde_json::value::Serializer).unwrap();
@@ -144,7 +153,7 @@ impl ConsensusDal<'_, '_> {
             fork_number: old.fork_number.next(),
             first_block,
 
-            protocol_version: validator::ProtocolVersion::CURRENT,
+            protocol_version: old.protocol_version,
             committee: old.committee.clone(),
             leader_selection: old.leader_selection.clone(),
         }

core/lib/protobuf_config/Cargo.toml

@@ -22,7 +22,8 @@ zksync_types.workspace = true
 anyhow.workspace = true
 prost.workspace = true
 rand.workspace = true
-hex = "0.4.3"
+hex.workspace = true
+secrecy.workspace = true
 
 [build-dependencies]
 zksync_protobuf_build.workspace = true

core/lib/protobuf_config/src/consensus.rs

@@ -1,10 +1,59 @@
 use anyhow::Context as _;
+use secrecy::ExposeSecret as _;
+use zksync_basic_types::L2ChainId;
 use zksync_config::configs::consensus::{
-    ConsensusConfig, ConsensusSecrets, Host, NodePublicKey, NodeSecretKey, ValidatorSecretKey,
+    ConsensusConfig, ConsensusSecrets, GenesisSpec, Host, NodePublicKey, NodeSecretKey,
+    ProtocolVersion, ValidatorPublicKey, ValidatorSecretKey, WeightedValidator,
 };
 use zksync_protobuf::{repr::ProtoRepr, required};
 
-use crate::proto::consensus as proto;
+use crate::{proto::consensus as proto, read_optional_repr};
+
+impl ProtoRepr for proto::WeightedValidator {
+    type Type = WeightedValidator;
+    fn read(&self) -> anyhow::Result<Self::Type> {
+        Ok(Self::Type {
+            key: ValidatorPublicKey(required(&self.key).context("key")?.clone()),
+            weight: *required(&self.weight).context("weight")?,
+        })
+    }
+    fn build(this: &Self::Type) -> Self {
+        Self {
+            key: Some(this.key.0.clone()),
+            weight: Some(this.weight),
+        }
+    }
+}
+
+impl ProtoRepr for proto::GenesisSpec {
+    type Type = GenesisSpec;
+    fn read(&self) -> anyhow::Result<Self::Type> {
+        Ok(Self::Type {
+            chain_id: required(&self.chain_id)
+                .and_then(|x| L2ChainId::try_from(*x).map_err(|a| anyhow::anyhow!(a)))
+                .context("chain_id")?,
+            protocol_version: ProtocolVersion(
+                *required(&self.protocol_version).context("protocol_version")?,
+            ),
+            validators: self
+                .validators
+                .iter()
+                .enumerate()
+                .map(|(i, x)| x.read().context(i))
+                .collect::<Result<_, _>>()
+                .context("validators")?,
+            leader: ValidatorPublicKey(required(&self.leader).context("leader")?.clone()),
+        })
+    }
+    fn build(this: &Self::Type) -> Self {
+        Self {
+            chain_id: Some(this.chain_id.as_u64()),
+            protocol_version: Some(this.protocol_version.0),
+            validators: this.validators.iter().map(ProtoRepr::build).collect(),
+            leader: Some(this.leader.0.clone()),
+        }
+    }
+}
 
 impl ProtoRepr for proto::Config {
     type Type = ConsensusConfig;
@@ -36,6 +85,7 @@ impl ProtoRepr for proto::Config {
                 .enumerate()
                 .map(|(i, e)| read_addr(e).context(i))
                 .collect::<Result<_, _>>()?,
+            genesis_spec: read_optional_repr(&self.genesis_spec).context("genesis_spec")?,
         })
     }
 
@@ -60,6 +110,7 @@ impl ProtoRepr for proto::Config {
                     addr: Some(x.1 .0.clone()),
                 })
                 .collect(),
+            genesis_spec: this.genesis_spec.as_ref().map(ProtoRepr::build),
         }
     }
 }
@@ -71,15 +122,21 @@ impl ProtoRepr for proto::Secrets {
             validator_key: self
                 .validator_key
                 .as_ref()
-                .map(|x| ValidatorSecretKey(x.clone())),
-            node_key: self.node_key.as_ref().map(|x| NodeSecretKey(x.clone())),
+                .map(|x| ValidatorSecretKey(x.clone().into())),
+            node_key: self
+                .node_key
+                .as_ref()
+                .map(|x| NodeSecretKey(x.clone().into())),
         })
     }
 
     fn build(this: &Self::Type) -> Self {
         Self {
-            validator_key: this.validator_key.as_ref().map(|x| x.0.clone()),
-            node_key: this.node_key.as_ref().map(|x| x.0.clone()),
+            validator_key: this
+                .validator_key
+                .as_ref()
+                .map(|x| x.0.expose_secret().clone()),
+            node_key: this.node_key.as_ref().map(|x| x.0.expose_secret().clone()),
         }
     }
 }