Merge branch 'master' into standard-http

Cargo.toml

@@ -46,6 +46,7 @@ members = [
     "consensus/ssz",
     "consensus/ssz_derive",
     "consensus/ssz_types",
+    "consensus/serde_hex",
     "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",

beacon_node/beacon_chain/Cargo.toml

@@ -58,3 +58,4 @@ environment = { path = "../../lighthouse/environment" }
 bus = "2.2.3"
 derivative = "2.1.1"
 itertools = "0.9.0"
+regex = "1.3.9"

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -31,6 +31,7 @@ use fork_choice::ForkChoice;
 use itertools::process_results;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
+use regex::bytes::Regex;
 use slog::{crit, debug, error, info, trace, warn, Logger};
 use slot_clock::SlotClock;
 use state_processing::{
@@ -1362,8 +1363,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         block: SignedBeaconBlock<T::EthSpec>,
     ) -> Result<GossipVerifiedBlock<T>, BlockError<T::EthSpec>> {
         let slot = block.message.slot;
-        let graffiti_string = String::from_utf8(block.message.body.graffiti[..].to_vec())
-            .unwrap_or_else(|_| format!("{:?}", &block.message.body.graffiti[..]));
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("regex is valid");
+        let graffiti_string =
+            String::from_utf8_lossy(&re.replace_all(&block.message.body.graffiti[..], &b""[..]))
+                .to_string();
 
         match GossipVerifiedBlock::new(block, self) {
             Ok(verified) => {

beacon_node/src/config.rs

@@ -236,8 +236,8 @@ pub fn get_config<E: EthSpec>(
 
     client_config.eth1.deposit_contract_address =
         format!("{:?}", eth2_testnet_config.deposit_contract_address()?);
-    if client_config.eth1.deposit_contract_address != spec.deposit_contract_address {
-        return Error("Testnet contract address does not match spec".into());
+    if client_config.eth1.deposit_contract_address != spec.deposit_contract_address.to_string() {
+        return Err("Testnet contract address does not match spec".into());
     }
 
     client_config.eth1.deposit_contract_deploy_block =

common/eth2/src/types.rs

@@ -6,7 +6,7 @@ use types::serde_utils;
 pub use types::{
     Address, Attestation, AttesterSlashing, BeaconBlockHeader, BeaconState, Checkpoint, Epoch,
     EthSpec, Fork, Hash256, ProposerSlashing, PublicKeyBytes, SignatureBytes, SignedBeaconBlock,
-    SignedVoluntaryExit, Slot, Validator, YamlConfig, U256,
+    SignedVoluntaryExit, Slot, Validator, YamlConfig,
 };
 
 /// An API error serializable to JSON.
@@ -20,7 +20,7 @@ pub struct ErrorMessage {
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct GenesisData {
-    #[serde(with = "serde_utils::quoted")]
+    #[serde(with = "serde_utils::quoted_u64")]
     pub genesis_time: u64,
     pub genesis_validators_root: Hash256,
     #[serde(with = "serde_utils::fork_bytes_4")]
@@ -198,9 +198,9 @@ impl fmt::Display for ValidatorId {
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ValidatorData {
-    #[serde(with = "serde_utils::quoted")]
+    #[serde(with = "serde_utils::quoted_u64")]
     pub index: u64,
-    #[serde(with = "serde_utils::quoted")]
+    #[serde(with = "serde_utils::quoted_u64")]
     pub balance: u64,
     pub status: ValidatorStatus,
     pub validator: Validator,
@@ -279,7 +279,7 @@ pub struct CommitteesQuery {
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct CommitteeData {
-    #[serde(with = "serde_utils::quoted")]
+    #[serde(with = "serde_utils::quoted_u64")]
     pub index: u64,
     pub slot: Slot,
     #[serde(with = "serde_utils::quoted_u64_vec")]
@@ -307,7 +307,7 @@ pub struct BlockHeaderData {
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct DepositContractData {
-    #[serde(with = "serde_utils::quoted")]
+    #[serde(with = "serde_utils::quoted_u64")]
     pub chain_id: u64,
     pub address: Address,
 }

consensus/serde_hex/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "serde_hex"
+version = "0.2.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+serde = "1.0.110"
+hex = "0.4.2"

consensus/serde_hex/src/lib.rs

@@ -0,0 +1,69 @@
+use serde::de::{self, Visitor};
+use std::fmt;
+
+pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
+    let hex = hex::encode(data);
+    let mut s = "0x".to_string();
+    s.push_str(hex.as_str());
+    s
+}
+
+pub struct PrefixedHexVisitor;
+
+impl<'de> Visitor<'de> for PrefixedHexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string with 0x prefix")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.starts_with("0x") {
+            Ok(hex::decode(&value[2..])
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+        } else {
+            Err(de::Error::custom("missing 0x prefix"))
+        }
+    }
+}
+
+pub struct HexVisitor;
+
+impl<'de> Visitor<'de> for HexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string (irrelevant of prefix)")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        Ok(hex::decode(value.trim_start_matches("0x"))
+            .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn encoding() {
+        let bytes = vec![0, 255];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x00ff");
+
+        let bytes = vec![];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x");
+
+        let bytes = vec![1, 2, 3];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x010203");
+    }
+}

consensus/serde_utils/Cargo.toml

@@ -1,9 +1,13 @@
 [package]
 name = "serde_utils"
-version = "0.2.0"
-authors = ["Paul Hauner <paul@paulhauner.com>"]
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com", "Michael Sproul <michael@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
-serde = "1.0.110"
+serde = { version = "1.0.110", features = ["derive"] }
+serde_derive = "1.0.110"
 hex = "0.4.2"
+
+[dev-dependencies]
+serde_json = "1.0.52"

consensus/serde_utils/src/lib.rs

@@ -1,2 +1,3 @@
 pub mod hex;
-pub mod quoted;
+pub mod quoted_u64;
+pub mod quoted_u64_vec;

consensus/serde_utils/src/quoted_u64.rs

@@ -0,0 +1,115 @@
+use serde::{Deserializer, Serializer};
+use serde_derive::{Deserialize, Serialize};
+use std::marker::PhantomData;
+
+/// Serde support for deserializing quoted integers.
+///
+/// Configurable so that quotes are either required or optional.
+pub struct QuotedIntVisitor<T> {
+    require_quotes: bool,
+    _phantom: PhantomData<T>,
+}
+
+impl<'a, T> serde::de::Visitor<'a> for QuotedIntVisitor<T>
+where
+    T: From<u64> + Into<u64> + Copy,
+{
+    type Value = T;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        if self.require_quotes {
+            write!(formatter, "a quoted integer")
+        } else {
+            write!(formatter, "a quoted or unquoted integer")
+        }
+    }
+
+    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        s.parse::<u64>()
+            .map(T::from)
+            .map_err(serde::de::Error::custom)
+    }
+
+    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        if self.require_quotes {
+            Err(serde::de::Error::custom(
+                "received unquoted integer when quotes are required",
+            ))
+        } else {
+            Ok(T::from(v))
+        }
+    }
+}
+
+/// Wrapper type for requiring quotes on a `u64`-like type.
+///
+/// Unlike using `serde(with = "quoted_u64::require_quotes")` this is composable, and can be nested
+/// inside types like `Option`, `Result` and `Vec`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
+#[serde(transparent)]
+pub struct Quoted<T>
+where
+    T: From<u64> + Into<u64> + Copy,
+{
+    #[serde(with = "require_quotes")]
+    pub value: T,
+}
+
+/// Serialize with quotes.
+pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    T: From<u64> + Into<u64> + Copy,
+{
+    let v: u64 = (*value).into();
+    serializer.serialize_str(&format!("{}", v))
+}
+
+/// Deserialize with or without quotes.
+pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+where
+    D: Deserializer<'de>,
+    T: From<u64> + Into<u64> + Copy,
+{
+    deserializer.deserialize_any(QuotedIntVisitor {
+        require_quotes: false,
+        _phantom: PhantomData,
+    })
+}
+
+/// Requires quotes when deserializing.
+///
+/// Usage: `#[serde(with = "quoted_u64::require_quotes")]`.
+pub mod require_quotes {
+    pub use super::serialize;
+    use super::*;
+
+    pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
+    where
+        D: Deserializer<'de>,
+        T: From<u64> + Into<u64> + Copy,
+    {
+        deserializer.deserialize_any(QuotedIntVisitor {
+            require_quotes: true,
+            _phantom: PhantomData,
+        })
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn require_quotes() {
+        let x = serde_json::from_str::<Quoted<u64>>("\"8\"").unwrap();
+        assert_eq!(x.value, 8);
+        serde_json::from_str::<Quoted<u64>>("8").unwrap_err();
+    }
+}