Merge pull request #93 from teor2345/external-size-vec-serialize

External size vec serialize

tower-batch/Cargo.toml

@@ -6,7 +6,7 @@ license = "MIT"
 edition = "2018"
 
 [dependencies]
-futures = "0.3.13"
+futures = "0.3.14"
 futures-core = "0.3.13"
 pin-project = "0.4.27"
 tokio = { version = "0.3.6", features = ["time", "sync", "stream", "tracing", "macros"] }
@@ -15,7 +15,7 @@ tracing = "0.1.25"
 tracing-futures = "0.2.5"
 
 [dev-dependencies]
-color-eyre = "0.5.10"
+color-eyre = "0.5.11"
 ed25519-zebra = "2.2.0"
 rand = "0.8"
 tokio = { version = "0.3.6", features = ["full"]}

zebra-chain/Cargo.toml

@@ -46,7 +46,7 @@ redjubjub = "0.4"
 
 [dev-dependencies]
 bincode = "1"
-color-eyre = "0.5.10"
+color-eyre = "0.5.11"
 spandoc = "0.2"
 tracing = "0.1.25"
 proptest = "0.10"

zebra-chain/src/serialization.rs

@@ -19,8 +19,14 @@ pub mod sha256d;
 pub use error::SerializationError;
 pub use read_zcash::ReadZcashExt;
 pub use write_zcash::WriteZcashExt;
-pub use zcash_deserialize::{TrustedPreallocate, ZcashDeserialize, ZcashDeserializeInto};
-pub use zcash_serialize::{ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN};
+pub use zcash_deserialize::{
+    zcash_deserialize_bytes_external_count, zcash_deserialize_external_count, TrustedPreallocate,
+    ZcashDeserialize, ZcashDeserializeInto,
+};
+pub use zcash_serialize::{
+    zcash_serialize_bytes_external_count, zcash_serialize_external_count, ZcashSerialize,
+    MAX_PROTOCOL_MESSAGE_LEN,
+};
 
 #[cfg(test)]
-mod proptests;
+mod tests;

zebra-chain/src/serialization/tests.rs

@@ -0,0 +1,2 @@
+mod preallocate;
+mod prop;

zebra-chain/src/serialization/tests/preallocate.rs

@@ -0,0 +1,101 @@
+//! Tests for trusted preallocation during deserialization.
+
+use proptest::{collection::size_range, prelude::*};
+
+use std::matches;
+
+use crate::serialization::{
+    zcash_deserialize::MAX_U8_ALLOCATION, SerializationError, ZcashDeserialize, ZcashSerialize,
+    MAX_PROTOCOL_MESSAGE_LEN,
+};
+
+// Allow direct serialization of Vec<u8> for these tests. We don't usually
+// allow this because some types have specific rules for about serialization
+// of their inner Vec<u8>. This method could be easily misused if it applied
+// more generally.
+impl ZcashSerialize for u8 {
+    fn zcash_serialize<W: std::io::Write>(&self, mut writer: W) -> Result<(), std::io::Error> {
+        writer.write_all(&[*self])
+    }
+}
+
+proptest! {
+#![proptest_config(ProptestConfig::with_cases(4))]
+
+    #[test]
+    /// Confirm that deserialize yields the expected result for any vec smaller than `MAX_U8_ALLOCATION`
+    fn u8_ser_deser_roundtrip(input in any_with::<Vec<u8>>(size_range(MAX_U8_ALLOCATION).lift()) ) {
+        let serialized = input.zcash_serialize_to_vec().expect("Serialization to vec must succeed");
+        let cursor = std::io::Cursor::new(serialized);
+        let deserialized = <Vec<u8>>::zcash_deserialize(cursor).expect("deserialization from vec must succeed");
+        prop_assert_eq!(deserialized, input)
+    }
+}
+
+#[test]
+/// Confirm that deserialize allows vectors with length up to and including `MAX_U8_ALLOCATION`
+fn u8_deser_accepts_max_valid_input() {
+    let serialized = vec![0u8; MAX_U8_ALLOCATION]
+        .zcash_serialize_to_vec()
+        .expect("Serialization to vec must succeed");
+    let cursor = std::io::Cursor::new(serialized);
+    let deserialized = <Vec<u8>>::zcash_deserialize(cursor);
+    assert!(deserialized.is_ok())
+}
+
+#[test]
+/// Confirm that rejects vectors longer than `MAX_U8_ALLOCATION`
+fn u8_deser_throws_when_input_too_large() {
+    let serialized = vec![0u8; MAX_U8_ALLOCATION + 1]
+        .zcash_serialize_to_vec()
+        .expect("Serialization to vec must succeed");
+    let cursor = std::io::Cursor::new(serialized);
+    let deserialized = <Vec<u8>>::zcash_deserialize(cursor);
+
+    assert!(matches!(
+        deserialized,
+        Err(SerializationError::Parse(
+            "Byte vector longer than MAX_U8_ALLOCATION"
+        ))
+    ))
+}
+
+#[test]
+/// Confirm that every u8 takes exactly 1 byte when serialized.
+/// This verifies that our calculated `MAX_U8_ALLOCATION` is indeed an upper bound.
+fn u8_size_is_correct() {
+    for byte in std::u8::MIN..=std::u8::MAX {
+        let serialized = byte
+            .zcash_serialize_to_vec()
+            .expect("Serialization to vec must succeed");
+        assert!(serialized.len() == 1)
+    }
+}
+
+#[test]
+/// Verify that...
+/// 1. The smallest disallowed `Vec<u8>` is too big to include in a Zcash Wire Protocol message
+/// 2. The largest allowed `Vec<u8>`is exactly the size of a maximal Zcash Wire Protocol message
+fn u8_max_allocation_is_correct() {
+    let mut shortest_disallowed_vec = vec![0u8; MAX_U8_ALLOCATION + 1];
+    let shortest_disallowed_serialized = shortest_disallowed_vec
+        .zcash_serialize_to_vec()
+        .expect("Serialization to vec must succeed");
+
+    // Confirm that shortest_disallowed_vec is only one item larger than the limit
+    assert_eq!((shortest_disallowed_vec.len() - 1), MAX_U8_ALLOCATION);
+    // Confirm that shortest_disallowed_vec is too large to be included in a valid zcash message
+    assert!(shortest_disallowed_serialized.len() > MAX_PROTOCOL_MESSAGE_LEN);
+
+    // Create largest_allowed_vec by removing one element from smallest_disallowed_vec without copying (for efficiency)
+    shortest_disallowed_vec.pop();
+    let longest_allowed_vec = shortest_disallowed_vec;
+    let longest_allowed_serialized = longest_allowed_vec
+        .zcash_serialize_to_vec()
+        .expect("serialization to vec must succed");
+
+    // Check that our largest_allowed_vec contains the maximum number of items
+    assert_eq!(longest_allowed_vec.len(), MAX_U8_ALLOCATION);
+    // Check that our largest_allowed_vec is the size of a maximal protocol message
+    assert_eq!(longest_allowed_serialized.len(), MAX_PROTOCOL_MESSAGE_LEN);
+}

zebra-chain/src/serialization/proptests.rs → zebra-chain/src/serialization/tests/prop.rs

@@ -1,14 +1,12 @@
 //! Property-based tests for basic serialization primitives.
 
-use super::*;
 use proptest::prelude::*;
 
 use std::io::Cursor;
 
-proptest! {
-    // The tests below are cheap so we can run them a lot.
-    #![proptest_config(ProptestConfig::with_cases(100_000))]
+use crate::serialization::{ReadZcashExt, WriteZcashExt};
 
+proptest! {
     #[test]
     fn compactsize_write_then_read_round_trip(s in 0u64..0x2_0000u64) {
         zebra_test::init();